Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/skeleton.py: 0%
964 statements
« prev ^ index » next coverage.py v7.6.12, created at 2025-02-27 07:59 +0000
1from __future__ import annotations # noqa: required for pre-defined annotations
3import copy
4import fnmatch
5import inspect
6import logging
7import os
8import string
9import sys
10import time
11import typing as t
12import warnings
13from functools import partial
14from itertools import chain
16from deprecated.sphinx import deprecated
18from viur.core import conf, current, db, email, errors, translate, utils
19from viur.core.bones import (
20 BaseBone,
21 DateBone,
22 KeyBone,
23 ReadFromClientException,
24 RelationalBone,
25 RelationalConsistency,
26 RelationalUpdateLevel,
27 SelectBone,
28 StringBone,
29)
30from viur.core.bones.base import (
31 Compute,
32 ComputeInterval,
33 ComputeMethod,
34 ReadFromClientError,
35 ReadFromClientErrorSeverity,
36 getSystemInitialized,
37)
38from viur.core.tasks import CallDeferred, CallableTask, CallableTaskBase, QueryIter
40_UNDEFINED = object()
41ABSTRACT_SKEL_CLS_SUFFIX = "AbstractSkel"
42KeyType: t.TypeAlias = db.Key | str | int
45class MetaBaseSkel(type):
46 """
47 This is the metaclass for Skeletons.
48 It is used to enforce several restrictions on bone names, etc.
49 """
50 _skelCache = {} # Mapping kindName -> SkelCls
51 _allSkelClasses = set() # list of all known skeleton classes (including Ref and Mail-Skels)
53 # List of reserved keywords and function names
54 __reserved_keywords = {
55 "all",
56 "bounce",
57 "clone",
58 "cursor",
59 "delete",
60 "errors",
61 "fromClient",
62 "fromDB",
63 "get",
64 "getCurrentSEOKeys",
65 "items",
66 "keys",
67 "limit",
68 "orderby",
69 "orderdir",
70 "patch",
71 "postDeletedHandler",
72 "postSavedHandler",
73 "preProcessBlobLocks",
74 "preProcessSerializedData",
75 "read",
76 "refresh",
77 "self",
78 "serialize",
79 "setBoneValue",
80 "structure",
81 "style",
82 "toDB",
83 "unserialize",
84 "values",
85 "write",
86 }
88 __allowed_chars = string.ascii_letters + string.digits + "_"
90 def __init__(cls, name, bases, dct, **kwargs):
91 cls.__boneMap__ = MetaBaseSkel.generate_bonemap(cls)
93 if not getSystemInitialized() and not cls.__name__.endswith(ABSTRACT_SKEL_CLS_SUFFIX):
94 MetaBaseSkel._allSkelClasses.add(cls)
96 super().__init__(name, bases, dct)
98 @staticmethod
99 def generate_bonemap(cls):
100 """
101 Recursively constructs a dict of bones from
102 """
103 map = {}
105 for base in cls.__bases__:
106 if "__viurBaseSkeletonMarker__" in dir(base):
107 map |= MetaBaseSkel.generate_bonemap(base)
109 for key in cls.__dict__:
110 prop = getattr(cls, key)
112 if isinstance(prop, BaseBone):
113 if not all([c in MetaBaseSkel.__allowed_chars for c in key]):
114 raise AttributeError(f"Invalid bone name: {key!r} contains invalid characters")
115 elif key in MetaBaseSkel.__reserved_keywords:
116 raise AttributeError(f"Invalid bone name: {key!r} is reserved and cannot be used")
118 map[key] = prop
120 elif prop is None and key in map: # Allow removing a bone in a subclass by setting it to None
121 del map[key]
123 return map
125 def __setattr__(self, key, value):
126 super().__setattr__(key, value)
127 if isinstance(value, BaseBone):
128 # Call BaseBone.__set_name__ manually for bones that are assigned at runtime
129 value.__set_name__(self, key)
132class SkeletonInstance:
133 """
134 The actual wrapper around a Skeleton-Class. An object of this class is what's actually returned when you
135 call a Skeleton-Class. With ViUR3, you don't get an instance of a Skeleton-Class any more - it's always this
136 class. This is much faster as this is a small class.
137 """
138 __slots__ = {
139 "accessedValues",
140 "boneMap",
141 "dbEntity",
142 "errors",
143 "is_cloned",
144 "renderAccessedValues",
145 "renderPreparation",
146 "skeletonCls",
147 }
149 def __init__(
150 self,
151 skel_cls: t.Type[Skeleton],
152 *,
153 bones: t.Iterable[str] = (),
154 bone_map: t.Optional[t.Dict[str, BaseBone]] = None,
155 clone: bool = False,
156 # FIXME: BELOW IS DEPRECATED!
157 clonedBoneMap: t.Optional[t.Dict[str, BaseBone]] = None,
158 ):
159 """
160 Creates a new SkeletonInstance based on `skel_cls`.
162 :param skel_cls: Is the base skeleton class to inherit from and reference to.
163 :param bones: If given, defines an iterable of bones that are take into the SkeletonInstance.
164 The order of the bones defines the order in the SkeletonInstance.
165 :param bone_map: A pre-defined bone map to use, or extend.
166 :param clone: If set True, performs a cloning of the used bone map, to be entirely stand-alone.
167 """
169 # TODO: Remove with ViUR-core 3.8; required by viur-datastore :'-(
170 if clonedBoneMap:
171 msg = "'clonedBoneMap' was renamed into 'bone_map'"
172 warnings.warn(msg, DeprecationWarning, stacklevel=2)
173 # logging.warning(msg, stacklevel=2)
175 if bone_map:
176 raise ValueError("Can't provide both 'bone_map' and 'clonedBoneMap'")
178 bone_map = clonedBoneMap
180 bone_map = bone_map or {}
182 if bones:
183 names = ("key", ) + tuple(bones)
185 # generate full keys sequence based on definition; keeps order of patterns!
186 keys = []
187 for name in names:
188 if name in skel_cls.__boneMap__:
189 keys.append(name)
190 else:
191 keys.extend(fnmatch.filter(skel_cls.__boneMap__.keys(), name))
193 if clone:
194 bone_map |= {k: copy.deepcopy(skel_cls.__boneMap__[k]) for k in keys if skel_cls.__boneMap__[k]}
195 else:
196 bone_map |= {k: skel_cls.__boneMap__[k] for k in keys if skel_cls.__boneMap__[k]}
198 elif clone:
199 if bone_map:
200 bone_map = copy.deepcopy(bone_map)
201 else:
202 bone_map = copy.deepcopy(skel_cls.__boneMap__)
204 # generated or use provided bone_map
205 if bone_map:
206 self.boneMap = bone_map
208 else: # No Subskel, no Clone
209 self.boneMap = skel_cls.__boneMap__.copy()
211 if clone:
212 for v in self.boneMap.values():
213 v.isClonedInstance = True
215 self.accessedValues = {}
216 self.dbEntity = None
217 self.errors = []
218 self.is_cloned = clone
219 self.renderAccessedValues = {}
220 self.renderPreparation = None
221 self.skeletonCls = skel_cls
223 def items(self, yieldBoneValues: bool = False) -> t.Iterable[tuple[str, BaseBone]]:
224 if yieldBoneValues:
225 for key in self.boneMap.keys():
226 yield key, self[key]
227 else:
228 yield from self.boneMap.items()
230 def keys(self) -> t.Iterable[str]:
231 yield from self.boneMap.keys()
233 def values(self) -> t.Iterable[t.Any]:
234 yield from self.boneMap.values()
236 def __iter__(self) -> t.Iterable[str]:
237 yield from self.keys()
239 def __contains__(self, item):
240 return item in self.boneMap
242 def get(self, item, default=None):
243 if item not in self:
244 return default
246 return self[item]
248 def update(self, *args, **kwargs) -> None:
249 self.__ior__(dict(*args, **kwargs))
251 def __setitem__(self, key, value):
252 assert self.renderPreparation is None, "Cannot modify values while rendering"
253 if isinstance(value, BaseBone):
254 raise AttributeError(f"Don't assign this bone object as skel[\"{key}\"] = ... anymore to the skeleton. "
255 f"Use skel.{key} = ... for bone to skeleton assignment!")
256 self.accessedValues[key] = value
258 def __getitem__(self, key):
259 if self.renderPreparation:
260 if key in self.renderAccessedValues:
261 return self.renderAccessedValues[key]
262 if key not in self.accessedValues:
263 boneInstance = self.boneMap.get(key, None)
264 if boneInstance:
265 if self.dbEntity is not None:
266 boneInstance.unserialize(self, key)
267 else:
268 self.accessedValues[key] = boneInstance.getDefaultValue(self)
269 if not self.renderPreparation:
270 return self.accessedValues.get(key)
271 value = self.renderPreparation(getattr(self, key), self, key, self.accessedValues.get(key))
272 self.renderAccessedValues[key] = value
273 return value
275 def __getattr__(self, item: str):
276 """
277 Get a special attribute from the SkeletonInstance
279 __getattr__ is called when an attribute access fails with an
280 AttributeError. So we know that this is not a real attribute of
281 the SkeletonInstance. But there are still a few special cases in which
282 attributes are loaded from the skeleton class.
283 """
284 if item == "boneMap":
285 return {} # There are __setAttr__ calls before __init__ has run
287 # Load attribute value from the Skeleton class
288 elif item in {
289 "database_adapters",
290 "interBoneValidations",
291 "kindName",
292 }:
293 return getattr(self.skeletonCls, item)
295 # FIXME: viur-datastore backward compatiblity REMOVE WITH VIUR4
296 elif item == "customDatabaseAdapter":
297 if prop := getattr(self.skeletonCls, "database_adapters"):
298 return prop[0] # viur-datastore assumes there is only ONE!
300 return None
302 # Load a @classmethod from the Skeleton class and bound this SkeletonInstance
303 elif item in {
304 "all",
305 "delete",
306 "patch",
307 "fromClient",
308 "fromDB",
309 "getCurrentSEOKeys",
310 "postDeletedHandler",
311 "postSavedHandler",
312 "preProcessBlobLocks",
313 "preProcessSerializedData",
314 "read",
315 "refresh",
316 "serialize",
317 "setBoneValue",
318 "toDB",
319 "unserialize",
320 "write",
321 }:
322 return partial(getattr(self.skeletonCls, item), self)
324 # Load a @property from the Skeleton class
325 try:
326 # Use try/except to save an if check
327 class_value = getattr(self.skeletonCls, item)
329 except AttributeError:
330 # Not inside the Skeleton class, okay at this point.
331 pass
333 else:
334 if isinstance(class_value, property):
335 # The attribute is a @property and can be called
336 # Note: `self` is this SkeletonInstance, not the Skeleton class.
337 # Therefore, you can access values inside the property method
338 # with item-access like `self["key"]`.
339 try:
340 return class_value.fget(self)
341 except AttributeError as exc:
342 # The AttributeError cannot be re-raised any further at this point.
343 # Since this would then be evaluated as an access error
344 # to the property attribute.
345 # Otherwise, it would be lost that it is an incorrect attribute access
346 # within this property (during the method call).
347 msg, *args = exc.args
348 msg = f"AttributeError: {msg}"
349 raise ValueError(msg, *args) from exc
350 # Load the bone instance from the bone map of this SkeletonInstance
351 try:
352 return self.boneMap[item]
353 except KeyError as exc:
354 raise AttributeError(f"{self.__class__.__name__!r} object has no attribute '{item}'") from exc
356 def __delattr__(self, item):
357 del self.boneMap[item]
358 if item in self.accessedValues:
359 del self.accessedValues[item]
360 if item in self.renderAccessedValues:
361 del self.renderAccessedValues[item]
363 def __setattr__(self, key, value):
364 if key in self.boneMap or isinstance(value, BaseBone):
365 if value is None:
366 del self.boneMap[key]
367 else:
368 value.__set_name__(self.skeletonCls, key)
369 self.boneMap[key] = value
370 elif key == "renderPreparation":
371 super().__setattr__(key, value)
372 self.renderAccessedValues.clear()
373 else:
374 super().__setattr__(key, value)
376 def __repr__(self) -> str:
377 return f"<SkeletonInstance of {self.skeletonCls.__name__} with {dict(self)}>"
379 def __str__(self) -> str:
380 return str(dict(self))
382 def __len__(self) -> int:
383 return len(self.boneMap)
385 def __ior__(self, other: dict | SkeletonInstance | db.Entity) -> SkeletonInstance:
386 if isinstance(other, dict):
387 for key, value in other.items():
388 self.setBoneValue(key, value)
389 elif isinstance(other, db.Entity):
390 new_entity = self.dbEntity or db.Entity()
391 # We're not overriding the key
392 for key, value in other.items():
393 new_entity[key] = value
394 self.setEntity(new_entity)
395 elif isinstance(other, SkeletonInstance):
396 for key, value in other.accessedValues.items():
397 self.accessedValues[key] = value
398 for key, value in other.dbEntity.items():
399 self.dbEntity[key] = value
400 else:
401 raise ValueError("Unsupported Type")
402 return self
404 def clone(self, *, apply_clone_strategy: bool = False) -> t.Self:
405 """
406 Clones a SkeletonInstance into a modificable, stand-alone instance.
407 This will also allow to modify the underlying data model.
408 """
409 res = SkeletonInstance(self.skeletonCls, bone_map=self.boneMap, clone=True)
410 if apply_clone_strategy:
411 for bone_name, bone_instance in self.items():
412 bone_instance.clone_value(res, self, bone_name)
413 else:
414 res.accessedValues = copy.deepcopy(self.accessedValues)
415 res.dbEntity = copy.deepcopy(self.dbEntity)
416 res.is_cloned = True
417 if not apply_clone_strategy:
418 res.renderAccessedValues = copy.deepcopy(self.renderAccessedValues)
419 # else: Depending on the strategy the values are cloned in bone_instance.clone_value too
421 return res
423 def ensure_is_cloned(self):
424 """
425 Ensured this SkeletonInstance is a stand-alone clone, which can be modified.
426 Does nothing in case it was already cloned before.
427 """
428 if not self.is_cloned:
429 return self.clone()
431 return self
433 def setEntity(self, entity: db.Entity):
434 self.dbEntity = entity
435 self.accessedValues = {}
436 self.renderAccessedValues = {}
438 def structure(self) -> dict:
439 return {
440 key: bone.structure() | {"sortindex": i}
441 for i, (key, bone) in enumerate(self.items())
442 }
444 def __deepcopy__(self, memodict):
445 res = self.clone()
446 memodict[id(self)] = res
447 return res
450class BaseSkeleton(object, metaclass=MetaBaseSkel):
451 """
452 This is a container-object holding information about one database entity.
454 It has to be sub-classed with individual information about the kindName of the entities
455 and its specific data attributes, the so called bones.
456 The Skeleton stores its bones in an :class:`OrderedDict`-Instance, so the definition order of the
457 contained bones remains constant.
459 :ivar key: This bone stores the current database key of this entity. \
460 Assigning to this bones value is dangerous and does *not* affect the actual key its stored in.
462 :vartype key: server.bones.BaseBone
464 :ivar creationdate: The date and time where this entity has been created.
465 :vartype creationdate: server.bones.DateBone
467 :ivar changedate: The date and time of the last change to this entity.
468 :vartype changedate: server.bones.DateBone
469 """
470 __viurBaseSkeletonMarker__ = True
471 boneMap = None
473 @classmethod
474 @deprecated(
475 version="3.7.0",
476 reason="Function renamed. Use subskel function as alternative implementation.",
477 )
478 def subSkel(cls, *subskel_names, fullClone: bool = False, **kwargs) -> SkeletonInstance:
479 return cls.subskel(*subskel_names, clone=fullClone) # FIXME: REMOVE WITH VIUR4
481 @classmethod
482 def subskel(
483 cls,
484 *names: str,
485 bones: t.Iterable[str] = (),
486 clone: bool = False,
487 ) -> SkeletonInstance:
488 """
489 Creates a new sub-skeleton from the current skeleton.
491 A sub-skeleton is a copy of the original skeleton, containing only a subset of its bones.
493 Sub-skeletons can either be defined using the the subSkels property of the Skeleton object,
494 or freely by giving patterns for bone names which shall be part of the sub-skeleton.
496 1. Giving names as parameter merges the bones of all Skeleton.subSkels-configurations together.
497 This is the usual behavior. By passing multiple sub-skeleton names to this function, a sub-skeleton
498 with the union of all bones of the specified sub-skeletons is returned. If an entry called "*"
499 exists in the subSkels-dictionary, the bones listed in this entry will always be part of the
500 generated sub-skeleton.
501 2. Given the *bones* parameter allows to freely specify a sub-skeleton; One specialty here is,
502 that the order of the bones can also be changed in this mode. This mode is the new way of defining
503 sub-skeletons, and might become the primary way to define sub-skeletons in future.
504 3. Both modes (1 + 2) can be combined, but then the original order of the bones is kept.
505 4. The "key" bone is automatically available in each sub-skeleton.
506 5. An fnmatch-compatible wildcard pattern is allowed both in the subSkels-bone-list and the
507 free bone list.
509 Example (TodoSkel is the example skeleton from viur-base):
510 ```py
511 # legacy mode (see 1)
512 subskel = TodoSkel.subskel("add")
513 # creates subskel: key, firstname, lastname, subject
515 # free mode (see 2) allows to specify a different order!
516 subskel = TodoSkel.subskel(bones=("subject", "message", "*stname"))
517 # creates subskel: key, subject, message, firstname, lastname
519 # mixed mode (see 3)
520 subskel = TodoSkel.subskel("add", bones=("message", ))
521 # creates subskel: key, firstname, lastname, subject, message
522 ```
524 :param bones: Allows to specify an iterator of bone names (more precisely, fnmatch-wildards) which allow
525 to freely define a subskel. If *only* this parameter is given, the order of the specification also
526 defines, the order of the list. Otherwise, the original order as defined in the skeleton is kept.
527 :param clone: If set True, performs a cloning of the used bone map, to be entirely stand-alone.
529 :return: The sub-skeleton of the specified type.
530 """
531 from_subskel = False
532 bones = list(bones)
534 for name in names:
535 # a str refers to a subskel name from the cls.subSkel dict
536 if isinstance(name, str):
537 # add bones from "*" subskel once
538 if not from_subskel:
539 bones.extend(cls.subSkels.get("*") or ())
540 from_subskel = True
542 bones.extend(cls.subSkels.get(name) or ())
544 else:
545 raise ValueError(f"Invalid subskel definition: {name!r}")
547 if from_subskel:
548 # when from_subskel is True, create bone names based on the order of the bones in the original skeleton
549 bones = tuple(k for k in cls.__boneMap__.keys() if any(fnmatch.fnmatch(k, n) for n in bones))
551 if not bones:
552 raise ValueError("The given subskel definition doesn't contain any bones!")
554 return cls(bones=bones, clone=clone)
556 @classmethod
557 def setSystemInitialized(cls):
558 for attrName in dir(cls):
559 bone = getattr(cls, attrName)
560 if isinstance(bone, BaseBone):
561 bone.setSystemInitialized()
563 @classmethod
564 def setBoneValue(
565 cls,
566 skel: SkeletonInstance,
567 boneName: str,
568 value: t.Any,
569 append: bool = False,
570 language: t.Optional[str] = None
571 ) -> bool:
572 """
573 Allows for setting a bones value without calling fromClient or assigning a value directly.
574 Sanity-Checks are performed; if the value is invalid, that bone flips back to its original
575 (default) value and false is returned.
577 :param boneName: The name of the bone to be modified
578 :param value: The value that should be assigned. It's type depends on the type of that bone
579 :param append: If True, the given value is appended to the values of that bone instead of
580 replacing it. Only supported on bones with multiple=True
581 :param language: Language to set
583 :return: Wherever that operation succeeded or not.
584 """
585 bone = getattr(skel, boneName, None)
587 if not isinstance(bone, BaseBone):
588 raise ValueError(f"{boneName!r} is no valid bone on this skeleton ({skel!r})")
590 if language:
591 if not bone.languages:
592 raise ValueError("The bone {boneName!r} has no language setting")
593 elif language not in bone.languages:
594 raise ValueError("The language {language!r} is not available for bone {boneName!r}")
596 if value is None:
597 if append:
598 raise ValueError("Cannot append None-value to bone {boneName!r}")
600 if language:
601 skel[boneName][language] = [] if bone.multiple else None
602 else:
603 skel[boneName] = [] if bone.multiple else None
605 return True
607 _ = skel[boneName] # ensure the bone is being unserialized first
608 return bone.setBoneValue(skel, boneName, value, append, language)
610 @classmethod
611 def fromClient(
612 cls,
613 skel: SkeletonInstance,
614 data: dict[str, list[str] | str],
615 *,
616 amend: bool = False,
617 ignore: t.Optional[t.Iterable[str]] = None,
618 ) -> bool:
619 """
620 Load supplied *data* into Skeleton.
622 This function works similar to :func:`~viur.core.skeleton.Skeleton.setValues`, except that
623 the values retrieved from *data* are checked against the bones and their validity checks.
625 Even if this function returns False, all bones are guaranteed to be in a valid state.
626 The ones which have been read correctly are set to their valid values;
627 Bones with invalid values are set back to a safe default (None in most cases).
628 So its possible to call :func:`~viur.core.skeleton.Skeleton.write` afterwards even if reading
629 data with this function failed (through this might violates the assumed consistency-model).
631 :param skel: The skeleton instance to be filled.
632 :param data: Dictionary from which the data is read.
633 :param amend: Defines whether content of data may be incomplete to amend the skel,
634 which is useful for edit-actions.
635 :param ignore: optional list of bones to be ignored; Defaults to all readonly-bones when set to None.
637 :returns: True if all data was successfully read and complete. \
638 False otherwise (e.g. some required fields where missing or where invalid).
639 """
640 complete = True
641 skel.errors = []
643 for key, bone in skel.items():
644 if (ignore is None and bone.readOnly) or key in (ignore or ()):
645 continue
647 if errors := bone.fromClient(skel, key, data):
648 for error in errors:
649 # insert current bone name into error's fieldPath
650 error.fieldPath.insert(0, str(key))
652 # logging.debug(f"BaseSkel.fromClient {key=} {error=}")
654 incomplete = (
655 # always when something is invalid
656 error.severity == ReadFromClientErrorSeverity.Invalid
657 or (
658 # only when path is top-level
659 len(error.fieldPath) == 1
660 and (
661 # bone is generally required
662 bool(bone.required)
663 and (
664 # and value is either empty
665 error.severity == ReadFromClientErrorSeverity.Empty
666 # or when not amending, not set
667 or (not amend and error.severity == ReadFromClientErrorSeverity.NotSet)
668 )
669 )
670 )
671 )
673 # in case there are language requirements, test additionally
674 if bone.languages and isinstance(bone.required, (list, tuple)):
675 incomplete &= any([key, lang] == error.fieldPath for lang in bone.required)
677 # logging.debug(f"BaseSkel.fromClient {incomplete=} {error.severity=} {bone.required=}")
679 if incomplete:
680 complete = False
682 if conf.debug.skeleton_from_client:
683 logging.error(
684 f"""{getattr(cls, "kindName", cls.__name__)}: {".".join(error.fieldPath)}: """
685 f"""({error.severity}) {error.errorMessage}"""
686 )
688 skel.errors += errors
690 return complete
692 @classmethod
693 def refresh(cls, skel: SkeletonInstance):
694 """
695 Refresh the bones current content.
697 This function causes a refresh of all relational bones and their associated
698 information.
699 """
700 logging.debug(f"""Refreshing {skel["key"]!r} ({skel.get("name")!r})""")
702 for key, bone in skel.items():
703 if not isinstance(bone, BaseBone):
704 continue
706 _ = skel[key] # Ensure value gets loaded
707 bone.refresh(skel, key)
709 def __new__(cls, *args, **kwargs) -> SkeletonInstance:
710 return SkeletonInstance(cls, *args, **kwargs)
713class MetaSkel(MetaBaseSkel):
715 def __init__(cls, name, bases, dct, **kwargs):
716 super().__init__(name, bases, dct, **kwargs)
718 relNewFileName = inspect.getfile(cls) \
719 .replace(str(conf.instance.project_base_path), "") \
720 .replace(str(conf.instance.core_base_path), "")
722 # Check if we have an abstract skeleton
723 if cls.__name__.endswith(ABSTRACT_SKEL_CLS_SUFFIX):
724 # Ensure that it doesn't have a kindName
725 assert cls.kindName is _UNDEFINED or cls.kindName is None, "Abstract Skeletons can't have a kindName"
726 # Prevent any further processing by this class; it has to be sub-classed before it can be used
727 return
729 # Automatic determination of the kindName, if the class is not part of viur.core.
730 if (cls.kindName is _UNDEFINED
731 and not relNewFileName.strip(os.path.sep).startswith("viur")
732 and not "viur_doc_build" in dir(sys)):
733 if cls.__name__.endswith("Skel"):
734 cls.kindName = cls.__name__.lower()[:-4]
735 else:
736 cls.kindName = cls.__name__.lower()
738 # Try to determine which skeleton definition takes precedence
739 if cls.kindName and cls.kindName is not _UNDEFINED and cls.kindName in MetaBaseSkel._skelCache:
740 relOldFileName = inspect.getfile(MetaBaseSkel._skelCache[cls.kindName]) \
741 .replace(str(conf.instance.project_base_path), "") \
742 .replace(str(conf.instance.core_base_path), "")
743 idxOld = min(
744 [x for (x, y) in enumerate(conf.skeleton_search_path) if relOldFileName.startswith(y)] + [999])
745 idxNew = min(
746 [x for (x, y) in enumerate(conf.skeleton_search_path) if relNewFileName.startswith(y)] + [999])
747 if idxNew == 999:
748 # We could not determine a priority for this class as its from a path not listed in the config
749 raise NotImplementedError(
750 "Skeletons must be defined in a folder listed in conf.skeleton_search_path")
751 elif idxOld < idxNew: # Lower index takes precedence
752 # The currently processed skeleton has a lower priority than the one we already saw - just ignore it
753 return
754 elif idxOld > idxNew:
755 # The currently processed skeleton has a higher priority, use that from now
756 MetaBaseSkel._skelCache[cls.kindName] = cls
757 else: # They seem to be from the same Package - raise as something is messed up
758 raise ValueError(f"Duplicate definition for {cls.kindName} in {relNewFileName} and {relOldFileName}")
760 # Ensure that all skeletons are defined in folders listed in conf.skeleton_search_path
761 if (not any([relNewFileName.startswith(x) for x in conf.skeleton_search_path])
762 and not "viur_doc_build" in dir(sys)): # Do not check while documentation build
763 raise NotImplementedError(
764 f"""{relNewFileName} must be defined in a folder listed in {conf.skeleton_search_path}""")
766 if cls.kindName and cls.kindName is not _UNDEFINED:
767 MetaBaseSkel._skelCache[cls.kindName] = cls
769 # Auto-Add ViUR Search Tags Adapter if the skeleton has no adapter attached
770 if cls.database_adapters is _UNDEFINED:
771 cls.database_adapters = ViurTagsSearchAdapter()
773 # Always ensure that skel.database_adapters is an iterable
774 cls.database_adapters = utils.ensure_iterable(cls.database_adapters)
777class DatabaseAdapter:
778 """
779 Adapter class used to bind or use other databases and hook operations when working with a Skeleton.
780 """
782 providesFulltextSearch: bool = False
783 """Set to True if we can run a fulltext search using this database."""
785 fulltextSearchGuaranteesQueryConstrains = False
786 """Are results returned by `meth:fulltextSearch` guaranteed to also match the databaseQuery"""
788 providesCustomQueries: bool = False
789 """Indicate that we can run more types of queries than originally supported by datastore"""
791 def prewrite(self, skel: SkeletonInstance, is_add: bool, change_list: t.Iterable[str] = ()):
792 """
793 Hook being called on a add, edit or delete operation before the skeleton-specific action is performed.
795 The hook can be used to modifiy the skeleton before writing.
796 The raw entity can be obainted using `skel.dbEntity`.
798 :param action: Either contains "add", "edit" or "delete", depending on the operation.
799 :param skel: is the skeleton that is being read before written.
800 :param change_list: is a list of bone names which are being changed within the write.
801 """
802 pass
804 def write(self, skel: SkeletonInstance, is_add: bool, change_list: t.Iterable[str] = ()):
805 """
806 Hook being called on a write operations after the skeleton is written.
808 The raw entity can be obainted using `skel.dbEntity`.
810 :param action: Either contains "add" or "edit", depending on the operation.
811 :param skel: is the skeleton that is being read before written.
812 :param change_list: is a list of bone names which are being changed within the write.
813 """
814 pass
816 def delete(self, skel: SkeletonInstance):
817 """
818 Hook being called on a delete operation after the skeleton is deleted.
819 """
820 pass
822 def fulltextSearch(self, queryString: str, databaseQuery: db.Query) -> list[db.Entity]:
823 """
824 If this database supports fulltext searches, this method has to implement them.
825 If it's a plain fulltext search engine, leave 'prop:fulltextSearchGuaranteesQueryConstrains' set to False,
826 then the server will post-process the list of entries returned from this function and drop any entry that
827 cannot be returned due to other constrains set in 'param:databaseQuery'. If you can obey *every* constrain
828 set in that Query, we can skip this post-processing and save some CPU-cycles.
829 :param queryString: the string as received from the user (no quotation or other safety checks applied!)
830 :param databaseQuery: The query containing any constrains that returned entries must also match
831 :return:
832 """
833 raise NotImplementedError
836class ViurTagsSearchAdapter(DatabaseAdapter):
837 """
838 This Adapter implements a simple fulltext search on top of the datastore.
840 On skel.write(), all words from String-/TextBones are collected with all *min_length* postfixes and dumped
841 into the property `viurTags`. When queried, we'll run a prefix-match against this property - thus returning
842 entities with either an exact match or a match within a word.
844 Example:
845 For the word "hello" we'll write "hello", "ello" and "llo" into viurTags.
846 When queried with "hello" we'll have an exact match.
847 When queried with "hel" we'll match the prefix for "hello"
848 When queried with "ell" we'll prefix-match "ello" - this is only enabled when substring_matching is True.
850 We'll automatically add this adapter if a skeleton has no other database adapter defined.
851 """
852 providesFulltextSearch = True
853 fulltextSearchGuaranteesQueryConstrains = True
855 def __init__(self, min_length: int = 2, max_length: int = 50, substring_matching: bool = False):
856 super().__init__()
857 self.min_length = min_length
858 self.max_length = max_length
859 self.substring_matching = substring_matching
861 def _tags_from_str(self, value: str) -> set[str]:
862 """
863 Extract all words including all min_length postfixes from given string
864 """
865 res = set()
867 for tag in value.split(" "):
868 tag = "".join([x for x in tag.lower() if x in conf.search_valid_chars])
870 if len(tag) >= self.min_length:
871 res.add(tag)
873 if self.substring_matching:
874 for i in range(1, 1 + len(tag) - self.min_length):
875 res.add(tag[i:])
877 return res
879 def prewrite(self, skel: SkeletonInstance, *args, **kwargs):
880 """
881 Collect searchTags from skeleton and build viurTags
882 """
883 tags = set()
885 for name, bone in skel.items():
886 if bone.searchable:
887 tags = tags.union(bone.getSearchTags(skel, name))
889 skel.dbEntity["viurTags"] = list(
890 chain(*[self._tags_from_str(tag) for tag in tags if len(tag) <= self.max_length])
891 )
893 def fulltextSearch(self, queryString: str, databaseQuery: db.Query) -> list[db.Entity]:
894 """
895 Run a fulltext search
896 """
897 keywords = list(self._tags_from_str(queryString))[:10]
898 resultScoreMap = {}
899 resultEntryMap = {}
901 for keyword in keywords:
902 qryBase = databaseQuery.clone()
903 for entry in qryBase.filter("viurTags >=", keyword).filter("viurTags <", keyword + "\ufffd").run():
904 if not entry.key in resultScoreMap:
905 resultScoreMap[entry.key] = 1
906 else:
907 resultScoreMap[entry.key] += 1
908 if not entry.key in resultEntryMap:
909 resultEntryMap[entry.key] = entry
911 resultList = [(k, v) for k, v in resultScoreMap.items()]
912 resultList.sort(key=lambda x: x[1], reverse=True)
914 return [resultEntryMap[x[0]] for x in resultList[:databaseQuery.queries.limit]]
917class SeoKeyBone(StringBone):
918 """
919 Special kind of StringBone saving its contents as `viurCurrentSeoKeys` into the entity's `viur` dict.
920 """
922 def unserialize(self, skel: SkeletonInstance, name: str) -> bool:
923 try:
924 skel.accessedValues[name] = skel.dbEntity["viur"]["viurCurrentSeoKeys"]
925 except KeyError:
926 skel.accessedValues[name] = self.getDefaultValue(skel)
928 def serialize(self, skel: SkeletonInstance, name: str, parentIndexed: bool) -> bool:
929 # Serialize also to skel["viur"]["viurCurrentSeoKeys"], so we can use this bone in relations
930 if name in skel.accessedValues:
931 newVal = skel.accessedValues[name]
932 if not skel.dbEntity.get("viur"):
933 skel.dbEntity["viur"] = db.Entity()
934 res = db.Entity()
935 res["_viurLanguageWrapper_"] = True
936 for language in (self.languages or []):
937 if not self.indexed:
938 res.exclude_from_indexes.add(language)
939 res[language] = None
940 if language in newVal:
941 res[language] = self.singleValueSerialize(newVal[language], skel, name, parentIndexed)
942 skel.dbEntity["viur"]["viurCurrentSeoKeys"] = res
943 return True
946class Skeleton(BaseSkeleton, metaclass=MetaSkel):
947 kindName: str = _UNDEFINED
948 """
949 Specifies the entity kind name this Skeleton is associated with.
950 Will be determined automatically when not explicitly set.
951 """
953 database_adapters: DatabaseAdapter | t.Iterable[DatabaseAdapter] | None = _UNDEFINED
954 """
955 Custom database adapters.
956 Allows to hook special functionalities that during skeleton modifications.
957 """
959 subSkels = {} # List of pre-defined sub-skeletons of this type
961 interBoneValidations: list[
962 t.Callable[[Skeleton], list[ReadFromClientError]]] = [] # List of functions checking inter-bone dependencies
964 __seo_key_trans = str.maketrans(
965 {"<": "",
966 ">": "",
967 "\"": "",
968 "'": "",
969 "\n": "",
970 "\0": "",
971 "/": "",
972 "\\": "",
973 "?": "",
974 "&": "",
975 "#": ""
976 })
978 # The "key" bone stores the current database key of this skeleton.
979 # Warning: Assigning to this bones value now *will* set the key
980 # it gets stored in. Must be kept readOnly to avoid security-issues with add/edit.
981 key = KeyBone(
982 descr="Key"
983 )
985 name = StringBone(
986 descr="Name",
987 visible=False,
988 compute=Compute(
989 fn=lambda skel: str(skel["key"]),
990 interval=ComputeInterval(ComputeMethod.OnWrite)
991 )
992 )
994 # The date (including time) when this entry has been created
995 creationdate = DateBone(
996 descr="created at",
997 readOnly=True,
998 visible=False,
999 indexed=True,
1000 compute=Compute(fn=utils.utcNow, interval=ComputeInterval(ComputeMethod.Once)),
1001 )
1003 # The last date (including time) when this entry has been updated
1005 changedate = DateBone(
1006 descr="updated at",
1007 readOnly=True,
1008 visible=False,
1009 indexed=True,
1010 compute=Compute(fn=utils.utcNow, interval=ComputeInterval(ComputeMethod.OnWrite)),
1011 )
1013 viurCurrentSeoKeys = SeoKeyBone(
1014 descr="SEO-Keys",
1015 readOnly=True,
1016 visible=False,
1017 languages=conf.i18n.available_languages
1018 )
1020 def __repr__(self):
1021 return "<skeleton %s with data=%r>" % (self.kindName, {k: self[k] for k in self.keys()})
1023 def __str__(self):
1024 return str({k: self[k] for k in self.keys()})
1026 def __init__(self, *args, **kwargs):
1027 super(Skeleton, self).__init__(*args, **kwargs)
1028 assert self.kindName and self.kindName is not _UNDEFINED, "You must set kindName on this skeleton!"
1030 @classmethod
1031 def all(cls, skel, **kwargs) -> db.Query:
1032 """
1033 Create a query with the current Skeletons kindName.
1035 :returns: A db.Query object which allows for entity filtering and sorting.
1036 """
1037 return db.Query(skel.kindName, srcSkelClass=skel, **kwargs)
1039 @classmethod
1040 def fromClient(
1041 cls,
1042 skel: SkeletonInstance,
1043 data: dict[str, list[str] | str],
1044 *,
1045 amend: bool = False,
1046 ignore: t.Optional[t.Iterable[str]] = None,
1047 ) -> bool:
1048 """
1049 This function works similar to :func:`~viur.core.skeleton.Skeleton.setValues`, except that
1050 the values retrieved from *data* are checked against the bones and their validity checks.
1052 Even if this function returns False, all bones are guaranteed to be in a valid state.
1053 The ones which have been read correctly are set to their valid values;
1054 Bones with invalid values are set back to a safe default (None in most cases).
1055 So its possible to call :func:`~viur.core.skeleton.Skeleton.write` afterwards even if reading
1056 data with this function failed (through this might violates the assumed consistency-model).
1058 :param skel: The skeleton instance to be filled.
1059 :param data: Dictionary from which the data is read.
1060 :param amend: Defines whether content of data may be incomplete to amend the skel,
1061 which is useful for edit-actions.
1062 :param ignore: optional list of bones to be ignored; Defaults to all readonly-bones when set to None.
1064 :returns: True if all data was successfully read and complete. \
1065 False otherwise (e.g. some required fields where missing or where invalid).
1066 """
1067 assert skel.renderPreparation is None, "Cannot modify values while rendering"
1069 # Load data into this skeleton
1070 complete = bool(data) and super().fromClient(skel, data, amend=amend, ignore=ignore)
1072 if (
1073 not data # in case data is empty
1074 or (len(data) == 1 and "key" in data)
1075 or (utils.parse.bool(data.get("nomissing")))
1076 ):
1077 skel.errors = []
1079 # Check if all unique values are available
1080 for boneName, boneInstance in skel.items():
1081 if boneInstance.unique:
1082 lockValues = boneInstance.getUniquePropertyIndexValues(skel, boneName)
1083 for lockValue in lockValues:
1084 dbObj = db.Get(db.Key(f"{skel.kindName}_{boneName}_uniquePropertyIndex", lockValue))
1085 if dbObj and (not skel["key"] or dbObj["references"] != skel["key"].id_or_name):
1086 # This value is taken (sadly, not by us)
1087 complete = False
1088 errorMsg = boneInstance.unique.message
1089 skel.errors.append(
1090 ReadFromClientError(ReadFromClientErrorSeverity.Invalid, errorMsg, [boneName]))
1092 # Check inter-Bone dependencies
1093 for checkFunc in skel.interBoneValidations:
1094 errors = checkFunc(skel)
1095 if errors:
1096 for error in errors:
1097 if error.severity.value > 1:
1098 complete = False
1099 if conf.debug.skeleton_from_client:
1100 logging.debug(f"{cls.kindName}: {error.fieldPath}: {error.errorMessage!r}")
1102 skel.errors.extend(errors)
1104 return complete
1106 @classmethod
1107 @deprecated(
1108 version="3.7.0",
1109 reason="Use skel.read() instead of skel.fromDB()",
1110 )
1111 def fromDB(cls, skel: SkeletonInstance, key: KeyType) -> bool:
1112 """
1113 Deprecated function, replaced by Skeleton.read().
1114 """
1115 return bool(cls.read(skel, key, _check_legacy=False))
1117 @classmethod
1118 def read(
1119 cls,
1120 skel: SkeletonInstance,
1121 key: t.Optional[KeyType] = None,
1122 *,
1123 create: bool | dict | t.Callable[[SkeletonInstance], None] = False,
1124 _check_legacy: bool = True
1125 ) -> t.Optional[SkeletonInstance]:
1126 """
1127 Read Skeleton with *key* from the datastore into the Skeleton.
1128 If not key is given, skel["key"] will be used.
1130 Reads all available data of entity kind *kindName* and the key *key*
1131 from the Datastore into the Skeleton structure's bones. Any previous
1132 data of the bones will discard.
1134 To store a Skeleton object to the Datastore, see :func:`~viur.core.skeleton.Skeleton.write`.
1136 :param key: A :class:`viur.core.db.Key`, string, or int; from which the data shall be fetched.
1137 If not provided, skel["key"] will be used.
1138 :param create: Allows to specify a dict or initial callable that is executed in case the Skeleton with the
1139 given key does not exist, it will be created.
1141 :returns: None on error, or the given SkeletonInstance on success.
1143 """
1144 # FIXME VIUR4: Stay backward compatible, call sub-classed fromDB if available first!
1145 if _check_legacy and "fromDB" in cls.__dict__:
1146 with warnings.catch_warnings():
1147 warnings.simplefilter("ignore", DeprecationWarning)
1148 return cls.fromDB(skel, key=key)
1150 assert skel.renderPreparation is None, "Cannot modify values while rendering"
1152 try:
1153 db_key = db.keyHelper(key or skel["key"], skel.kindName)
1154 except (ValueError, NotImplementedError): # This key did not parse
1155 return None
1157 if db_res := db.Get(db_key):
1158 skel.setEntity(db_res)
1159 return skel
1160 elif create in (False, None):
1161 return None
1162 elif isinstance(create, dict):
1163 if create and not skel.fromClient(create, amend=True):
1164 raise ReadFromClientException(skel.errors)
1165 elif callable(create):
1166 create(skel)
1167 elif create is not True:
1168 raise ValueError("'create' must either be dict, a callable or True.")
1170 return skel.write()
1172 @classmethod
1173 @deprecated(
1174 version="3.7.0",
1175 reason="Use skel.write() instead of skel.toDB()",
1176 )
1177 def toDB(cls, skel: SkeletonInstance, update_relations: bool = True, **kwargs) -> db.Key:
1178 """
1179 Deprecated function, replaced by Skeleton.write().
1180 """
1182 # TODO: Remove with ViUR4
1183 if "clearUpdateTag" in kwargs:
1184 msg = "clearUpdateTag was replaced by update_relations"
1185 warnings.warn(msg, DeprecationWarning, stacklevel=3)
1186 logging.warning(msg, stacklevel=3)
1187 update_relations = not kwargs["clearUpdateTag"]
1189 skel = cls.write(skel, update_relations=update_relations, _check_legacy=False)
1190 return skel["key"]
1192 @classmethod
1193 def write(
1194 cls,
1195 skel: SkeletonInstance,
1196 key: t.Optional[KeyType] = None,
1197 *,
1198 update_relations: bool = True,
1199 _check_legacy: bool = True,
1200 ) -> SkeletonInstance:
1201 """
1202 Write current Skeleton to the datastore.
1204 Stores the current data of this instance into the database.
1205 If an *key* value is set to the object, this entity will ne updated;
1206 Otherwise a new entity will be created.
1208 To read a Skeleton object from the data store, see :func:`~viur.core.skeleton.Skeleton.read`.
1210 :param key: Allows to specify a key that is set to the skeleton and used for writing.
1211 :param update_relations: If False, this entity won't be marked dirty;
1212 This avoids from being fetched by the background task updating relations.
1214 :returns: The Skeleton.
1215 """
1216 # FIXME VIUR4: Stay backward compatible, call sub-classed toDB if available first!
1217 if _check_legacy and "toDB" in cls.__dict__:
1218 with warnings.catch_warnings():
1219 warnings.simplefilter("ignore", DeprecationWarning)
1220 return cls.toDB(skel, update_relations=update_relations)
1222 assert skel.renderPreparation is None, "Cannot modify values while rendering"
1224 def __txn_write(write_skel):
1225 db_key = write_skel["key"]
1226 skel = write_skel.skeletonCls()
1228 blob_list = set()
1229 change_list = []
1230 old_copy = {}
1231 # Load the current values from Datastore or create a new, empty db.Entity
1232 if not db_key:
1233 # We'll generate the key we'll be stored under early so we can use it for locks etc
1234 db_key = db.AllocateIDs(db.Key(skel.kindName))
1235 skel.dbEntity = db.Entity(db_key)
1236 is_add = True
1237 else:
1238 db_key = db.keyHelper(db_key, skel.kindName)
1239 if db_obj := db.Get(db_key):
1240 skel.dbEntity = db_obj
1241 old_copy = {k: v for k, v in skel.dbEntity.items()}
1242 is_add = False
1243 else:
1244 skel.dbEntity = db.Entity(db_key)
1245 is_add = True
1247 skel.dbEntity.setdefault("viur", {})
1249 # Merge values and assemble unique properties
1250 # Move accessed Values from srcSkel over to skel
1251 skel.accessedValues = write_skel.accessedValues
1253 write_skel["key"] = skel["key"] = db_key # Ensure key stays set
1254 write_skel.dbEntity = skel.dbEntity # update write_skel's dbEntity
1256 for bone_name, bone in skel.items():
1257 if bone_name == "key": # Explicitly skip key on top-level - this had been set above
1258 continue
1260 # Allow bones to perform outstanding "magic" operations before saving to db
1261 bone.performMagic(skel, bone_name, isAdd=is_add) # FIXME VIUR4: ANY MAGIC IN OUR CODE IS DEPRECATED!!!
1263 if not (bone_name in skel.accessedValues or bone.compute) and bone_name not in skel.dbEntity:
1264 _ = skel[bone_name] # Ensure the datastore is filled with the default value
1266 if (
1267 bone_name in skel.accessedValues or bone.compute # We can have a computed value on store
1268 or bone_name not in skel.dbEntity # It has not been written and is not in the database
1269 ):
1270 # Serialize bone into entity
1271 try:
1272 bone.serialize(skel, bone_name, True)
1273 except Exception as e:
1274 logging.error(
1275 f"Failed to serialize {bone_name=} ({bone=}): {skel.accessedValues[bone_name]=}"
1276 )
1277 raise e
1279 # Obtain referenced blobs
1280 blob_list.update(bone.getReferencedBlobs(skel, bone_name))
1282 # Check if the value has actually changed
1283 if skel.dbEntity.get(bone_name) != old_copy.get(bone_name):
1284 change_list.append(bone_name)
1286 # Lock hashes from bones that must have unique values
1287 if bone.unique:
1288 # Remember old hashes for bones that must have an unique value
1289 old_unique_values = []
1291 if f"{bone_name}_uniqueIndexValue" in skel.dbEntity["viur"]:
1292 old_unique_values = skel.dbEntity["viur"][f"{bone_name}_uniqueIndexValue"]
1293 # Check if the property is unique
1294 new_unique_values = bone.getUniquePropertyIndexValues(skel, bone_name)
1295 new_lock_kind = f"{skel.kindName}_{bone_name}_uniquePropertyIndex"
1296 for new_lock_value in new_unique_values:
1297 new_lock_key = db.Key(new_lock_kind, new_lock_value)
1298 if lock_db_obj := db.Get(new_lock_key):
1300 # There's already a lock for that value, check if we hold it
1301 if lock_db_obj["references"] != skel.dbEntity.key.id_or_name:
1302 # This value has already been claimed, and not by us
1303 # TODO: Use a custom exception class which is catchable with an try/except
1304 raise ValueError(
1305 f"The unique value {skel[bone_name]!r} of bone {bone_name!r} "
1306 f"has been recently claimed (by {new_lock_key=}).")
1307 else:
1308 # This value is locked for the first time, create a new lock-object
1309 lock_obj = db.Entity(new_lock_key)
1310 lock_obj["references"] = skel.dbEntity.key.id_or_name
1311 db.Put(lock_obj)
1312 if new_lock_value in old_unique_values:
1313 old_unique_values.remove(new_lock_value)
1314 skel.dbEntity["viur"][f"{bone_name}_uniqueIndexValue"] = new_unique_values
1316 # Remove any lock-object we're holding for values that we don't have anymore
1317 for old_unique_value in old_unique_values:
1318 # Try to delete the old lock
1320 old_lock_key = db.Key(f"{skel.kindName}_{bone_name}_uniquePropertyIndex", old_unique_value)
1321 if old_lock_obj := db.Get(old_lock_key):
1322 if old_lock_obj["references"] != skel.dbEntity.key.id_or_name:
1324 # We've been supposed to have that lock - but we don't.
1325 # Don't remove that lock as it now belongs to a different entry
1326 logging.critical("Detected Database corruption! A Value-Lock had been reassigned!")
1327 else:
1328 # It's our lock which we don't need anymore
1329 db.Delete(old_lock_key)
1330 else:
1331 logging.critical("Detected Database corruption! Could not delete stale lock-object!")
1333 # Delete legacy property (PR #1244) #TODO: Remove in ViUR4
1334 skel.dbEntity.pop("viur_incomming_relational_locks", None)
1336 # Ensure the SEO-Keys are up-to-date
1337 last_requested_seo_keys = skel.dbEntity["viur"].get("viurLastRequestedSeoKeys") or {}
1338 last_set_seo_keys = skel.dbEntity["viur"].get("viurCurrentSeoKeys") or {}
1339 # Filter garbage serialized into this field by the SeoKeyBone
1340 last_set_seo_keys = {k: v for k, v in last_set_seo_keys.items() if not k.startswith("_") and v}
1342 if not isinstance(skel.dbEntity["viur"].get("viurCurrentSeoKeys"), dict):
1343 skel.dbEntity["viur"]["viurCurrentSeoKeys"] = {}
1345 if current_seo_keys := skel.getCurrentSEOKeys():
1346 # Convert to lower-case and remove certain characters
1347 for lang, value in current_seo_keys.items():
1348 current_seo_keys[lang] = value.lower().translate(Skeleton.__seo_key_trans).strip()
1350 for language in (conf.i18n.available_languages or [conf.i18n.default_language]):
1351 if current_seo_keys and language in current_seo_keys:
1352 current_seo_key = current_seo_keys[language]
1354 if current_seo_key != last_requested_seo_keys.get(language): # This one is new or has changed
1355 new_seo_key = current_seo_keys[language]
1357 for _ in range(0, 3):
1358 entry_using_key = db.Query(skel.kindName).filter(
1359 "viur.viurActiveSeoKeys =", new_seo_key).getEntry()
1361 if entry_using_key and entry_using_key.key != skel.dbEntity.key:
1362 # It's not unique; append a random string and try again
1363 new_seo_key = f"{current_seo_keys[language]}-{utils.string.random(5).lower()}"
1365 else:
1366 # We found a new SeoKey
1367 break
1368 else:
1369 raise ValueError("Could not generate an unique seo key in 3 attempts")
1370 else:
1371 new_seo_key = current_seo_key
1372 last_set_seo_keys[language] = new_seo_key
1374 else:
1375 # We'll use the database-key instead
1376 last_set_seo_keys[language] = str(skel.dbEntity.key.id_or_name)
1378 # Store the current, active key for that language
1379 skel.dbEntity["viur"]["viurCurrentSeoKeys"][language] = last_set_seo_keys[language]
1381 skel.dbEntity["viur"].setdefault("viurActiveSeoKeys", [])
1382 for language, seo_key in last_set_seo_keys.items():
1383 if skel.dbEntity["viur"]["viurCurrentSeoKeys"][language] not in \
1384 skel.dbEntity["viur"]["viurActiveSeoKeys"]:
1385 # Ensure the current, active seo key is in the list of all seo keys
1386 skel.dbEntity["viur"]["viurActiveSeoKeys"].insert(0, seo_key)
1387 if str(skel.dbEntity.key.id_or_name) not in skel.dbEntity["viur"]["viurActiveSeoKeys"]:
1388 # Ensure that key is also in there
1389 skel.dbEntity["viur"]["viurActiveSeoKeys"].insert(0, str(skel.dbEntity.key.id_or_name))
1390 # Trim to the last 200 used entries
1391 skel.dbEntity["viur"]["viurActiveSeoKeys"] = skel.dbEntity["viur"]["viurActiveSeoKeys"][:200]
1392 # Store lastRequestedKeys so further updates can run more efficient
1393 skel.dbEntity["viur"]["viurLastRequestedSeoKeys"] = current_seo_keys
1395 # mark entity as "dirty" when update_relations is set, to zero otherwise.
1396 skel.dbEntity["viur"]["delayedUpdateTag"] = time.time() if update_relations else 0
1398 skel.dbEntity = skel.preProcessSerializedData(skel.dbEntity)
1400 # Allow the database adapter to apply last minute changes to the object
1401 for adapter in skel.database_adapters:
1402 adapter.prewrite(skel, is_add, change_list)
1404 # ViUR2 import compatibility - remove properties containing. if we have a dict with the same name
1405 def fixDotNames(entity):
1406 for k, v in list(entity.items()):
1407 if isinstance(v, dict):
1408 for k2, v2 in list(entity.items()):
1409 if k2.startswith(f"{k}."):
1410 del entity[k2]
1411 backupKey = k2.replace(".", "__")
1412 entity[backupKey] = v2
1413 entity.exclude_from_indexes = set(entity.exclude_from_indexes) | {backupKey}
1414 fixDotNames(v)
1415 elif isinstance(v, list):
1416 for x in v:
1417 if isinstance(x, dict):
1418 fixDotNames(x)
1420 # FIXME: REMOVE IN VIUR4
1421 if conf.viur2import_blobsource: # Try to fix these only when converting from ViUR2
1422 fixDotNames(skel.dbEntity)
1424 # Write the core entry back
1425 db.Put(skel.dbEntity)
1427 # Now write the blob-lock object
1428 blob_list = skel.preProcessBlobLocks(blob_list)
1429 if blob_list is None:
1430 raise ValueError("Did you forget to return the blob_list somewhere inside getReferencedBlobs()?")
1431 if None in blob_list:
1432 msg = f"None is not valid in {blob_list=}"
1433 logging.error(msg)
1434 raise ValueError(msg)
1436 if not is_add and (old_blob_lock_obj := db.Get(db.Key("viur-blob-locks", db_key.id_or_name))):
1437 removed_blobs = set(old_blob_lock_obj.get("active_blob_references", [])) - blob_list
1438 old_blob_lock_obj["active_blob_references"] = list(blob_list)
1439 if old_blob_lock_obj["old_blob_references"] is None:
1440 old_blob_lock_obj["old_blob_references"] = list(removed_blobs)
1441 else:
1442 old_blob_refs = set(old_blob_lock_obj["old_blob_references"])
1443 old_blob_refs.update(removed_blobs) # Add removed blobs
1444 old_blob_refs -= blob_list # Remove active blobs
1445 old_blob_lock_obj["old_blob_references"] = list(old_blob_refs)
1447 old_blob_lock_obj["has_old_blob_references"] = bool(old_blob_lock_obj["old_blob_references"])
1448 old_blob_lock_obj["is_stale"] = False
1449 db.Put(old_blob_lock_obj)
1450 else: # We need to create a new blob-lock-object
1451 blob_lock_obj = db.Entity(db.Key("viur-blob-locks", skel.dbEntity.key.id_or_name))
1452 blob_lock_obj["active_blob_references"] = list(blob_list)
1453 blob_lock_obj["old_blob_references"] = []
1454 blob_lock_obj["has_old_blob_references"] = False
1455 blob_lock_obj["is_stale"] = False
1456 db.Put(blob_lock_obj)
1458 return skel.dbEntity.key, write_skel, change_list, is_add
1460 # Parse provided key, if any, and set it to skel["key"]
1461 if key:
1462 skel["key"] = db.keyHelper(key, skel.kindName)
1464 # Run transactional function
1465 if db.IsInTransaction():
1466 key, skel, change_list, is_add = __txn_write(skel)
1467 else:
1468 key, skel, change_list, is_add = db.RunInTransaction(__txn_write, skel)
1470 for bone_name, bone in skel.items():
1471 bone.postSavedHandler(skel, bone_name, key)
1473 skel.postSavedHandler(key, skel.dbEntity)
1475 if update_relations and not is_add:
1476 if change_list and len(change_list) < 5: # Only a few bones have changed, process these individually
1477 for idx, changed_bone in enumerate(change_list):
1478 updateRelations(key, time.time() + 1, changed_bone, _countdown=10 * idx)
1479 else: # Update all inbound relations, regardless of which bones they mirror
1480 updateRelations(key, time.time() + 1, None)
1482 # Trigger the database adapter of the changes made to the entry
1483 for adapter in skel.database_adapters:
1484 adapter.write(skel, is_add, change_list)
1486 return skel
1488 @classmethod
1489 def delete(cls, skel: SkeletonInstance, key: t.Optional[KeyType] = None) -> None:
1490 """
1491 Deletes the entity associated with the current Skeleton from the data store.
1493 :param key: Allows to specify a key that is used for deletion, otherwise skel["key"] will be used.
1494 """
1496 def __txn_delete(skel: SkeletonInstance, key: db.Key):
1497 if not skel.read(key):
1498 raise ValueError("This skeleton is not in the database (anymore?)!")
1500 # Is there any relation to this Skeleton which prevents the deletion?
1501 locked_relation = (
1502 db.Query("viur-relations")
1503 .filter("dest.__key__ =", key)
1504 .filter("viur_relational_consistency =", RelationalConsistency.PreventDeletion.value)
1505 ).getEntry()
1507 if locked_relation is not None:
1508 raise errors.Locked("This entry is still referenced by other Skeletons, which prevents deleting!")
1510 # Ensure that any value lock objects remaining for this entry are being deleted
1511 viur_data = skel.dbEntity.get("viur") or {}
1513 for boneName, bone in skel.items():
1514 bone.delete(skel, boneName)
1515 if bone.unique:
1516 flushList = []
1517 for lockValue in viur_data.get(f"{boneName}_uniqueIndexValue") or []:
1518 lockKey = db.Key(f"{skel.kindName}_{boneName}_uniquePropertyIndex", lockValue)
1519 lockObj = db.Get(lockKey)
1520 if not lockObj:
1521 logging.error(f"{lockKey=} missing!")
1522 elif lockObj["references"] != key.id_or_name:
1523 logging.error(
1524 f"""{key!r} does not hold lock for {lockKey!r}""")
1525 else:
1526 flushList.append(lockObj)
1527 if flushList:
1528 db.Delete(flushList)
1530 # Delete the blob-key lock object
1531 lockObjectKey = db.Key("viur-blob-locks", key.id_or_name)
1532 lockObj = db.Get(lockObjectKey)
1534 if lockObj is not None:
1535 if lockObj["old_blob_references"] is None and lockObj["active_blob_references"] is None:
1536 db.Delete(lockObjectKey) # Nothing to do here
1537 else:
1538 if lockObj["old_blob_references"] is None:
1539 # No old stale entries, move active_blob_references -> old_blob_references
1540 lockObj["old_blob_references"] = lockObj["active_blob_references"]
1541 elif lockObj["active_blob_references"] is not None:
1542 # Append the current references to the list of old & stale references
1543 lockObj["old_blob_references"] += lockObj["active_blob_references"]
1544 lockObj["active_blob_references"] = [] # There are no active ones left
1545 lockObj["is_stale"] = True
1546 lockObj["has_old_blob_references"] = True
1547 db.Put(lockObj)
1549 db.Delete(key)
1550 processRemovedRelations(key)
1552 if key := (key or skel["key"]):
1553 key = db.keyHelper(key, skel.kindName)
1554 else:
1555 raise ValueError("This skeleton has no key!")
1557 # Full skeleton is required to have all bones!
1558 skel = skeletonByKind(skel.kindName)()
1560 if db.IsInTransaction():
1561 __txn_delete(skel, key)
1562 else:
1563 db.RunInTransaction(__txn_delete, skel, key)
1565 for boneName, bone in skel.items():
1566 bone.postDeletedHandler(skel, boneName, key)
1568 skel.postDeletedHandler(key)
1570 # Inform the custom DB Adapter
1571 for adapter in skel.database_adapters:
1572 adapter.delete(skel)
1574 @classmethod
1575 def patch(
1576 cls,
1577 skel: SkeletonInstance,
1578 values: t.Optional[dict | t.Callable[[SkeletonInstance], None]] = {},
1579 *,
1580 key: t.Optional[db.Key | int | str] = None,
1581 check: t.Optional[dict | t.Callable[[SkeletonInstance], None]] = None,
1582 create: t.Optional[bool | dict | t.Callable[[SkeletonInstance], None]] = None,
1583 update_relations: bool = True,
1584 ignore: t.Optional[t.Iterable[str]] = (),
1585 retry: int = 0,
1586 ) -> SkeletonInstance:
1587 """
1588 Performs an edit operation on a Skeleton within a transaction.
1590 The transaction performs a read, sets bones and afterwards does a write with exclusive access on the
1591 given Skeleton and its underlying database entity.
1593 All value-dicts that are being fed to this function are provided to `skel.fromClient()`. Instead of dicts,
1594 a callable can also be given that can individually modify the Skeleton that is edited.
1596 :param values: A dict of key-values to update on the entry, or a callable that is executed within
1597 the transaction.
1599 This dict allows for a special notation: Keys starting with "+" or "-" are added or substracted to the
1600 given value, which can be used for counters.
1601 :param key: A :class:`viur.core.db.Key`, string, or int; from which the data shall be fetched.
1602 If not provided, skel["key"] will be used.
1603 :param check: An optional dict of key-values or a callable to check on the Skeleton before updating.
1604 If something fails within this check, an AssertionError is being raised.
1605 :param create: Allows to specify a dict or initial callable that is executed in case the Skeleton with the
1606 given key does not exist.
1607 :param update_relations: Trigger update relations task on success. Defaults to False.
1608 :param trust: Use internal `fromClient` with trusted data (may change readonly-bones)
1609 :param retry: On ViurDatastoreError, retry for this amount of times.
1611 If the function does not raise an Exception, all went well. The function always returns the input Skeleton.
1613 Raises:
1614 ValueError: In case parameters where given wrong or incomplete.
1615 AssertionError: In case an asserted check parameter did not match.
1616 ReadFromClientException: In case a skel.fromClient() failed with a high severity.
1617 """
1619 # Transactional function
1620 def __update_txn():
1621 # Try to read the skeleton, create on demand
1622 if not skel.read(key):
1623 if create is None or create is False:
1624 raise ValueError("Creation during update is forbidden - explicitly provide `create=True` to allow.")
1626 if not (key or skel["key"]) and create in (False, None):
1627 return ValueError("No valid key provided")
1629 if key or skel["key"]:
1630 skel["key"] = db.keyHelper(key or skel["key"], skel.kindName)
1632 if isinstance(create, dict):
1633 if create and not skel.fromClient(create, amend=True, ignore=ignore):
1634 raise ReadFromClientException(skel.errors)
1635 elif callable(create):
1636 create(skel)
1637 elif create is not True:
1638 raise ValueError("'create' must either be dict or a callable.")
1640 # Handle check
1641 if isinstance(check, dict):
1642 for bone, value in check.items():
1643 if skel[bone] != value:
1644 raise AssertionError(f"{bone} contains {skel[bone]!r}, expecting {value!r}")
1646 elif callable(check):
1647 check(skel)
1649 # Set values
1650 if isinstance(values, dict):
1651 if values and not skel.fromClient(values, amend=True, ignore=ignore):
1652 raise ReadFromClientException(skel.errors)
1654 # Special-feature: "+" and "-" prefix for simple calculations
1655 # TODO: This can maybe integrated into skel.fromClient() later...
1656 for name, value in values.items():
1657 match name[0]:
1658 case "+": # Increment by value?
1659 skel[name[1:]] += value
1660 case "-": # Decrement by value?
1661 skel[name[1:]] -= value
1663 elif callable(values):
1664 values(skel)
1666 else:
1667 raise ValueError("'values' must either be dict or a callable.")
1669 return skel.write(update_relations=update_relations)
1671 if not db.IsInTransaction:
1672 # Retry loop
1673 while True:
1674 try:
1675 return db.RunInTransaction(__update_txn)
1677 except db.ViurDatastoreError as e:
1678 retry -= 1
1679 if retry < 0:
1680 raise
1682 logging.debug(f"{e}, retrying {retry} more times")
1684 time.sleep(1)
1685 else:
1686 return __update_txn()
1688 @classmethod
1689 def preProcessBlobLocks(cls, skel: SkeletonInstance, locks):
1690 """
1691 Can be overridden to modify the list of blobs referenced by this skeleton
1692 """
1693 return locks
1695 @classmethod
1696 def preProcessSerializedData(cls, skel: SkeletonInstance, entity):
1697 """
1698 Can be overridden to modify the :class:`viur.core.db.Entity` before its actually
1699 written to the data store.
1700 """
1701 return entity
1703 @classmethod
1704 def postSavedHandler(cls, skel: SkeletonInstance, key, dbObj):
1705 """
1706 Can be overridden to perform further actions after the entity has been written
1707 to the data store.
1708 """
1709 pass
1711 @classmethod
1712 def postDeletedHandler(cls, skel: SkeletonInstance, key):
1713 """
1714 Can be overridden to perform further actions after the entity has been deleted
1715 from the data store.
1716 """
1717 pass
1719 @classmethod
1720 def getCurrentSEOKeys(cls, skel: SkeletonInstance) -> None | dict[str, str]:
1721 """
1722 Should be overridden to return a dictionary of language -> SEO-Friendly key
1723 this entry should be reachable under. How theses names are derived are entirely up to the application.
1724 If the name is already in use for this module, the server will automatically append some random string
1725 to make it unique.
1726 :return:
1727 """
1728 return
1731class RelSkel(BaseSkeleton):
1732 """
1733 This is a Skeleton-like class that acts as a container for Skeletons used as a
1734 additional information data skeleton for
1735 :class:`~viur.core.bones.extendedRelationalBone.extendedRelationalBone`.
1737 It needs to be sub-classed where information about the kindName and its attributes
1738 (bones) are specified.
1740 The Skeleton stores its bones in an :class:`OrderedDict`-Instance, so the definition order of the
1741 contained bones remains constant.
1742 """
1744 def serialize(self, parentIndexed):
1745 if self.dbEntity is None:
1746 self.dbEntity = db.Entity()
1747 for key, _bone in self.items():
1748 # if key in self.accessedValues:
1749 _bone.serialize(self, key, parentIndexed)
1750 # if "key" in self: # Write the key seperatly, as the base-bone doesn't store it
1751 # dbObj["key"] = self["key"]
1752 # FIXME: is this a good idea? Any other way to ensure only bones present in refKeys are serialized?
1753 return self.dbEntity
1755 def unserialize(self, values: db.Entity | dict):
1756 """
1757 Loads 'values' into this skeleton.
1759 :param values: dict with values we'll assign to our bones
1760 """
1761 if not isinstance(values, db.Entity):
1762 self.dbEntity = db.Entity()
1764 if values:
1765 self.dbEntity.update(values)
1766 else:
1767 self.dbEntity = values
1769 self.accessedValues = {}
1770 self.renderAccessedValues = {}
1773class RefSkel(RelSkel):
1774 @classmethod
1775 def fromSkel(cls, kindName: str, *args: list[str]) -> t.Type[RefSkel]:
1776 """
1777 Creates a ``RefSkel`` from a skeleton-class using only the bones explicitly named in ``args``.
1779 :param args: List of bone names we'll adapt
1780 :return: A new instance of RefSkel
1781 """
1782 newClass = type("RefSkelFor" + kindName, (RefSkel,), {})
1783 fromSkel = skeletonByKind(kindName)
1784 newClass.kindName = kindName
1785 bone_map = {}
1786 for arg in args:
1787 bone_map |= {k: fromSkel.__boneMap__[k] for k in fnmatch.filter(fromSkel.__boneMap__.keys(), arg)}
1788 newClass.__boneMap__ = bone_map
1789 return newClass
1791 def read(self, key: t.Optional[db.Key | str | int] = None) -> SkeletonInstance:
1792 """
1793 Read full skeleton instance referenced by the RefSkel from the database.
1795 Can be used for reading the full Skeleton from a RefSkel.
1796 The `key` parameter also allows to read another, given key from the related kind.
1798 :raise ValueError: If the entry is no longer in the database.
1799 """
1800 skel = skeletonByKind(self.kindName)()
1802 if not skel.read(key or self["key"]):
1803 raise ValueError(f"""The key {key or self["key"]!r} seems to be gone""")
1805 return skel
1808class SkelList(list):
1809 """
1810 This class is used to hold multiple skeletons together with other, commonly used information.
1812 SkelLists are returned by Skel().all()...fetch()-constructs and provide additional information
1813 about the data base query, for fetching additional entries.
1815 :ivar cursor: Holds the cursor within a query.
1816 :vartype cursor: str
1817 """
1819 __slots__ = (
1820 "baseSkel",
1821 "customQueryInfo",
1822 "getCursor",
1823 "get_orders",
1824 "renderPreparation",
1825 )
1827 def __init__(self, baseSkel=None):
1828 """
1829 :param baseSkel: The baseclass for all entries in this list
1830 """
1831 super(SkelList, self).__init__()
1832 self.baseSkel = baseSkel or {}
1833 self.getCursor = lambda: None
1834 self.get_orders = lambda: None
1835 self.renderPreparation = None
1836 self.customQueryInfo = {}
1839# Module functions
1842def skeletonByKind(kindName: str) -> t.Type[Skeleton]:
1843 """
1844 Returns the Skeleton-Class for the given kindName. That skeleton must exist, otherwise an exception is raised.
1845 :param kindName: The kindname to retreive the skeleton for
1846 :return: The skeleton-class for that kind
1847 """
1848 assert kindName in MetaBaseSkel._skelCache, f"Unknown skeleton {kindName=}"
1849 return MetaBaseSkel._skelCache[kindName]
1852def listKnownSkeletons() -> list[str]:
1853 """
1854 :return: A list of all known kindnames (all kindnames for which a skeleton is defined)
1855 """
1856 return sorted(MetaBaseSkel._skelCache.keys())
1859def iterAllSkelClasses() -> t.Iterable[Skeleton]:
1860 """
1861 :return: An iterator that yields each Skeleton-Class once. (Only top-level skeletons are returned, so no
1862 RefSkel classes will be included)
1863 """
1864 for cls in list(MetaBaseSkel._allSkelClasses): # We'll add new classes here during setSystemInitialized()
1865 yield cls
1868### Tasks ###
1870@CallDeferred
1871def processRemovedRelations(removedKey, cursor=None):
1872 updateListQuery = (
1873 db.Query("viur-relations")
1874 .filter("dest.__key__ =", removedKey)
1875 .filter("viur_relational_consistency >", RelationalConsistency.PreventDeletion.value)
1876 )
1877 updateListQuery = updateListQuery.setCursor(cursor)
1878 updateList = updateListQuery.run(limit=5)
1880 for entry in updateList:
1881 skel = skeletonByKind(entry["viur_src_kind"])()
1883 if not skel.read(entry["src"].key):
1884 raise ValueError(f"processRemovedRelations detects inconsistency on src={entry['src'].key!r}")
1886 if entry["viur_relational_consistency"] == RelationalConsistency.SetNull.value:
1887 found = False
1889 for key, bone in skel.items():
1890 if isinstance(bone, RelationalBone):
1891 if relational_value := skel[key]:
1892 if isinstance(relational_value, dict) and relational_value["dest"]["key"] == removedKey:
1893 skel[key] = None
1894 found = True
1896 elif isinstance(relational_value, list):
1897 skel[key] = [entry for entry in relational_value if entry["dest"]["key"] != removedKey]
1898 found = True
1900 else:
1901 raise NotImplementedError(f"In {entry['src'].key!r}, no handling for {relational_value=}")
1903 if found:
1904 skel.write(update_relations=False)
1906 else:
1907 logging.critical(f"""Cascade deletion of {skel["key"]!r}""")
1908 skel.delete()
1910 if len(updateList) == 5:
1911 processRemovedRelations(removedKey, updateListQuery.getCursor())
1914@CallDeferred
1915def updateRelations(destKey: db.Key, minChangeTime: int, changedBone: t.Optional[str], cursor: t.Optional[str] = None):
1916 """
1917 This function updates Entities, which may have a copy of values from another entity which has been recently
1918 edited (updated). In ViUR, relations are implemented by copying the values from the referenced entity into the
1919 entity that's referencing them. This allows ViUR to run queries over properties of referenced entities and
1920 prevents additional db.Get's to these referenced entities if the main entity is read. However, this forces
1921 us to track changes made to entities as we might have to update these mirrored values. This is the deferred
1922 call from meth:`viur.core.skeleton.Skeleton.write()` after an update (edit) on one Entity to do exactly that.
1924 :param destKey: The database-key of the entity that has been edited
1925 :param minChangeTime: The timestamp on which the edit occurred. As we run deferred, and the entity might have
1926 been edited multiple times before we get acutally called, we can ignore entities that have been updated
1927 in the meantime as they're already up2date
1928 :param changedBone: If set, we'll update only entites that have a copy of that bone. Relations mirror only
1929 key and name by default, so we don't have to update these if only another bone has been changed.
1930 :param cursor: The database cursor for the current request as we only process five entities at once and then
1931 defer again.
1932 """
1933 logging.debug(f"Starting updateRelations for {destKey=}; {minChangeTime=}, {changedBone=}, {cursor=}")
1934 current.request_data.get()["__update_relations_bone"] = changedBone
1935 updateListQuery = (
1936 db.Query("viur-relations")
1937 .filter("dest.__key__ =", destKey)
1938 .filter("viur_delayed_update_tag <", minChangeTime)
1939 .filter("viur_relational_updateLevel =", RelationalUpdateLevel.Always.value)
1940 )
1941 if changedBone:
1942 updateListQuery.filter("viur_foreign_keys =", changedBone)
1943 if cursor:
1944 updateListQuery.setCursor(cursor)
1945 updateList = updateListQuery.run(limit=5)
1947 def updateTxn(skel, key, srcRelKey):
1948 if not skel.read(key):
1949 logging.warning(f"Cannot update stale reference to {key=} (referenced from {srcRelKey=})")
1950 return
1952 skel.refresh()
1953 skel.write(update_relations=False)
1955 for srcRel in updateList:
1956 try:
1957 skel = skeletonByKind(srcRel["viur_src_kind"])()
1958 except AssertionError:
1959 logging.info(f"""Ignoring {srcRel.key!r} which refers to unknown kind {srcRel["viur_src_kind"]!r}""")
1960 continue
1961 if db.IsInTransaction():
1962 updateTxn(skel, srcRel["src"].key, srcRel.key)
1963 else:
1964 db.RunInTransaction(updateTxn, skel, srcRel["src"].key, srcRel.key)
1965 nextCursor = updateListQuery.getCursor()
1966 if len(updateList) == 5 and nextCursor:
1967 updateRelations(destKey, minChangeTime, changedBone, nextCursor)
1970@CallableTask
1971class TaskUpdateSearchIndex(CallableTaskBase):
1972 """
1973 This tasks loads and saves *every* entity of the given module.
1974 This ensures an updated searchIndex and verifies consistency of this data.
1975 """
1976 key = "rebuildSearchIndex"
1977 name = "Rebuild search index"
1978 descr = "This task can be called to update search indexes and relational information."
1980 def canCall(self) -> bool:
1981 """Checks wherever the current user can execute this task"""
1982 user = current.user.get()
1983 return user is not None and "root" in user["access"]
1985 def dataSkel(self):
1986 modules = ["*"] + listKnownSkeletons()
1987 modules.sort()
1988 skel = BaseSkeleton().clone()
1989 skel.module = SelectBone(descr="Module", values={x: translate(x) for x in modules}, required=True)
1990 return skel
1992 def execute(self, module, *args, **kwargs):
1993 usr = current.user.get()
1994 if not usr:
1995 logging.warning("Don't know who to inform after rebuilding finished")
1996 notify = None
1997 else:
1998 notify = usr["name"]
2000 if module == "*":
2001 for module in listKnownSkeletons():
2002 logging.info("Rebuilding search index for module %r", module)
2003 self._run(module, notify)
2004 else:
2005 self._run(module, notify)
2007 @staticmethod
2008 def _run(module: str, notify: str):
2009 Skel = skeletonByKind(module)
2010 if not Skel:
2011 logging.error("TaskUpdateSearchIndex: Invalid module")
2012 return
2013 RebuildSearchIndex.startIterOnQuery(Skel().all(), {"notify": notify, "module": module})
2016class RebuildSearchIndex(QueryIter):
2017 @classmethod
2018 def handleEntry(cls, skel: SkeletonInstance, customData: dict[str, str]):
2019 skel.refresh()
2020 skel.write(update_relations=False)
2022 @classmethod
2023 def handleFinish(cls, totalCount: int, customData: dict[str, str]):
2024 QueryIter.handleFinish(totalCount, customData)
2025 if not customData["notify"]:
2026 return
2027 txt = (
2028 f"{conf.instance.project_id}: Rebuild search index finished for {customData['module']}\n\n"
2029 f"ViUR finished to rebuild the search index for module {customData['module']}.\n"
2030 f"{totalCount} records updated in total on this kind."
2031 )
2032 try:
2033 email.send_email(dests=customData["notify"], stringTemplate=txt, skel=None)
2034 except Exception as exc: # noqa; OverQuota, whatever
2035 logging.exception(f'Failed to notify {customData["notify"]}')
2038### Vacuum Relations
2040@CallableTask
2041class TaskVacuumRelations(TaskUpdateSearchIndex):
2042 """
2043 Checks entries in viur-relations and verifies that the src-kind
2044 and it's RelationalBone still exists.
2045 """
2046 key = "vacuumRelations"
2047 name = "Vacuum viur-relations (dangerous)"
2048 descr = "Drop stale inbound relations for the given kind"
2050 def execute(self, module: str, *args, **kwargs):
2051 usr = current.user.get()
2052 if not usr:
2053 logging.warning("Don't know who to inform after rebuilding finished")
2054 notify = None
2055 else:
2056 notify = usr["name"]
2057 processVacuumRelationsChunk(module.strip(), None, notify=notify)
2060@CallDeferred
2061def processVacuumRelationsChunk(
2062 module: str, cursor, count_total: int = 0, count_removed: int = 0, notify=None
2063):
2064 """
2065 Processes 25 Entries and calls the next batch
2066 """
2067 query = db.Query("viur-relations")
2068 if module != "*":
2069 query.filter("viur_src_kind =", module)
2070 query.setCursor(cursor)
2071 for relation_object in query.run(25):
2072 count_total += 1
2073 if not (src_kind := relation_object.get("viur_src_kind")):
2074 logging.critical("We got an relation-object without a src_kind!")
2075 continue
2076 if not (src_prop := relation_object.get("viur_src_property")):
2077 logging.critical("We got an relation-object without a src_prop!")
2078 continue
2079 try:
2080 skel = skeletonByKind(src_kind)()
2081 except AssertionError:
2082 # The referenced skeleton does not exist in this data model -> drop that relation object
2083 logging.info(f"Deleting {relation_object.key} which refers to unknown kind {src_kind}")
2084 db.Delete(relation_object)
2085 count_removed += 1
2086 continue
2087 if src_prop not in skel:
2088 logging.info(f"Deleting {relation_object.key} which refers to "
2089 f"non-existing RelationalBone {src_prop} of {src_kind}")
2090 db.Delete(relation_object)
2091 count_removed += 1
2092 logging.info(f"END processVacuumRelationsChunk {module}, "
2093 f"{count_total} records processed, {count_removed} removed")
2094 if new_cursor := query.getCursor():
2095 # Start processing of the next chunk
2096 processVacuumRelationsChunk(module, new_cursor, count_total, count_removed, notify)
2097 elif notify:
2098 txt = (
2099 f"{conf.instance.project_id}: Vacuum relations finished for {module}\n\n"
2100 f"ViUR finished to vacuum viur-relations for module {module}.\n"
2101 f"{count_total} records processed, "
2102 f"{count_removed} entries removed"
2103 )
2104 try:
2105 email.send_email(dests=notify, stringTemplate=txt, skel=None)
2106 except Exception as exc: # noqa; OverQuota, whatever
2107 logging.exception(f"Failed to notify {notify}")
2110# Forward our references to SkelInstance to the database (needed for queries)
2111db.config["SkeletonInstanceRef"] = SkeletonInstance
2113# DEPRECATED ATTRIBUTES HANDLING
2115__DEPRECATED_NAMES = {
2116 # stuff prior viur-core < 3.6
2117 "seoKeyBone": ("SeoKeyBone", SeoKeyBone),
2118}
2121def __getattr__(attr: str) -> object:
2122 if entry := __DEPRECATED_NAMES.get(attr):
2123 func = entry[1]
2124 msg = f"{attr} was replaced by {entry[0]}"
2125 warnings.warn(msg, DeprecationWarning, stacklevel=2)
2126 logging.warning(msg, stacklevel=2)
2127 return func
2129 return super(__import__(__name__).__class__).__getattribute__(attr)