Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/skeleton.py: 0%
975 statements
« prev ^ index » next coverage.py v7.9.1, created at 2025-06-26 11:31 +0000
1from __future__ import annotations # noqa: required for pre-defined annotations
3import copy
4import fnmatch
5import inspect
6import logging
7import os
8import string
9import sys
10import time
11import typing as t
12import warnings
13from functools import partial
14from itertools import chain
16from deprecated.sphinx import deprecated
18from viur.core import conf, current, db, email, errors, translate, utils
19from viur.core.bones import (
20 BaseBone,
21 DateBone,
22 KeyBone,
23 ReadFromClientException,
24 RelationalBone,
25 RelationalConsistency,
26 RelationalUpdateLevel,
27 SelectBone,
28 StringBone,
29)
30from viur.core.bones.base import (
31 Compute,
32 ComputeInterval,
33 ComputeMethod,
34 ReadFromClientError,
35 ReadFromClientErrorSeverity,
36 getSystemInitialized,
37)
38from viur.core.tasks import CallDeferred, CallableTask, CallableTaskBase, QueryIter
40_UNDEFINED = object()
41ABSTRACT_SKEL_CLS_SUFFIX = "AbstractSkel"
42KeyType: t.TypeAlias = db.Key | str | int
45class MetaBaseSkel(type):
46 """
47 This is the metaclass for Skeletons.
48 It is used to enforce several restrictions on bone names, etc.
49 """
50 _skelCache = {} # Mapping kindName -> SkelCls
51 _allSkelClasses = set() # list of all known skeleton classes (including Ref and Mail-Skels)
53 # List of reserved keywords and function names
54 __reserved_keywords = {
55 "all",
56 "bounce",
57 "clone",
58 "cursor",
59 "delete",
60 "errors",
61 "fromClient",
62 "fromDB",
63 "get",
64 "getCurrentSEOKeys",
65 "items",
66 "keys",
67 "limit",
68 "orderby",
69 "orderdir",
70 "patch",
71 "postDeletedHandler",
72 "postSavedHandler",
73 "preProcessBlobLocks",
74 "preProcessSerializedData",
75 "read",
76 "refresh",
77 "self",
78 "serialize",
79 "setBoneValue",
80 "structure",
81 "style",
82 "toDB",
83 "unserialize",
84 "values",
85 "write",
86 }
88 __allowed_chars = string.ascii_letters + string.digits + "_"
90 def __init__(cls, name, bases, dct, **kwargs):
91 cls.__boneMap__ = MetaBaseSkel.generate_bonemap(cls)
93 if not getSystemInitialized() and not cls.__name__.endswith(ABSTRACT_SKEL_CLS_SUFFIX):
94 MetaBaseSkel._allSkelClasses.add(cls)
96 super().__init__(name, bases, dct)
98 @staticmethod
99 def generate_bonemap(cls):
100 """
101 Recursively constructs a dict of bones from
102 """
103 map = {}
105 for base in cls.__bases__:
106 if "__viurBaseSkeletonMarker__" in dir(base):
107 map |= MetaBaseSkel.generate_bonemap(base)
109 for key in cls.__dict__:
110 prop = getattr(cls, key)
112 if isinstance(prop, BaseBone):
113 if not all([c in MetaBaseSkel.__allowed_chars for c in key]):
114 raise AttributeError(f"Invalid bone name: {key!r} contains invalid characters")
115 elif key in MetaBaseSkel.__reserved_keywords:
116 raise AttributeError(f"Invalid bone name: {key!r} is reserved and cannot be used")
118 map[key] = prop
120 elif prop is None and key in map: # Allow removing a bone in a subclass by setting it to None
121 del map[key]
123 return map
125 def __setattr__(self, key, value):
126 super().__setattr__(key, value)
127 if isinstance(value, BaseBone):
128 # Call BaseBone.__set_name__ manually for bones that are assigned at runtime
129 value.__set_name__(self, key)
132class SkeletonInstance:
133 """
134 The actual wrapper around a Skeleton-Class. An object of this class is what's actually returned when you
135 call a Skeleton-Class. With ViUR3, you don't get an instance of a Skeleton-Class any more - it's always this
136 class. This is much faster as this is a small class.
137 """
138 __slots__ = {
139 "accessedValues",
140 "boneMap",
141 "dbEntity",
142 "errors",
143 "is_cloned",
144 "renderAccessedValues",
145 "renderPreparation",
146 "skeletonCls",
147 }
149 def __init__(
150 self,
151 skel_cls: t.Type[Skeleton],
152 *,
153 bones: t.Iterable[str] = (),
154 bone_map: t.Optional[t.Dict[str, BaseBone]] = None,
155 clone: bool = False,
156 # FIXME: BELOW IS DEPRECATED!
157 clonedBoneMap: t.Optional[t.Dict[str, BaseBone]] = None,
158 ):
159 """
160 Creates a new SkeletonInstance based on `skel_cls`.
162 :param skel_cls: Is the base skeleton class to inherit from and reference to.
163 :param bones: If given, defines an iterable of bones that are take into the SkeletonInstance.
164 The order of the bones defines the order in the SkeletonInstance.
165 :param bone_map: A pre-defined bone map to use, or extend.
166 :param clone: If set True, performs a cloning of the used bone map, to be entirely stand-alone.
167 """
169 # TODO: Remove with ViUR-core 3.8; required by viur-datastore :'-(
170 if clonedBoneMap:
171 msg = "'clonedBoneMap' was renamed into 'bone_map'"
172 warnings.warn(msg, DeprecationWarning, stacklevel=2)
173 # logging.warning(msg, stacklevel=2)
175 if bone_map:
176 raise ValueError("Can't provide both 'bone_map' and 'clonedBoneMap'")
178 bone_map = clonedBoneMap
180 bone_map = bone_map or {}
182 if bones:
183 names = ("key", ) + tuple(bones)
185 # generate full keys sequence based on definition; keeps order of patterns!
186 keys = []
187 for name in names:
188 if name in skel_cls.__boneMap__:
189 keys.append(name)
190 else:
191 keys.extend(fnmatch.filter(skel_cls.__boneMap__.keys(), name))
193 if clone:
194 bone_map |= {k: copy.deepcopy(skel_cls.__boneMap__[k]) for k in keys if skel_cls.__boneMap__[k]}
195 else:
196 bone_map |= {k: skel_cls.__boneMap__[k] for k in keys if skel_cls.__boneMap__[k]}
198 elif clone:
199 if bone_map:
200 bone_map = copy.deepcopy(bone_map)
201 else:
202 bone_map = copy.deepcopy(skel_cls.__boneMap__)
204 # generated or use provided bone_map
205 if bone_map:
206 self.boneMap = bone_map
208 else: # No Subskel, no Clone
209 self.boneMap = skel_cls.__boneMap__.copy()
211 if clone:
212 for v in self.boneMap.values():
213 v.isClonedInstance = True
215 self.accessedValues = {}
216 self.dbEntity = None
217 self.errors = []
218 self.is_cloned = clone
219 self.renderAccessedValues = {}
220 self.renderPreparation = None
221 self.skeletonCls = skel_cls
223 def items(self, yieldBoneValues: bool = False) -> t.Iterable[tuple[str, BaseBone]]:
224 if yieldBoneValues:
225 for key in self.boneMap.keys():
226 yield key, self[key]
227 else:
228 yield from self.boneMap.items()
230 def keys(self) -> t.Iterable[str]:
231 yield from self.boneMap.keys()
233 def values(self) -> t.Iterable[t.Any]:
234 yield from self.boneMap.values()
236 def __iter__(self) -> t.Iterable[str]:
237 yield from self.keys()
239 def __contains__(self, item):
240 return item in self.boneMap
242 def get(self, item, default=None):
243 if item not in self:
244 return default
246 return self[item]
248 def update(self, *args, **kwargs) -> None:
249 self.__ior__(dict(*args, **kwargs))
251 def __setitem__(self, key, value):
252 assert self.renderPreparation is None, "Cannot modify values while rendering"
253 if isinstance(value, BaseBone):
254 raise AttributeError(f"Don't assign this bone object as skel[\"{key}\"] = ... anymore to the skeleton. "
255 f"Use skel.{key} = ... for bone to skeleton assignment!")
256 self.accessedValues[key] = value
258 def __getitem__(self, key):
259 if self.renderPreparation:
260 if key in self.renderAccessedValues:
261 return self.renderAccessedValues[key]
262 if key not in self.accessedValues:
263 boneInstance = self.boneMap.get(key, None)
264 if boneInstance:
265 if self.dbEntity is not None:
266 boneInstance.unserialize(self, key)
267 else:
268 self.accessedValues[key] = boneInstance.getDefaultValue(self)
269 if not self.renderPreparation:
270 return self.accessedValues.get(key)
271 value = self.renderPreparation(getattr(self, key), self, key, self.accessedValues.get(key))
272 self.renderAccessedValues[key] = value
273 return value
275 def __getattr__(self, item: str):
276 """
277 Get a special attribute from the SkeletonInstance
279 __getattr__ is called when an attribute access fails with an
280 AttributeError. So we know that this is not a real attribute of
281 the SkeletonInstance. But there are still a few special cases in which
282 attributes are loaded from the skeleton class.
283 """
284 if item == "boneMap":
285 return {} # There are __setAttr__ calls before __init__ has run
287 # Load attribute value from the Skeleton class
288 elif item in {
289 "database_adapters",
290 "interBoneValidations",
291 "kindName",
292 }:
293 return getattr(self.skeletonCls, item)
295 # FIXME: viur-datastore backward compatiblity REMOVE WITH VIUR4
296 elif item == "customDatabaseAdapter":
297 if prop := getattr(self.skeletonCls, "database_adapters"):
298 return prop[0] # viur-datastore assumes there is only ONE!
300 return None
302 # Load a @classmethod from the Skeleton class and bound this SkeletonInstance
303 elif item in {
304 "all",
305 "delete",
306 "patch",
307 "fromClient",
308 "fromDB",
309 "getCurrentSEOKeys",
310 "postDeletedHandler",
311 "postSavedHandler",
312 "preProcessBlobLocks",
313 "preProcessSerializedData",
314 "read",
315 "refresh",
316 "serialize",
317 "setBoneValue",
318 "toDB",
319 "unserialize",
320 "write",
321 }:
322 return partial(getattr(self.skeletonCls, item), self)
324 # Load a @property from the Skeleton class
325 try:
326 # Use try/except to save an if check
327 class_value = getattr(self.skeletonCls, item)
329 except AttributeError:
330 # Not inside the Skeleton class, okay at this point.
331 pass
333 else:
334 if isinstance(class_value, property):
335 # The attribute is a @property and can be called
336 # Note: `self` is this SkeletonInstance, not the Skeleton class.
337 # Therefore, you can access values inside the property method
338 # with item-access like `self["key"]`.
339 try:
340 return class_value.fget(self)
341 except AttributeError as exc:
342 # The AttributeError cannot be re-raised any further at this point.
343 # Since this would then be evaluated as an access error
344 # to the property attribute.
345 # Otherwise, it would be lost that it is an incorrect attribute access
346 # within this property (during the method call).
347 msg, *args = exc.args
348 msg = f"AttributeError: {msg}"
349 raise ValueError(msg, *args) from exc
350 # Load the bone instance from the bone map of this SkeletonInstance
351 try:
352 return self.boneMap[item]
353 except KeyError as exc:
354 raise AttributeError(f"{self.__class__.__name__!r} object has no attribute '{item}'") from exc
356 def __delattr__(self, item):
357 del self.boneMap[item]
358 if item in self.accessedValues:
359 del self.accessedValues[item]
360 if item in self.renderAccessedValues:
361 del self.renderAccessedValues[item]
363 def __setattr__(self, key, value):
364 if key in self.boneMap or isinstance(value, BaseBone):
365 if value is None:
366 del self.boneMap[key]
367 else:
368 value.__set_name__(self.skeletonCls, key)
369 self.boneMap[key] = value
370 elif key == "renderPreparation":
371 super().__setattr__(key, value)
372 self.renderAccessedValues.clear()
373 else:
374 super().__setattr__(key, value)
376 def __repr__(self) -> str:
377 return f"<SkeletonInstance of {self.skeletonCls.__name__} with {dict(self)}>"
379 def __str__(self) -> str:
380 return str(dict(self))
382 def __len__(self) -> int:
383 return len(self.boneMap)
385 def __ior__(self, other: dict | SkeletonInstance | db.Entity) -> SkeletonInstance:
386 if isinstance(other, dict):
387 for key, value in other.items():
388 self.setBoneValue(key, value)
389 elif isinstance(other, db.Entity):
390 new_entity = self.dbEntity or db.Entity()
391 # We're not overriding the key
392 for key, value in other.items():
393 new_entity[key] = value
394 self.setEntity(new_entity)
395 elif isinstance(other, SkeletonInstance):
396 for key, value in other.accessedValues.items():
397 self.accessedValues[key] = value
398 for key, value in other.dbEntity.items():
399 self.dbEntity[key] = value
400 else:
401 raise ValueError("Unsupported Type")
402 return self
404 def clone(self, *, apply_clone_strategy: bool = False) -> t.Self:
405 """
406 Clones a SkeletonInstance into a modificable, stand-alone instance.
407 This will also allow to modify the underlying data model.
408 """
409 res = SkeletonInstance(self.skeletonCls, bone_map=self.boneMap, clone=True)
410 if apply_clone_strategy:
411 for bone_name, bone_instance in self.items():
412 bone_instance.clone_value(res, self, bone_name)
413 else:
414 res.accessedValues = copy.deepcopy(self.accessedValues)
415 res.dbEntity = copy.deepcopy(self.dbEntity)
416 res.is_cloned = True
417 if not apply_clone_strategy:
418 res.renderAccessedValues = copy.deepcopy(self.renderAccessedValues)
419 # else: Depending on the strategy the values are cloned in bone_instance.clone_value too
421 return res
423 def ensure_is_cloned(self):
424 """
425 Ensured this SkeletonInstance is a stand-alone clone, which can be modified.
426 Does nothing in case it was already cloned before.
427 """
428 if not self.is_cloned:
429 return self.clone()
431 return self
433 def setEntity(self, entity: db.Entity):
434 self.dbEntity = entity
435 self.accessedValues = {}
436 self.renderAccessedValues = {}
438 def structure(self) -> dict:
439 return {
440 key: bone.structure() | {"sortindex": i}
441 for i, (key, bone) in enumerate(self.items())
442 }
444 def __deepcopy__(self, memodict):
445 res = self.clone()
446 memodict[id(self)] = res
447 return res
450class BaseSkeleton(object, metaclass=MetaBaseSkel):
451 """
452 This is a container-object holding information about one database entity.
454 It has to be sub-classed with individual information about the kindName of the entities
455 and its specific data attributes, the so called bones.
456 The Skeleton stores its bones in an :class:`OrderedDict`-Instance, so the definition order of the
457 contained bones remains constant.
459 :ivar key: This bone stores the current database key of this entity. \
460 Assigning to this bones value is dangerous and does *not* affect the actual key its stored in.
462 :vartype key: server.bones.BaseBone
464 :ivar creationdate: The date and time where this entity has been created.
465 :vartype creationdate: server.bones.DateBone
467 :ivar changedate: The date and time of the last change to this entity.
468 :vartype changedate: server.bones.DateBone
469 """
470 __viurBaseSkeletonMarker__ = True
471 boneMap = None
473 @classmethod
474 @deprecated(
475 version="3.7.0",
476 reason="Function renamed. Use subskel function as alternative implementation.",
477 )
478 def subSkel(cls, *subskel_names, fullClone: bool = False, **kwargs) -> SkeletonInstance:
479 return cls.subskel(*subskel_names, clone=fullClone) # FIXME: REMOVE WITH VIUR4
481 @classmethod
482 def subskel(
483 cls,
484 *names: str,
485 bones: t.Iterable[str] = (),
486 clone: bool = False,
487 ) -> SkeletonInstance:
488 """
489 Creates a new sub-skeleton from the current skeleton.
491 A sub-skeleton is a copy of the original skeleton, containing only a subset of its bones.
493 Sub-skeletons can either be defined using the the subSkels property of the Skeleton object,
494 or freely by giving patterns for bone names which shall be part of the sub-skeleton.
496 1. Giving names as parameter merges the bones of all Skeleton.subSkels-configurations together.
497 This is the usual behavior. By passing multiple sub-skeleton names to this function, a sub-skeleton
498 with the union of all bones of the specified sub-skeletons is returned. If an entry called "*"
499 exists in the subSkels-dictionary, the bones listed in this entry will always be part of the
500 generated sub-skeleton.
501 2. Given the *bones* parameter allows to freely specify a sub-skeleton; One specialty here is,
502 that the order of the bones can also be changed in this mode. This mode is the new way of defining
503 sub-skeletons, and might become the primary way to define sub-skeletons in future.
504 3. Both modes (1 + 2) can be combined, but then the original order of the bones is kept.
505 4. The "key" bone is automatically available in each sub-skeleton.
506 5. An fnmatch-compatible wildcard pattern is allowed both in the subSkels-bone-list and the
507 free bone list.
509 Example (TodoSkel is the example skeleton from viur-base):
510 ```py
511 # legacy mode (see 1)
512 subskel = TodoSkel.subskel("add")
513 # creates subskel: key, firstname, lastname, subject
515 # free mode (see 2) allows to specify a different order!
516 subskel = TodoSkel.subskel(bones=("subject", "message", "*stname"))
517 # creates subskel: key, subject, message, firstname, lastname
519 # mixed mode (see 3)
520 subskel = TodoSkel.subskel("add", bones=("message", ))
521 # creates subskel: key, firstname, lastname, subject, message
522 ```
524 :param bones: Allows to specify an iterator of bone names (more precisely, fnmatch-wildards) which allow
525 to freely define a subskel. If *only* this parameter is given, the order of the specification also
526 defines, the order of the list. Otherwise, the original order as defined in the skeleton is kept.
527 :param clone: If set True, performs a cloning of the used bone map, to be entirely stand-alone.
529 :return: The sub-skeleton of the specified type.
530 """
531 from_subskel = False
532 bones = list(bones)
534 for name in names:
535 # a str refers to a subskel name from the cls.subSkel dict
536 if isinstance(name, str):
537 # add bones from "*" subskel once
538 if not from_subskel:
539 bones.extend(cls.subSkels.get("*") or ())
540 from_subskel = True
542 bones.extend(cls.subSkels.get(name) or ())
544 else:
545 raise ValueError(f"Invalid subskel definition: {name!r}")
547 if from_subskel:
548 # when from_subskel is True, create bone names based on the order of the bones in the original skeleton
549 bones = tuple(k for k in cls.__boneMap__.keys() if any(fnmatch.fnmatch(k, n) for n in bones))
551 if not bones:
552 raise ValueError("The given subskel definition doesn't contain any bones!")
554 return cls(bones=bones, clone=clone)
556 @classmethod
557 def setSystemInitialized(cls):
558 for attrName in dir(cls):
559 bone = getattr(cls, attrName)
560 if isinstance(bone, BaseBone):
561 bone.setSystemInitialized()
563 @classmethod
564 def setBoneValue(
565 cls,
566 skel: SkeletonInstance,
567 boneName: str,
568 value: t.Any,
569 append: bool = False,
570 language: t.Optional[str] = None
571 ) -> bool:
572 """
573 Allows for setting a bones value without calling fromClient or assigning a value directly.
574 Sanity-Checks are performed; if the value is invalid, that bone flips back to its original
575 (default) value and false is returned.
577 :param boneName: The name of the bone to be modified
578 :param value: The value that should be assigned. It's type depends on the type of that bone
579 :param append: If True, the given value is appended to the values of that bone instead of
580 replacing it. Only supported on bones with multiple=True
581 :param language: Language to set
583 :return: Wherever that operation succeeded or not.
584 """
585 bone = getattr(skel, boneName, None)
587 if not isinstance(bone, BaseBone):
588 raise ValueError(f"{boneName!r} is no valid bone on this skeleton ({skel!r})")
590 if language:
591 if not bone.languages:
592 raise ValueError("The bone {boneName!r} has no language setting")
593 elif language not in bone.languages:
594 raise ValueError("The language {language!r} is not available for bone {boneName!r}")
596 if value is None:
597 if append:
598 raise ValueError("Cannot append None-value to bone {boneName!r}")
600 if language:
601 skel[boneName][language] = [] if bone.multiple else None
602 else:
603 skel[boneName] = [] if bone.multiple else None
605 return True
607 _ = skel[boneName] # ensure the bone is being unserialized first
608 return bone.setBoneValue(skel, boneName, value, append, language)
610 @classmethod
611 def fromClient(
612 cls,
613 skel: SkeletonInstance,
614 data: dict[str, list[str] | str],
615 *,
616 amend: bool = False,
617 ignore: t.Optional[t.Iterable[str]] = None,
618 ) -> bool:
619 """
620 Load supplied *data* into Skeleton.
622 This function works similar to :func:`~viur.core.skeleton.Skeleton.setValues`, except that
623 the values retrieved from *data* are checked against the bones and their validity checks.
625 Even if this function returns False, all bones are guaranteed to be in a valid state.
626 The ones which have been read correctly are set to their valid values;
627 Bones with invalid values are set back to a safe default (None in most cases).
628 So its possible to call :func:`~viur.core.skeleton.Skeleton.write` afterwards even if reading
629 data with this function failed (through this might violates the assumed consistency-model).
631 :param skel: The skeleton instance to be filled.
632 :param data: Dictionary from which the data is read.
633 :param amend: Defines whether content of data may be incomplete to amend the skel,
634 which is useful for edit-actions.
635 :param ignore: optional list of bones to be ignored; Defaults to all readonly-bones when set to None.
637 :returns: True if all data was successfully read and complete. \
638 False otherwise (e.g. some required fields where missing or where invalid).
639 """
640 complete = True
641 skel.errors = []
643 for key, bone in skel.items():
644 if (ignore is None and bone.readOnly) or key in (ignore or ()):
645 continue
647 if errors := bone.fromClient(skel, key, data):
648 for error in errors:
649 # insert current bone name into error's fieldPath
650 error.fieldPath.insert(0, str(key))
652 # logging.debug(f"BaseSkel.fromClient {key=} {error=}")
654 incomplete = (
655 # always when something is invalid
656 error.severity == ReadFromClientErrorSeverity.Invalid
657 or (
658 # only when path is top-level
659 len(error.fieldPath) == 1
660 and (
661 # bone is generally required
662 bool(bone.required)
663 and (
664 # and value is either empty
665 error.severity == ReadFromClientErrorSeverity.Empty
666 # or when not amending, not set
667 or (not amend and error.severity == ReadFromClientErrorSeverity.NotSet)
668 )
669 )
670 )
671 )
673 # in case there are language requirements, test additionally
674 if bone.languages and isinstance(bone.required, (list, tuple)):
675 incomplete &= any([key, lang] == error.fieldPath for lang in bone.required)
677 # logging.debug(f"BaseSkel.fromClient {incomplete=} {error.severity=} {bone.required=}")
679 if incomplete:
680 complete = False
682 if conf.debug.skeleton_from_client:
683 logging.error(
684 f"""{getattr(cls, "kindName", cls.__name__)}: {".".join(error.fieldPath)}: """
685 f"""({error.severity}) {error.errorMessage}"""
686 )
688 skel.errors += errors
690 return complete
692 @classmethod
693 def refresh(cls, skel: SkeletonInstance):
694 """
695 Refresh the bones current content.
697 This function causes a refresh of all relational bones and their associated
698 information.
699 """
700 logging.debug(f"""Refreshing {skel["key"]!r} ({skel.get("name")!r})""")
702 for key, bone in skel.items():
703 if not isinstance(bone, BaseBone):
704 continue
706 _ = skel[key] # Ensure value gets loaded
707 bone.refresh(skel, key)
709 def __new__(cls, *args, **kwargs) -> SkeletonInstance:
710 return SkeletonInstance(cls, *args, **kwargs)
713class MetaSkel(MetaBaseSkel):
715 def __init__(cls, name, bases, dct, **kwargs):
716 super().__init__(name, bases, dct, **kwargs)
718 relNewFileName = inspect.getfile(cls) \
719 .replace(str(conf.instance.project_base_path), "") \
720 .replace(str(conf.instance.core_base_path), "")
722 # Check if we have an abstract skeleton
723 if cls.__name__.endswith(ABSTRACT_SKEL_CLS_SUFFIX):
724 # Ensure that it doesn't have a kindName
725 assert cls.kindName is _UNDEFINED or cls.kindName is None, "Abstract Skeletons can't have a kindName"
726 # Prevent any further processing by this class; it has to be sub-classed before it can be used
727 return
729 # Automatic determination of the kindName, if the class is not part of viur.core.
730 if (cls.kindName is _UNDEFINED
731 and not relNewFileName.strip(os.path.sep).startswith("viur")
732 and not "viur_doc_build" in dir(sys)):
733 if cls.__name__.endswith("Skel"):
734 cls.kindName = cls.__name__.lower()[:-4]
735 else:
736 cls.kindName = cls.__name__.lower()
738 # Try to determine which skeleton definition takes precedence
739 if cls.kindName and cls.kindName is not _UNDEFINED and cls.kindName in MetaBaseSkel._skelCache:
740 relOldFileName = inspect.getfile(MetaBaseSkel._skelCache[cls.kindName]) \
741 .replace(str(conf.instance.project_base_path), "") \
742 .replace(str(conf.instance.core_base_path), "")
743 idxOld = min(
744 [x for (x, y) in enumerate(conf.skeleton_search_path) if relOldFileName.startswith(y)] + [999])
745 idxNew = min(
746 [x for (x, y) in enumerate(conf.skeleton_search_path) if relNewFileName.startswith(y)] + [999])
747 if idxNew == 999:
748 # We could not determine a priority for this class as its from a path not listed in the config
749 raise NotImplementedError(
750 "Skeletons must be defined in a folder listed in conf.skeleton_search_path")
751 elif idxOld < idxNew: # Lower index takes precedence
752 # The currently processed skeleton has a lower priority than the one we already saw - just ignore it
753 return
754 elif idxOld > idxNew:
755 # The currently processed skeleton has a higher priority, use that from now
756 MetaBaseSkel._skelCache[cls.kindName] = cls
757 else: # They seem to be from the same Package - raise as something is messed up
758 raise ValueError(f"Duplicate definition for {cls.kindName} in {relNewFileName} and {relOldFileName}")
760 # Ensure that all skeletons are defined in folders listed in conf.skeleton_search_path
761 if (not any([relNewFileName.startswith(x) for x in conf.skeleton_search_path])
762 and not "viur_doc_build" in dir(sys)): # Do not check while documentation build
763 raise NotImplementedError(
764 f"""{relNewFileName} must be defined in a folder listed in {conf.skeleton_search_path}""")
766 if cls.kindName and cls.kindName is not _UNDEFINED:
767 MetaBaseSkel._skelCache[cls.kindName] = cls
769 # Auto-Add ViUR Search Tags Adapter if the skeleton has no adapter attached
770 if cls.database_adapters is _UNDEFINED:
771 cls.database_adapters = ViurTagsSearchAdapter()
773 # Always ensure that skel.database_adapters is an iterable
774 cls.database_adapters = utils.ensure_iterable(cls.database_adapters)
777class DatabaseAdapter:
778 """
779 Adapter class used to bind or use other databases and hook operations when working with a Skeleton.
780 """
782 providesFulltextSearch: bool = False
783 """Set to True if we can run a fulltext search using this database."""
785 fulltextSearchGuaranteesQueryConstrains = False
786 """Are results returned by `meth:fulltextSearch` guaranteed to also match the databaseQuery"""
788 providesCustomQueries: bool = False
789 """Indicate that we can run more types of queries than originally supported by datastore"""
791 def prewrite(self, skel: SkeletonInstance, is_add: bool, change_list: t.Iterable[str] = ()):
792 """
793 Hook being called on a add, edit or delete operation before the skeleton-specific action is performed.
795 The hook can be used to modifiy the skeleton before writing.
796 The raw entity can be obainted using `skel.dbEntity`.
798 :param action: Either contains "add", "edit" or "delete", depending on the operation.
799 :param skel: is the skeleton that is being read before written.
800 :param change_list: is a list of bone names which are being changed within the write.
801 """
802 pass
804 def write(self, skel: SkeletonInstance, is_add: bool, change_list: t.Iterable[str] = ()):
805 """
806 Hook being called on a write operations after the skeleton is written.
808 The raw entity can be obainted using `skel.dbEntity`.
810 :param action: Either contains "add" or "edit", depending on the operation.
811 :param skel: is the skeleton that is being read before written.
812 :param change_list: is a list of bone names which are being changed within the write.
813 """
814 pass
816 def delete(self, skel: SkeletonInstance):
817 """
818 Hook being called on a delete operation after the skeleton is deleted.
819 """
820 pass
822 def fulltextSearch(self, queryString: str, databaseQuery: db.Query) -> list[db.Entity]:
823 """
824 If this database supports fulltext searches, this method has to implement them.
825 If it's a plain fulltext search engine, leave 'prop:fulltextSearchGuaranteesQueryConstrains' set to False,
826 then the server will post-process the list of entries returned from this function and drop any entry that
827 cannot be returned due to other constrains set in 'param:databaseQuery'. If you can obey *every* constrain
828 set in that Query, we can skip this post-processing and save some CPU-cycles.
829 :param queryString: the string as received from the user (no quotation or other safety checks applied!)
830 :param databaseQuery: The query containing any constrains that returned entries must also match
831 :return:
832 """
833 raise NotImplementedError
836class ViurTagsSearchAdapter(DatabaseAdapter):
837 """
838 This Adapter implements a simple fulltext search on top of the datastore.
840 On skel.write(), all words from String-/TextBones are collected with all *min_length* postfixes and dumped
841 into the property `viurTags`. When queried, we'll run a prefix-match against this property - thus returning
842 entities with either an exact match or a match within a word.
844 Example:
845 For the word "hello" we'll write "hello", "ello" and "llo" into viurTags.
846 When queried with "hello" we'll have an exact match.
847 When queried with "hel" we'll match the prefix for "hello"
848 When queried with "ell" we'll prefix-match "ello" - this is only enabled when substring_matching is True.
850 We'll automatically add this adapter if a skeleton has no other database adapter defined.
851 """
852 providesFulltextSearch = True
853 fulltextSearchGuaranteesQueryConstrains = True
855 def __init__(self, min_length: int = 2, max_length: int = 50, substring_matching: bool = False):
856 super().__init__()
857 self.min_length = min_length
858 self.max_length = max_length
859 self.substring_matching = substring_matching
861 def _tags_from_str(self, value: str) -> set[str]:
862 """
863 Extract all words including all min_length postfixes from given string
864 """
865 res = set()
867 for tag in value.split(" "):
868 tag = "".join([x for x in tag.lower() if x in conf.search_valid_chars])
870 if len(tag) >= self.min_length:
871 res.add(tag)
873 if self.substring_matching:
874 for i in range(1, 1 + len(tag) - self.min_length):
875 res.add(tag[i:])
877 return res
879 def prewrite(self, skel: SkeletonInstance, *args, **kwargs):
880 """
881 Collect searchTags from skeleton and build viurTags
882 """
883 tags = set()
885 for name, bone in skel.items():
886 if bone.searchable:
887 tags = tags.union(bone.getSearchTags(skel, name))
889 skel.dbEntity["viurTags"] = list(
890 chain(*[self._tags_from_str(tag) for tag in tags if len(tag) <= self.max_length])
891 )
893 def fulltextSearch(self, queryString: str, databaseQuery: db.Query) -> list[db.Entity]:
894 """
895 Run a fulltext search
896 """
897 keywords = list(self._tags_from_str(queryString))[:10]
898 resultScoreMap = {}
899 resultEntryMap = {}
901 for keyword in keywords:
902 qryBase = databaseQuery.clone()
903 for entry in qryBase.filter("viurTags >=", keyword).filter("viurTags <", keyword + "\ufffd").run():
904 if not entry.key in resultScoreMap:
905 resultScoreMap[entry.key] = 1
906 else:
907 resultScoreMap[entry.key] += 1
908 if not entry.key in resultEntryMap:
909 resultEntryMap[entry.key] = entry
911 resultList = [(k, v) for k, v in resultScoreMap.items()]
912 resultList.sort(key=lambda x: x[1], reverse=True)
914 return [resultEntryMap[x[0]] for x in resultList[:databaseQuery.queries.limit]]
917class SeoKeyBone(StringBone):
918 """
919 Special kind of StringBone saving its contents as `viurCurrentSeoKeys` into the entity's `viur` dict.
920 """
922 def unserialize(self, skel: SkeletonInstance, name: str) -> bool:
923 try:
924 skel.accessedValues[name] = skel.dbEntity["viur"]["viurCurrentSeoKeys"]
925 except KeyError:
926 skel.accessedValues[name] = self.getDefaultValue(skel)
928 def serialize(self, skel: SkeletonInstance, name: str, parentIndexed: bool) -> bool:
929 # Serialize also to skel["viur"]["viurCurrentSeoKeys"], so we can use this bone in relations
930 if name in skel.accessedValues:
931 newVal = skel.accessedValues[name]
932 if not skel.dbEntity.get("viur"):
933 skel.dbEntity["viur"] = db.Entity()
934 res = db.Entity()
935 res["_viurLanguageWrapper_"] = True
936 for language in (self.languages or []):
937 if not self.indexed:
938 res.exclude_from_indexes.add(language)
939 res[language] = None
940 if language in newVal:
941 res[language] = self.singleValueSerialize(newVal[language], skel, name, parentIndexed)
942 skel.dbEntity["viur"]["viurCurrentSeoKeys"] = res
943 return True
946class Skeleton(BaseSkeleton, metaclass=MetaSkel):
947 kindName: str = _UNDEFINED
948 """
949 Specifies the entity kind name this Skeleton is associated with.
950 Will be determined automatically when not explicitly set.
951 """
953 database_adapters: DatabaseAdapter | t.Iterable[DatabaseAdapter] | None = _UNDEFINED
954 """
955 Custom database adapters.
956 Allows to hook special functionalities that during skeleton modifications.
957 """
959 subSkels = {} # List of pre-defined sub-skeletons of this type
961 interBoneValidations: list[
962 t.Callable[[Skeleton], list[ReadFromClientError]]] = [] # List of functions checking inter-bone dependencies
964 __seo_key_trans = str.maketrans(
965 {"<": "",
966 ">": "",
967 "\"": "",
968 "'": "",
969 "\n": "",
970 "\0": "",
971 "/": "",
972 "\\": "",
973 "?": "",
974 "&": "",
975 "#": ""
976 })
978 # The "key" bone stores the current database key of this skeleton.
979 # Warning: Assigning to this bones value now *will* set the key
980 # it gets stored in. Must be kept readOnly to avoid security-issues with add/edit.
981 key = KeyBone(
982 descr="Key"
983 )
985 name = StringBone(
986 descr="Name",
987 visible=False,
988 compute=Compute(
989 fn=lambda skel: str(skel["key"]),
990 interval=ComputeInterval(ComputeMethod.OnWrite)
991 )
992 )
994 # The date (including time) when this entry has been created
995 creationdate = DateBone(
996 descr="created at",
997 readOnly=True,
998 visible=False,
999 indexed=True,
1000 compute=Compute(fn=utils.utcNow, interval=ComputeInterval(ComputeMethod.Once)),
1001 )
1003 # The last date (including time) when this entry has been updated
1005 changedate = DateBone(
1006 descr="updated at",
1007 readOnly=True,
1008 visible=False,
1009 indexed=True,
1010 compute=Compute(fn=utils.utcNow, interval=ComputeInterval(ComputeMethod.OnWrite)),
1011 )
1013 viurCurrentSeoKeys = SeoKeyBone(
1014 descr="SEO-Keys",
1015 readOnly=True,
1016 visible=False,
1017 languages=conf.i18n.available_languages
1018 )
1020 def __repr__(self):
1021 return "<skeleton %s with data=%r>" % (self.kindName, {k: self[k] for k in self.keys()})
1023 def __str__(self):
1024 return str({k: self[k] for k in self.keys()})
1026 def __init__(self, *args, **kwargs):
1027 super(Skeleton, self).__init__(*args, **kwargs)
1028 assert self.kindName and self.kindName is not _UNDEFINED, "You must set kindName on this skeleton!"
1030 @classmethod
1031 def all(cls, skel, **kwargs) -> db.Query:
1032 """
1033 Create a query with the current Skeletons kindName.
1035 :returns: A db.Query object which allows for entity filtering and sorting.
1036 """
1037 return db.Query(skel.kindName, srcSkelClass=skel, **kwargs)
1039 @classmethod
1040 def fromClient(
1041 cls,
1042 skel: SkeletonInstance,
1043 data: dict[str, list[str] | str],
1044 *,
1045 amend: bool = False,
1046 ignore: t.Optional[t.Iterable[str]] = None,
1047 ) -> bool:
1048 """
1049 This function works similar to :func:`~viur.core.skeleton.Skeleton.setValues`, except that
1050 the values retrieved from *data* are checked against the bones and their validity checks.
1052 Even if this function returns False, all bones are guaranteed to be in a valid state.
1053 The ones which have been read correctly are set to their valid values;
1054 Bones with invalid values are set back to a safe default (None in most cases).
1055 So its possible to call :func:`~viur.core.skeleton.Skeleton.write` afterwards even if reading
1056 data with this function failed (through this might violates the assumed consistency-model).
1058 :param skel: The skeleton instance to be filled.
1059 :param data: Dictionary from which the data is read.
1060 :param amend: Defines whether content of data may be incomplete to amend the skel,
1061 which is useful for edit-actions.
1062 :param ignore: optional list of bones to be ignored; Defaults to all readonly-bones when set to None.
1064 :returns: True if all data was successfully read and complete. \
1065 False otherwise (e.g. some required fields where missing or where invalid).
1066 """
1067 assert skel.renderPreparation is None, "Cannot modify values while rendering"
1069 # Load data into this skeleton
1070 complete = bool(data) and super().fromClient(skel, data, amend=amend, ignore=ignore)
1072 if (
1073 not data # in case data is empty
1074 or (len(data) == 1 and "key" in data)
1075 or (utils.parse.bool(data.get("nomissing")))
1076 ):
1077 skel.errors = []
1079 # Check if all unique values are available
1080 for boneName, boneInstance in skel.items():
1081 if boneInstance.unique:
1082 lockValues = boneInstance.getUniquePropertyIndexValues(skel, boneName)
1083 for lockValue in lockValues:
1084 dbObj = db.Get(db.Key(f"{skel.kindName}_{boneName}_uniquePropertyIndex", lockValue))
1085 if dbObj and (not skel["key"] or dbObj["references"] != skel["key"].id_or_name):
1086 # This value is taken (sadly, not by us)
1087 complete = False
1088 errorMsg = boneInstance.unique.message
1089 skel.errors.append(
1090 ReadFromClientError(ReadFromClientErrorSeverity.Invalid, errorMsg, [boneName]))
1092 # Check inter-Bone dependencies
1093 for checkFunc in skel.interBoneValidations:
1094 errors = checkFunc(skel)
1095 if errors:
1096 for error in errors:
1097 if error.severity.value > 1:
1098 complete = False
1099 if conf.debug.skeleton_from_client:
1100 logging.debug(f"{cls.kindName}: {error.fieldPath}: {error.errorMessage!r}")
1102 skel.errors.extend(errors)
1104 return complete
1106 @classmethod
1107 @deprecated(
1108 version="3.7.0",
1109 reason="Use skel.read() instead of skel.fromDB()",
1110 )
1111 def fromDB(cls, skel: SkeletonInstance, key: KeyType) -> bool:
1112 """
1113 Deprecated function, replaced by Skeleton.read().
1114 """
1115 return bool(cls.read(skel, key, _check_legacy=False))
1117 @classmethod
1118 def read(
1119 cls,
1120 skel: SkeletonInstance,
1121 key: t.Optional[KeyType] = None,
1122 *,
1123 create: bool | dict | t.Callable[[SkeletonInstance], None] = False,
1124 _check_legacy: bool = True
1125 ) -> t.Optional[SkeletonInstance]:
1126 """
1127 Read Skeleton with *key* from the datastore into the Skeleton.
1128 If not key is given, skel["key"] will be used.
1130 Reads all available data of entity kind *kindName* and the key *key*
1131 from the Datastore into the Skeleton structure's bones. Any previous
1132 data of the bones will discard.
1134 To store a Skeleton object to the Datastore, see :func:`~viur.core.skeleton.Skeleton.write`.
1136 :param key: A :class:`viur.core.db.Key`, string, or int; from which the data shall be fetched.
1137 If not provided, skel["key"] will be used.
1138 :param create: Allows to specify a dict or initial callable that is executed in case the Skeleton with the
1139 given key does not exist, it will be created.
1141 :returns: None on error, or the given SkeletonInstance on success.
1143 """
1144 # FIXME VIUR4: Stay backward compatible, call sub-classed fromDB if available first!
1145 if _check_legacy and "fromDB" in cls.__dict__:
1146 with warnings.catch_warnings():
1147 warnings.simplefilter("ignore", DeprecationWarning)
1148 return cls.fromDB(skel, key=key)
1150 assert skel.renderPreparation is None, "Cannot modify values while rendering"
1152 try:
1153 db_key = db.keyHelper(key or skel["key"], skel.kindName)
1154 except (ValueError, NotImplementedError): # This key did not parse
1155 return None
1157 if db_res := db.Get(db_key):
1158 skel.setEntity(db_res)
1159 return skel
1160 elif create in (False, None):
1161 return None
1162 elif isinstance(create, dict):
1163 if create and not skel.fromClient(create, amend=True):
1164 raise ReadFromClientException(skel.errors)
1165 elif callable(create):
1166 create(skel)
1167 elif create is not True:
1168 raise ValueError("'create' must either be dict, a callable or True.")
1170 return skel.write()
1172 @classmethod
1173 @deprecated(
1174 version="3.7.0",
1175 reason="Use skel.write() instead of skel.toDB()",
1176 )
1177 def toDB(cls, skel: SkeletonInstance, update_relations: bool = True, **kwargs) -> db.Key:
1178 """
1179 Deprecated function, replaced by Skeleton.write().
1180 """
1182 # TODO: Remove with ViUR4
1183 if "clearUpdateTag" in kwargs:
1184 msg = "clearUpdateTag was replaced by update_relations"
1185 warnings.warn(msg, DeprecationWarning, stacklevel=3)
1186 logging.warning(msg, stacklevel=3)
1187 update_relations = not kwargs["clearUpdateTag"]
1189 skel = cls.write(skel, update_relations=update_relations, _check_legacy=False)
1190 return skel["key"]
1192 @classmethod
1193 def write(
1194 cls,
1195 skel: SkeletonInstance,
1196 key: t.Optional[KeyType] = None,
1197 *,
1198 update_relations: bool = True,
1199 _check_legacy: bool = True,
1200 ) -> SkeletonInstance:
1201 """
1202 Write current Skeleton to the datastore.
1204 Stores the current data of this instance into the database.
1205 If an *key* value is set to the object, this entity will ne updated;
1206 Otherwise a new entity will be created.
1208 To read a Skeleton object from the data store, see :func:`~viur.core.skeleton.Skeleton.read`.
1210 :param key: Allows to specify a key that is set to the skeleton and used for writing.
1211 :param update_relations: If False, this entity won't be marked dirty;
1212 This avoids from being fetched by the background task updating relations.
1214 :returns: The Skeleton.
1215 """
1216 # FIXME VIUR4: Stay backward compatible, call sub-classed toDB if available first!
1217 if _check_legacy and "toDB" in cls.__dict__:
1218 with warnings.catch_warnings():
1219 warnings.simplefilter("ignore", DeprecationWarning)
1220 return cls.toDB(skel, update_relations=update_relations)
1222 assert skel.renderPreparation is None, "Cannot modify values while rendering"
1224 def __txn_write(write_skel):
1225 db_key = write_skel["key"]
1226 skel = write_skel.skeletonCls()
1228 blob_list = set()
1229 change_list = []
1230 old_copy = {}
1231 # Load the current values from Datastore or create a new, empty db.Entity
1232 if not db_key:
1233 # We'll generate the key we'll be stored under early so we can use it for locks etc
1234 db_key = db.AllocateIDs(db.Key(skel.kindName))
1235 skel.dbEntity = db.Entity(db_key)
1236 is_add = True
1237 else:
1238 db_key = db.keyHelper(db_key, skel.kindName)
1239 if db_obj := db.Get(db_key):
1240 skel.dbEntity = db_obj
1241 old_copy = {k: v for k, v in skel.dbEntity.items()}
1242 is_add = False
1243 else:
1244 skel.dbEntity = db.Entity(db_key)
1245 is_add = True
1247 skel.dbEntity.setdefault("viur", {})
1249 # Merge values and assemble unique properties
1250 # Move accessed Values from srcSkel over to skel
1251 skel.accessedValues = write_skel.accessedValues
1253 write_skel["key"] = skel["key"] = db_key # Ensure key stays set
1254 write_skel.dbEntity = skel.dbEntity # update write_skel's dbEntity
1256 for bone_name, bone in skel.items():
1257 if bone_name == "key": # Explicitly skip key on top-level - this had been set above
1258 continue
1260 # Allow bones to perform outstanding "magic" operations before saving to db
1261 bone.performMagic(skel, bone_name, isAdd=is_add) # FIXME VIUR4: ANY MAGIC IN OUR CODE IS DEPRECATED!!!
1263 if not (bone_name in skel.accessedValues or bone.compute) and bone_name not in skel.dbEntity:
1264 _ = skel[bone_name] # Ensure the datastore is filled with the default value
1266 if (
1267 bone_name in skel.accessedValues or bone.compute # We can have a computed value on store
1268 or bone_name not in skel.dbEntity # It has not been written and is not in the database
1269 ):
1270 # Serialize bone into entity
1271 try:
1272 bone.serialize(skel, bone_name, True)
1273 except Exception as e:
1274 logging.error(
1275 f"Failed to serialize {bone_name=} ({bone=}): {skel.accessedValues[bone_name]=}"
1276 )
1277 raise e
1279 # Obtain referenced blobs
1280 blob_list.update(bone.getReferencedBlobs(skel, bone_name))
1282 # Check if the value has actually changed
1283 # Ensure that only bones within the current subskel are processed.
1284 if bone_name in write_skel and skel.dbEntity.get(bone_name) != old_copy.get(bone_name):
1285 change_list.append(bone_name)
1287 # Lock hashes from bones that must have unique values
1288 if bone.unique:
1289 # Remember old hashes for bones that must have an unique value
1290 old_unique_values = []
1292 if f"{bone_name}_uniqueIndexValue" in skel.dbEntity["viur"]:
1293 old_unique_values = skel.dbEntity["viur"][f"{bone_name}_uniqueIndexValue"]
1294 # Check if the property is unique
1295 new_unique_values = bone.getUniquePropertyIndexValues(skel, bone_name)
1296 new_lock_kind = f"{skel.kindName}_{bone_name}_uniquePropertyIndex"
1297 for new_lock_value in new_unique_values:
1298 new_lock_key = db.Key(new_lock_kind, new_lock_value)
1299 if lock_db_obj := db.Get(new_lock_key):
1301 # There's already a lock for that value, check if we hold it
1302 if lock_db_obj["references"] != skel.dbEntity.key.id_or_name:
1303 # This value has already been claimed, and not by us
1304 # TODO: Use a custom exception class which is catchable with an try/except
1305 raise ValueError(
1306 f"The unique value {skel[bone_name]!r} of bone {bone_name!r} "
1307 f"has been recently claimed (by {new_lock_key=}).")
1308 else:
1309 # This value is locked for the first time, create a new lock-object
1310 lock_obj = db.Entity(new_lock_key)
1311 lock_obj["references"] = skel.dbEntity.key.id_or_name
1312 db.Put(lock_obj)
1313 if new_lock_value in old_unique_values:
1314 old_unique_values.remove(new_lock_value)
1315 skel.dbEntity["viur"][f"{bone_name}_uniqueIndexValue"] = new_unique_values
1317 # Remove any lock-object we're holding for values that we don't have anymore
1318 for old_unique_value in old_unique_values:
1319 # Try to delete the old lock
1321 old_lock_key = db.Key(f"{skel.kindName}_{bone_name}_uniquePropertyIndex", old_unique_value)
1322 if old_lock_obj := db.Get(old_lock_key):
1323 if old_lock_obj["references"] != skel.dbEntity.key.id_or_name:
1325 # We've been supposed to have that lock - but we don't.
1326 # Don't remove that lock as it now belongs to a different entry
1327 logging.critical("Detected Database corruption! A Value-Lock had been reassigned!")
1328 else:
1329 # It's our lock which we don't need anymore
1330 db.Delete(old_lock_key)
1331 else:
1332 logging.critical("Detected Database corruption! Could not delete stale lock-object!")
1334 # Delete legacy property (PR #1244) #TODO: Remove in ViUR4
1335 skel.dbEntity.pop("viur_incomming_relational_locks", None)
1337 # Ensure the SEO-Keys are up-to-date
1338 last_requested_seo_keys = skel.dbEntity["viur"].get("viurLastRequestedSeoKeys") or {}
1339 last_set_seo_keys = skel.dbEntity["viur"].get("viurCurrentSeoKeys") or {}
1340 # Filter garbage serialized into this field by the SeoKeyBone
1341 last_set_seo_keys = {k: v for k, v in last_set_seo_keys.items() if not k.startswith("_") and v}
1343 if not isinstance(skel.dbEntity["viur"].get("viurCurrentSeoKeys"), dict):
1344 skel.dbEntity["viur"]["viurCurrentSeoKeys"] = {}
1346 if current_seo_keys := skel.getCurrentSEOKeys():
1347 # Convert to lower-case and remove certain characters
1348 for lang, value in current_seo_keys.items():
1349 current_seo_keys[lang] = value.lower().translate(Skeleton.__seo_key_trans).strip()
1351 for language in (conf.i18n.available_languages or [conf.i18n.default_language]):
1352 if current_seo_keys and language in current_seo_keys:
1353 current_seo_key = current_seo_keys[language]
1355 if current_seo_key != last_requested_seo_keys.get(language): # This one is new or has changed
1356 new_seo_key = current_seo_keys[language]
1358 for _ in range(0, 3):
1359 entry_using_key = db.Query(skel.kindName).filter(
1360 "viur.viurActiveSeoKeys =", new_seo_key).getEntry()
1362 if entry_using_key and entry_using_key.key != skel.dbEntity.key:
1363 # It's not unique; append a random string and try again
1364 new_seo_key = f"{current_seo_keys[language]}-{utils.string.random(5).lower()}"
1366 else:
1367 # We found a new SeoKey
1368 break
1369 else:
1370 raise ValueError("Could not generate an unique seo key in 3 attempts")
1371 else:
1372 new_seo_key = current_seo_key
1373 last_set_seo_keys[language] = new_seo_key
1375 else:
1376 # We'll use the database-key instead
1377 last_set_seo_keys[language] = str(skel.dbEntity.key.id_or_name)
1379 # Store the current, active key for that language
1380 skel.dbEntity["viur"]["viurCurrentSeoKeys"][language] = last_set_seo_keys[language]
1382 skel.dbEntity["viur"].setdefault("viurActiveSeoKeys", [])
1383 for language, seo_key in last_set_seo_keys.items():
1384 if skel.dbEntity["viur"]["viurCurrentSeoKeys"][language] not in \
1385 skel.dbEntity["viur"]["viurActiveSeoKeys"]:
1386 # Ensure the current, active seo key is in the list of all seo keys
1387 skel.dbEntity["viur"]["viurActiveSeoKeys"].insert(0, seo_key)
1388 if str(skel.dbEntity.key.id_or_name) not in skel.dbEntity["viur"]["viurActiveSeoKeys"]:
1389 # Ensure that key is also in there
1390 skel.dbEntity["viur"]["viurActiveSeoKeys"].insert(0, str(skel.dbEntity.key.id_or_name))
1391 # Trim to the last 200 used entries
1392 skel.dbEntity["viur"]["viurActiveSeoKeys"] = skel.dbEntity["viur"]["viurActiveSeoKeys"][:200]
1393 # Store lastRequestedKeys so further updates can run more efficient
1394 skel.dbEntity["viur"]["viurLastRequestedSeoKeys"] = current_seo_keys
1396 # mark entity as "dirty" when update_relations is set, to zero otherwise.
1397 skel.dbEntity["viur"]["delayedUpdateTag"] = time.time() if update_relations else 0
1399 skel.dbEntity = skel.preProcessSerializedData(skel.dbEntity)
1401 # Allow the database adapter to apply last minute changes to the object
1402 for adapter in skel.database_adapters:
1403 adapter.prewrite(skel, is_add, change_list)
1405 # ViUR2 import compatibility - remove properties containing. if we have a dict with the same name
1406 def fixDotNames(entity):
1407 for k, v in list(entity.items()):
1408 if isinstance(v, dict):
1409 for k2, v2 in list(entity.items()):
1410 if k2.startswith(f"{k}."):
1411 del entity[k2]
1412 backupKey = k2.replace(".", "__")
1413 entity[backupKey] = v2
1414 entity.exclude_from_indexes = set(entity.exclude_from_indexes) | {backupKey}
1415 fixDotNames(v)
1416 elif isinstance(v, list):
1417 for x in v:
1418 if isinstance(x, dict):
1419 fixDotNames(x)
1421 # FIXME: REMOVE IN VIUR4
1422 if conf.viur2import_blobsource: # Try to fix these only when converting from ViUR2
1423 fixDotNames(skel.dbEntity)
1425 # Write the core entry back
1426 db.Put(skel.dbEntity)
1428 # Now write the blob-lock object
1429 blob_list = skel.preProcessBlobLocks(blob_list)
1430 if blob_list is None:
1431 raise ValueError("Did you forget to return the blob_list somewhere inside getReferencedBlobs()?")
1432 if None in blob_list:
1433 msg = f"None is not valid in {blob_list=}"
1434 logging.error(msg)
1435 raise ValueError(msg)
1437 if not is_add and (old_blob_lock_obj := db.Get(db.Key("viur-blob-locks", db_key.id_or_name))):
1438 removed_blobs = set(old_blob_lock_obj.get("active_blob_references", [])) - blob_list
1439 old_blob_lock_obj["active_blob_references"] = list(blob_list)
1440 if old_blob_lock_obj["old_blob_references"] is None:
1441 old_blob_lock_obj["old_blob_references"] = list(removed_blobs)
1442 else:
1443 old_blob_refs = set(old_blob_lock_obj["old_blob_references"])
1444 old_blob_refs.update(removed_blobs) # Add removed blobs
1445 old_blob_refs -= blob_list # Remove active blobs
1446 old_blob_lock_obj["old_blob_references"] = list(old_blob_refs)
1448 old_blob_lock_obj["has_old_blob_references"] = bool(old_blob_lock_obj["old_blob_references"])
1449 old_blob_lock_obj["is_stale"] = False
1450 db.Put(old_blob_lock_obj)
1451 else: # We need to create a new blob-lock-object
1452 blob_lock_obj = db.Entity(db.Key("viur-blob-locks", skel.dbEntity.key.id_or_name))
1453 blob_lock_obj["active_blob_references"] = list(blob_list)
1454 blob_lock_obj["old_blob_references"] = []
1455 blob_lock_obj["has_old_blob_references"] = False
1456 blob_lock_obj["is_stale"] = False
1457 db.Put(blob_lock_obj)
1459 return skel.dbEntity.key, write_skel, change_list, is_add
1461 # Parse provided key, if any, and set it to skel["key"]
1462 if key:
1463 skel["key"] = db.keyHelper(key, skel.kindName)
1465 # Run transactional function
1466 if db.IsInTransaction():
1467 key, skel, change_list, is_add = __txn_write(skel)
1468 else:
1469 key, skel, change_list, is_add = db.RunInTransaction(__txn_write, skel)
1471 for bone_name, bone in skel.items():
1472 bone.postSavedHandler(skel, bone_name, key)
1474 skel.postSavedHandler(key, skel.dbEntity)
1476 if update_relations and not is_add:
1477 if change_list and len(change_list) < 5: # Only a few bones have changed, process these individually
1478 for idx, changed_bone in enumerate(change_list):
1479 updateRelations(key, time.time() + 1, changed_bone, _countdown=10 * idx)
1480 else: # Update all inbound relations, regardless of which bones they mirror
1481 updateRelations(key, time.time() + 1, None)
1483 # Trigger the database adapter of the changes made to the entry
1484 for adapter in skel.database_adapters:
1485 adapter.write(skel, is_add, change_list)
1487 return skel
1489 @classmethod
1490 def delete(cls, skel: SkeletonInstance, key: t.Optional[KeyType] = None) -> None:
1491 """
1492 Deletes the entity associated with the current Skeleton from the data store.
1494 :param key: Allows to specify a key that is used for deletion, otherwise skel["key"] will be used.
1495 """
1497 def __txn_delete(skel: SkeletonInstance, key: db.Key):
1498 if not skel.read(key):
1499 raise ValueError("This skeleton is not in the database (anymore?)!")
1501 # Is there any relation to this Skeleton which prevents the deletion?
1502 locked_relation = (
1503 db.Query("viur-relations")
1504 .filter("dest.__key__ =", key)
1505 .filter("viur_relational_consistency =", RelationalConsistency.PreventDeletion.value)
1506 ).getEntry()
1508 if locked_relation is not None:
1509 raise errors.Locked("This entry is still referenced by other Skeletons, which prevents deleting!")
1511 # Ensure that any value lock objects remaining for this entry are being deleted
1512 viur_data = skel.dbEntity.get("viur") or {}
1514 for boneName, bone in skel.items():
1515 bone.delete(skel, boneName)
1516 if bone.unique:
1517 flushList = []
1518 for lockValue in viur_data.get(f"{boneName}_uniqueIndexValue") or []:
1519 lockKey = db.Key(f"{skel.kindName}_{boneName}_uniquePropertyIndex", lockValue)
1520 lockObj = db.Get(lockKey)
1521 if not lockObj:
1522 logging.error(f"{lockKey=} missing!")
1523 elif lockObj["references"] != key.id_or_name:
1524 logging.error(
1525 f"""{key!r} does not hold lock for {lockKey!r}""")
1526 else:
1527 flushList.append(lockObj)
1528 if flushList:
1529 db.Delete(flushList)
1531 # Delete the blob-key lock object
1532 lockObjectKey = db.Key("viur-blob-locks", key.id_or_name)
1533 lockObj = db.Get(lockObjectKey)
1535 if lockObj is not None:
1536 if lockObj["old_blob_references"] is None and lockObj["active_blob_references"] is None:
1537 db.Delete(lockObjectKey) # Nothing to do here
1538 else:
1539 if lockObj["old_blob_references"] is None:
1540 # No old stale entries, move active_blob_references -> old_blob_references
1541 lockObj["old_blob_references"] = lockObj["active_blob_references"]
1542 elif lockObj["active_blob_references"] is not None:
1543 # Append the current references to the list of old & stale references
1544 lockObj["old_blob_references"] += lockObj["active_blob_references"]
1545 lockObj["active_blob_references"] = [] # There are no active ones left
1546 lockObj["is_stale"] = True
1547 lockObj["has_old_blob_references"] = True
1548 db.Put(lockObj)
1550 db.Delete(key)
1551 processRemovedRelations(key)
1553 if key := (key or skel["key"]):
1554 key = db.keyHelper(key, skel.kindName)
1555 else:
1556 raise ValueError("This skeleton has no key!")
1558 # Full skeleton is required to have all bones!
1559 skel = skeletonByKind(skel.kindName)()
1561 if db.IsInTransaction():
1562 __txn_delete(skel, key)
1563 else:
1564 db.RunInTransaction(__txn_delete, skel, key)
1566 for boneName, bone in skel.items():
1567 bone.postDeletedHandler(skel, boneName, key)
1569 skel.postDeletedHandler(key)
1571 # Inform the custom DB Adapter
1572 for adapter in skel.database_adapters:
1573 adapter.delete(skel)
1575 @classmethod
1576 def patch(
1577 cls,
1578 skel: SkeletonInstance,
1579 values: t.Optional[dict | t.Callable[[SkeletonInstance], None]] = {},
1580 *,
1581 key: t.Optional[db.Key | int | str] = None,
1582 check: t.Optional[dict | t.Callable[[SkeletonInstance], None]] = None,
1583 create: t.Optional[bool | dict | t.Callable[[SkeletonInstance], None]] = None,
1584 update_relations: bool = True,
1585 ignore: t.Optional[t.Iterable[str]] = (),
1586 retry: int = 0,
1587 ) -> SkeletonInstance:
1588 """
1589 Performs an edit operation on a Skeleton within a transaction.
1591 The transaction performs a read, sets bones and afterwards does a write with exclusive access on the
1592 given Skeleton and its underlying database entity.
1594 All value-dicts that are being fed to this function are provided to `skel.fromClient()`. Instead of dicts,
1595 a callable can also be given that can individually modify the Skeleton that is edited.
1597 :param values: A dict of key-values to update on the entry, or a callable that is executed within
1598 the transaction.
1600 This dict allows for a special notation: Keys starting with "+" or "-" are added or substracted to the
1601 given value, which can be used for counters.
1602 :param key: A :class:`viur.core.db.Key`, string, or int; from which the data shall be fetched.
1603 If not provided, skel["key"] will be used.
1604 :param check: An optional dict of key-values or a callable to check on the Skeleton before updating.
1605 If something fails within this check, an AssertionError is being raised.
1606 :param create: Allows to specify a dict or initial callable that is executed in case the Skeleton with the
1607 given key does not exist.
1608 :param update_relations: Trigger update relations task on success. Defaults to False.
1609 :param trust: Use internal `fromClient` with trusted data (may change readonly-bones)
1610 :param retry: On ViurDatastoreError, retry for this amount of times.
1612 If the function does not raise an Exception, all went well. The function always returns the input Skeleton.
1614 Raises:
1615 ValueError: In case parameters where given wrong or incomplete.
1616 AssertionError: In case an asserted check parameter did not match.
1617 ReadFromClientException: In case a skel.fromClient() failed with a high severity.
1618 """
1620 # Transactional function
1621 def __update_txn():
1622 # Try to read the skeleton, create on demand
1623 if not skel.read(key):
1624 if create is None or create is False:
1625 raise ValueError("Creation during update is forbidden - explicitly provide `create=True` to allow.")
1627 if not (key or skel["key"]) and create in (False, None):
1628 return ValueError("No valid key provided")
1630 if key or skel["key"]:
1631 skel["key"] = db.keyHelper(key or skel["key"], skel.kindName)
1633 if isinstance(create, dict):
1634 if create and not skel.fromClient(create, amend=True, ignore=ignore):
1635 raise ReadFromClientException(skel.errors)
1636 elif callable(create):
1637 create(skel)
1638 elif create is not True:
1639 raise ValueError("'create' must either be dict or a callable.")
1641 # Handle check
1642 if isinstance(check, dict):
1643 for bone, value in check.items():
1644 if skel[bone] != value:
1645 raise AssertionError(f"{bone} contains {skel[bone]!r}, expecting {value!r}")
1647 elif callable(check):
1648 check(skel)
1650 # Set values
1651 if isinstance(values, dict):
1652 if values and not skel.fromClient(values, amend=True, ignore=ignore):
1653 raise ReadFromClientException(skel.errors)
1655 # Special-feature: "+" and "-" prefix for simple calculations
1656 # TODO: This can maybe integrated into skel.fromClient() later...
1657 for name, value in values.items():
1658 match name[0]:
1659 case "+": # Increment by value?
1660 skel[name[1:]] += value
1661 case "-": # Decrement by value?
1662 skel[name[1:]] -= value
1664 elif callable(values):
1665 values(skel)
1667 else:
1668 raise ValueError("'values' must either be dict or a callable.")
1670 return skel.write(update_relations=update_relations)
1672 if not db.IsInTransaction:
1673 # Retry loop
1674 while True:
1675 try:
1676 return db.RunInTransaction(__update_txn)
1678 except db.ViurDatastoreError as e:
1679 retry -= 1
1680 if retry < 0:
1681 raise
1683 logging.debug(f"{e}, retrying {retry} more times")
1685 time.sleep(1)
1686 else:
1687 return __update_txn()
1689 @classmethod
1690 def preProcessBlobLocks(cls, skel: SkeletonInstance, locks):
1691 """
1692 Can be overridden to modify the list of blobs referenced by this skeleton
1693 """
1694 return locks
1696 @classmethod
1697 def preProcessSerializedData(cls, skel: SkeletonInstance, entity):
1698 """
1699 Can be overridden to modify the :class:`viur.core.db.Entity` before its actually
1700 written to the data store.
1701 """
1702 return entity
1704 @classmethod
1705 def postSavedHandler(cls, skel: SkeletonInstance, key, dbObj):
1706 """
1707 Can be overridden to perform further actions after the entity has been written
1708 to the data store.
1709 """
1710 pass
1712 @classmethod
1713 def postDeletedHandler(cls, skel: SkeletonInstance, key):
1714 """
1715 Can be overridden to perform further actions after the entity has been deleted
1716 from the data store.
1717 """
1718 pass
1720 @classmethod
1721 def getCurrentSEOKeys(cls, skel: SkeletonInstance) -> None | dict[str, str]:
1722 """
1723 Should be overridden to return a dictionary of language -> SEO-Friendly key
1724 this entry should be reachable under. How theses names are derived are entirely up to the application.
1725 If the name is already in use for this module, the server will automatically append some random string
1726 to make it unique.
1727 :return:
1728 """
1729 return
1732class RelSkel(BaseSkeleton):
1733 """
1734 This is a Skeleton-like class that acts as a container for Skeletons used as a
1735 additional information data skeleton for
1736 :class:`~viur.core.bones.extendedRelationalBone.extendedRelationalBone`.
1738 It needs to be sub-classed where information about the kindName and its attributes
1739 (bones) are specified.
1741 The Skeleton stores its bones in an :class:`OrderedDict`-Instance, so the definition order of the
1742 contained bones remains constant.
1743 """
1745 def serialize(self, parentIndexed):
1746 if self.dbEntity is None:
1747 self.dbEntity = db.Entity()
1748 for key, _bone in self.items():
1749 # if key in self.accessedValues:
1750 _bone.serialize(self, key, parentIndexed)
1751 # if "key" in self: # Write the key seperatly, as the base-bone doesn't store it
1752 # dbObj["key"] = self["key"]
1753 # FIXME: is this a good idea? Any other way to ensure only bones present in refKeys are serialized?
1754 return self.dbEntity
1756 def unserialize(self, values: db.Entity | dict):
1757 """
1758 Loads 'values' into this skeleton.
1760 :param values: dict with values we'll assign to our bones
1761 """
1762 if not isinstance(values, db.Entity):
1763 self.dbEntity = db.Entity()
1765 if values:
1766 self.dbEntity.update(values)
1767 else:
1768 self.dbEntity = values
1770 self.accessedValues = {}
1771 self.renderAccessedValues = {}
1774class RefSkel(RelSkel):
1775 @classmethod
1776 def fromSkel(cls, kindName: str, *args: list[str]) -> t.Type[RefSkel]:
1777 """
1778 Creates a ``RefSkel`` from a skeleton-class using only the bones explicitly named in ``args``.
1780 :param args: List of bone names we'll adapt
1781 :return: A new instance of RefSkel
1782 """
1783 newClass = type("RefSkelFor" + kindName, (RefSkel,), {})
1784 fromSkel = skeletonByKind(kindName)
1785 newClass.kindName = kindName
1786 bone_map = {}
1787 for arg in args:
1788 bone_map |= {k: fromSkel.__boneMap__[k] for k in fnmatch.filter(fromSkel.__boneMap__.keys(), arg)}
1789 newClass.__boneMap__ = bone_map
1790 return newClass
1792 def read(self, key: t.Optional[db.Key | str | int] = None) -> SkeletonInstance:
1793 """
1794 Read full skeleton instance referenced by the RefSkel from the database.
1796 Can be used for reading the full Skeleton from a RefSkel.
1797 The `key` parameter also allows to read another, given key from the related kind.
1799 :raise ValueError: If the entry is no longer in the database.
1800 """
1801 skel = skeletonByKind(self.kindName)()
1803 if not skel.read(key or self["key"]):
1804 raise ValueError(f"""The key {key or self["key"]!r} seems to be gone""")
1806 return skel
1809class SkelList(list):
1810 """
1811 This class is used to hold multiple skeletons together with other, commonly used information.
1813 SkelLists are returned by Skel().all()...fetch()-constructs and provide additional information
1814 about the data base query, for fetching additional entries.
1816 :ivar cursor: Holds the cursor within a query.
1817 :vartype cursor: str
1818 """
1820 __slots__ = (
1821 "baseSkel",
1822 "customQueryInfo",
1823 "getCursor",
1824 "get_orders",
1825 "renderPreparation",
1826 )
1828 def __init__(self, baseSkel=None):
1829 """
1830 :param baseSkel: The baseclass for all entries in this list
1831 """
1832 super(SkelList, self).__init__()
1833 self.baseSkel = baseSkel or {}
1834 self.getCursor = lambda: None
1835 self.get_orders = lambda: None
1836 self.renderPreparation = None
1837 self.customQueryInfo = {}
1840# Module functions
1843def skeletonByKind(kindName: str) -> t.Type[Skeleton]:
1844 """
1845 Returns the Skeleton-Class for the given kindName. That skeleton must exist, otherwise an exception is raised.
1846 :param kindName: The kindname to retreive the skeleton for
1847 :return: The skeleton-class for that kind
1848 """
1849 assert kindName in MetaBaseSkel._skelCache, f"Unknown skeleton {kindName=}"
1850 return MetaBaseSkel._skelCache[kindName]
1853def listKnownSkeletons() -> list[str]:
1854 """
1855 :return: A list of all known kindnames (all kindnames for which a skeleton is defined)
1856 """
1857 return sorted(MetaBaseSkel._skelCache.keys())
1860def iterAllSkelClasses() -> t.Iterable[Skeleton]:
1861 """
1862 :return: An iterator that yields each Skeleton-Class once. (Only top-level skeletons are returned, so no
1863 RefSkel classes will be included)
1864 """
1865 for cls in list(MetaBaseSkel._allSkelClasses): # We'll add new classes here during setSystemInitialized()
1866 yield cls
1869### Tasks ###
1871@CallDeferred
1872def processRemovedRelations(removedKey: db.Key, cursor=None):
1873 updateListQuery = (
1874 db.Query("viur-relations")
1875 .filter("dest.__key__ =", removedKey)
1876 .filter("viur_relational_consistency >", RelationalConsistency.PreventDeletion.value)
1877 )
1878 updateListQuery = updateListQuery.setCursor(cursor)
1879 updateList = updateListQuery.run(limit=5)
1881 for entry in updateList:
1882 skel = skeletonByKind(entry["viur_src_kind"])()
1884 if not skel.read(entry["src"].key):
1885 raise ValueError(f"processRemovedRelations detects inconsistency on src={entry['src'].key!r}")
1887 if entry["viur_relational_consistency"] == RelationalConsistency.SetNull.value:
1888 found = False
1890 for key, bone in skel.items():
1891 if isinstance(bone, RelationalBone):
1892 if relational_value := skel[key]:
1893 # TODO: LanguageWrapper is not considered here (<RelationalBone(languages=[...])>)
1894 if isinstance(relational_value, dict):
1895 if relational_value["dest"]["key"] == removedKey:
1896 skel[key] = None
1897 found = True
1899 elif isinstance(relational_value, list):
1900 skel[key] = [entry for entry in relational_value if entry["dest"]["key"] != removedKey]
1901 found = True
1903 else:
1904 raise NotImplementedError(f"In {entry['src'].key!r}, no handling for {relational_value=}")
1906 if found:
1907 skel.write(update_relations=False)
1909 else:
1910 logging.critical(f"""Cascade deletion of {skel["key"]!r}""")
1911 skel.delete()
1913 if len(updateList) == 5:
1914 processRemovedRelations(removedKey, updateListQuery.getCursor())
1917@CallDeferred
1918def updateRelations(destKey: db.Key, minChangeTime: int, changedBone: t.Optional[str], cursor: t.Optional[str] = None):
1919 """
1920 This function updates Entities, which may have a copy of values from another entity which has been recently
1921 edited (updated). In ViUR, relations are implemented by copying the values from the referenced entity into the
1922 entity that's referencing them. This allows ViUR to run queries over properties of referenced entities and
1923 prevents additional db.Get's to these referenced entities if the main entity is read. However, this forces
1924 us to track changes made to entities as we might have to update these mirrored values. This is the deferred
1925 call from meth:`viur.core.skeleton.Skeleton.write()` after an update (edit) on one Entity to do exactly that.
1927 :param destKey: The database-key of the entity that has been edited
1928 :param minChangeTime: The timestamp on which the edit occurred. As we run deferred, and the entity might have
1929 been edited multiple times before we get acutally called, we can ignore entities that have been updated
1930 in the meantime as they're already up2date
1931 :param changedBone: If set, we'll update only entites that have a copy of that bone. Relations mirror only
1932 key and name by default, so we don't have to update these if only another bone has been changed.
1933 :param cursor: The database cursor for the current request as we only process five entities at once and then
1934 defer again.
1935 """
1936 logging.debug(f"Starting updateRelations for {destKey=}; {minChangeTime=}, {changedBone=}, {cursor=}")
1937 if request_data := current.request_data.get():
1938 request_data["__update_relations_bone"] = changedBone
1939 updateListQuery = (
1940 db.Query("viur-relations")
1941 .filter("dest.__key__ =", destKey)
1942 .filter("viur_delayed_update_tag <", minChangeTime)
1943 .filter("viur_relational_updateLevel =", RelationalUpdateLevel.Always.value)
1944 )
1945 if changedBone:
1946 updateListQuery.filter("viur_foreign_keys =", changedBone)
1947 if cursor:
1948 updateListQuery.setCursor(cursor)
1949 updateList = updateListQuery.run(limit=5)
1951 def updateTxn(skel, key, srcRelKey):
1952 if not skel.read(key):
1953 logging.warning(f"Cannot update stale reference to {key=} (referenced from {srcRelKey=})")
1954 return
1956 skel.refresh()
1957 skel.write(update_relations=False)
1959 for srcRel in updateList:
1960 try:
1961 skel = skeletonByKind(srcRel["viur_src_kind"])()
1962 except AssertionError:
1963 logging.info(f"""Ignoring {srcRel.key!r} which refers to unknown kind {srcRel["viur_src_kind"]!r}""")
1964 continue
1965 if db.IsInTransaction():
1966 updateTxn(skel, srcRel["src"].key, srcRel.key)
1967 else:
1968 db.RunInTransaction(updateTxn, skel, srcRel["src"].key, srcRel.key)
1969 nextCursor = updateListQuery.getCursor()
1970 if len(updateList) == 5 and nextCursor:
1971 updateRelations(destKey, minChangeTime, changedBone, nextCursor)
1974@CallableTask
1975class TaskUpdateSearchIndex(CallableTaskBase):
1976 """
1977 This tasks loads and saves *every* entity of the given module.
1978 This ensures an updated searchIndex and verifies consistency of this data.
1979 """
1980 key = "rebuildSearchIndex"
1981 name = "Rebuild search index"
1982 descr = "This task can be called to update search indexes and relational information."
1984 def canCall(self) -> bool:
1985 """Checks wherever the current user can execute this task"""
1986 user = current.user.get()
1987 return user is not None and "root" in user["access"]
1989 def dataSkel(self):
1990 modules = ["*"] + listKnownSkeletons()
1991 modules.sort()
1992 skel = BaseSkeleton().clone()
1993 skel.module = SelectBone(descr="Module", values={x: translate(x) for x in modules}, required=True)
1994 return skel
1996 def execute(self, module, *args, **kwargs):
1997 usr = current.user.get()
1998 if not usr:
1999 logging.warning("Don't know who to inform after rebuilding finished")
2000 notify = None
2001 else:
2002 notify = usr["name"]
2004 if module == "*":
2005 for module in listKnownSkeletons():
2006 logging.info("Rebuilding search index for module %r", module)
2007 self._run(module, notify)
2008 else:
2009 self._run(module, notify)
2011 @staticmethod
2012 def _run(module: str, notify: str):
2013 Skel = skeletonByKind(module)
2014 if not Skel:
2015 logging.error("TaskUpdateSearchIndex: Invalid module")
2016 return
2017 RebuildSearchIndex.startIterOnQuery(Skel().all(), {"notify": notify, "module": module})
2020class RebuildSearchIndex(QueryIter):
2021 @classmethod
2022 def handleEntry(cls, skel: SkeletonInstance, customData: dict[str, str]):
2023 skel.refresh()
2024 skel.write(update_relations=False)
2026 @classmethod
2027 def handleError(cls, skel, customData, exception) -> bool:
2028 logging.exception(f'{cls.__qualname__}.handleEntry failed on skel {skel["key"]=!r}: {exception}')
2029 try:
2030 logging.debug(f"{skel=!r}")
2031 except Exception: # noqa
2032 logging.warning("Failed to dump skel")
2033 logging.debug(f"{skel.dbEntity=}")
2034 return True
2036 @classmethod
2037 def handleFinish(cls, totalCount: int, customData: dict[str, str]):
2038 QueryIter.handleFinish(totalCount, customData)
2039 if not customData["notify"]:
2040 return
2041 txt = (
2042 f"{conf.instance.project_id}: Rebuild search index finished for {customData['module']}\n\n"
2043 f"ViUR finished to rebuild the search index for module {customData['module']}.\n"
2044 f"{totalCount} records updated in total on this kind."
2045 )
2046 try:
2047 email.send_email(dests=customData["notify"], stringTemplate=txt, skel=None)
2048 except Exception as exc: # noqa; OverQuota, whatever
2049 logging.exception(f'Failed to notify {customData["notify"]}')
2052### Vacuum Relations
2054@CallableTask
2055class TaskVacuumRelations(TaskUpdateSearchIndex):
2056 """
2057 Checks entries in viur-relations and verifies that the src-kind
2058 and it's RelationalBone still exists.
2059 """
2060 key = "vacuumRelations"
2061 name = "Vacuum viur-relations (dangerous)"
2062 descr = "Drop stale inbound relations for the given kind"
2064 def execute(self, module: str, *args, **kwargs):
2065 usr = current.user.get()
2066 if not usr:
2067 logging.warning("Don't know who to inform after rebuilding finished")
2068 notify = None
2069 else:
2070 notify = usr["name"]
2071 processVacuumRelationsChunk(module.strip(), None, notify=notify)
2074@CallDeferred
2075def processVacuumRelationsChunk(
2076 module: str, cursor, count_total: int = 0, count_removed: int = 0, notify=None
2077):
2078 """
2079 Processes 25 Entries and calls the next batch
2080 """
2081 query = db.Query("viur-relations")
2082 if module != "*":
2083 query.filter("viur_src_kind =", module)
2084 query.setCursor(cursor)
2085 for relation_object in query.run(25):
2086 count_total += 1
2087 if not (src_kind := relation_object.get("viur_src_kind")):
2088 logging.critical("We got an relation-object without a src_kind!")
2089 continue
2090 if not (src_prop := relation_object.get("viur_src_property")):
2091 logging.critical("We got an relation-object without a src_prop!")
2092 continue
2093 try:
2094 skel = skeletonByKind(src_kind)()
2095 except AssertionError:
2096 # The referenced skeleton does not exist in this data model -> drop that relation object
2097 logging.info(f"Deleting {relation_object.key} which refers to unknown kind {src_kind}")
2098 db.Delete(relation_object)
2099 count_removed += 1
2100 continue
2101 if src_prop not in skel:
2102 logging.info(f"Deleting {relation_object.key} which refers to "
2103 f"non-existing RelationalBone {src_prop} of {src_kind}")
2104 db.Delete(relation_object)
2105 count_removed += 1
2106 logging.info(f"END processVacuumRelationsChunk {module}, "
2107 f"{count_total} records processed, {count_removed} removed")
2108 if new_cursor := query.getCursor():
2109 # Start processing of the next chunk
2110 processVacuumRelationsChunk(module, new_cursor, count_total, count_removed, notify)
2111 elif notify:
2112 txt = (
2113 f"{conf.instance.project_id}: Vacuum relations finished for {module}\n\n"
2114 f"ViUR finished to vacuum viur-relations for module {module}.\n"
2115 f"{count_total} records processed, "
2116 f"{count_removed} entries removed"
2117 )
2118 try:
2119 email.send_email(dests=notify, stringTemplate=txt, skel=None)
2120 except Exception as exc: # noqa; OverQuota, whatever
2121 logging.exception(f"Failed to notify {notify}")
2124# Forward our references to SkelInstance to the database (needed for queries)
2125db.config["SkeletonInstanceRef"] = SkeletonInstance
2127# DEPRECATED ATTRIBUTES HANDLING
2129__DEPRECATED_NAMES = {
2130 # stuff prior viur-core < 3.6
2131 "seoKeyBone": ("SeoKeyBone", SeoKeyBone),
2132}
2135def __getattr__(attr: str) -> object:
2136 if entry := __DEPRECATED_NAMES.get(attr):
2137 func = entry[1]
2138 msg = f"{attr} was replaced by {entry[0]}"
2139 warnings.warn(msg, DeprecationWarning, stacklevel=2)
2140 logging.warning(msg, stacklevel=2)
2141 return func
2143 return super(__import__(__name__).__class__).__getattribute__(attr)