Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/modules/history.py: 0%
206 statements
« prev ^ index » next coverage.py v7.15.0, created at 2026-07-07 12:23 +0000
1import difflib
2import json
3import logging
4import typing as t
5from google.cloud import exceptions, bigquery
6from viur.core import db, conf, utils, current, tasks
7from viur.core.bones import *
8from viur.core.prototypes.list import List
9from viur.core.render.json.default import CustomJsonEncoder
10from viur.core.skeleton import SkeletonInstance, Skeleton, DatabaseAdapter
13class HistorySkel(Skeleton):
14 """
15 Skeleton used for a ViUR history entry to log any relevant changes
16 in other Skeletons.
18 The ViurHistorySkel is also used as the base for a biquery logging table,
19 see below.
20 """
22 kindName = "viur-history"
23 creationdate = changedate = None
25 version = NumericBone(
26 descr="Version",
27 )
29 action = StringBone(
30 descr="Action",
31 )
33 tags = StringBone(
34 descr="Tags",
35 multiple=True,
36 )
38 timestamp = DateBone(
39 descr="Timestamp",
40 defaultValue=lambda *args, **kwargs: utils.utcNow(),
41 localize=True,
42 )
44 user = UserBone(
45 updateLevel=RelationalUpdateLevel.OnValueAssignment,
46 searchable=True,
47 )
49 name = StringBone(
50 descr="Name",
51 searchable=True,
52 )
54 descr = StringBone(
55 descr="Description",
56 searchable=True,
57 )
59 current_kind = StringBone(
60 descr="Entry kind",
61 searchable=True,
62 )
64 current_key = KeyBone(
65 descr="Entity key",
66 )
68 current = JsonBone(
69 descr="Entity content",
70 indexed=False,
71 )
73 changed_fields = StringBone(
74 descr="Changed fields",
75 multiple=True
76 )
78 diff = RawBone(
79 type_suffix="code.diff",
80 descr="Human-readable diff",
81 indexed=False,
82 )
85class BigQueryHistory:
86 """
87 Connector for BigQuery history entries.
88 """
90 PATH = f"""{conf.instance.project_id}.history.default"""
91 """
92 Path to the big query table for history entries.
93 """
95 SCHEMA = (
96 {
97 "type": "STRING",
98 "name": "key",
99 "mode": "REQUIRED",
100 "description": "unique identifier, hashed from kindname + timestamp",
101 },
102 {
103 "type": "NUMERIC",
104 "name": "version",
105 "mode": "REQUIRED",
106 "description": "log version",
107 },
108 {
109 "type": "STRING",
110 "name": "action",
111 "mode": "NULLABLE",
112 "description": "logged action",
113 },
114 {
115 "type": "STRING",
116 "name": "tags",
117 "mode": "REPEATED",
118 "description": "Additional tags for filtering",
119 },
120 {
121 "type": "DATETIME",
122 "name": "timestamp",
123 "mode": "REQUIRED",
124 "description": "datetime of logevent",
125 },
126 {
127 "type": "STRING",
128 "name": "timestamp_date",
129 "mode": "REQUIRED",
130 "description": "datetime of logevent: date",
131 },
132 {
133 "type": "STRING",
134 "name": "timestamp_period",
135 "mode": "REQUIRED",
136 "description": "datetime of logevent: period",
137 },
138 {
139 "type": "STRING",
140 "name": "user",
141 "mode": "NULLABLE",
142 "description": "user who trigged log event: key",
143 },
144 {
145 "type": "STRING",
146 "name": "user_name",
147 "mode": "NULLABLE",
148 "description": "user who trigged log event: username",
149 },
150 {
151 "type": "STRING",
152 "name": "user_firstname",
153 "mode": "NULLABLE",
154 "description": "user who trigged log event: firstname",
155 },
156 {
157 "type": "STRING",
158 "name": "user_lastname",
159 "mode": "NULLABLE",
160 "description": "user who trigged log event: lastname",
161 },
162 {
163 "type": "STRING",
164 "name": "name",
165 "mode": "NULLABLE",
166 "description": "readable name of the action",
167 },
168 {
169 "type": "STRING",
170 "name": "descr",
171 "mode": "NULLABLE",
172 "description": "readable event description",
173 },
174 {
175 "type": "STRING",
176 "name": "current_kind",
177 "mode": "NULLABLE",
178 "description": "kindname",
179 },
180 {
181 "type": "STRING",
182 "name": "current_key",
183 "mode": "NULLABLE",
184 "description": "url encoded datastore key",
185 },
186 {
187 "type": "JSON",
188 "name": "current",
189 "mode": "NULLABLE",
190 "description": "full content of the current entry",
191 },
192 {
193 "type": "JSON",
194 "name": "previous",
195 "mode": "NULLABLE",
196 "description": "previous full content of the entry before it changed",
197 },
198 {
199 "type": "STRING",
200 "name": "diff",
201 "mode": "NULLABLE",
202 "description": "diff data",
203 },
204 {
205 "type": "STRING",
206 "name": "changed_fields",
207 "mode": "REPEATED",
208 "description": "Changed fields from old to new",
209 },
210 )
211 """
212 Schema used for the BigQuery table for its initial construction.
213 Keep to the provided format!
214 """
216 def __init__(self):
217 super().__init__()
219 # checks for the table_path
220 if self.PATH.count(".") != 2:
221 raise ValueError("{self.PATH!r} must have exactly 3 parts that separated by a dot.")
223 self.client = bigquery.Client()
224 self.table = self.select_or_create_table()
226 def select_or_create_table(self):
227 try:
228 return self.client.get_table(self.PATH)
230 except exceptions.NotFound:
231 app, dataset, table = self.PATH.split(".")
232 logging.error(f"{app}:{dataset}:{table}")
233 # create dataset if needed
234 try:
235 self.client.get_dataset(dataset)
236 except exceptions.NotFound:
237 logging.info(f"Dataset {dataset!r} does not exist, creating")
238 self.client.create_dataset(dataset)
240 # create table if needed
241 try:
242 return self.client.get_table(self.PATH)
243 except exceptions.NotFound:
244 logging.info(f"Table {self.PATH!r} does not exist, creating")
245 self.client.create_table(
246 bigquery.Table(
247 self.PATH,
248 schema=self.SCHEMA
249 )
250 )
251 return self.client.get_table(self.PATH)
253 def write_row(self, data):
254 if res := self.client.insert_rows(self.table, [data]):
255 raise ValueError(res)
258class HistoryAdapter(DatabaseAdapter):
259 """
260 Generalized adapter for handling history events.
261 """
263 DEFAULT_EXCLUDES = {
264 "key",
265 "changedate",
266 "creationdate",
267 "importdate",
268 "viurCurrentSeoKeys",
269 }
270 """
271 Bones being ignored within history.
272 """
274 def __init__(self, excludes: t.Iterable[str] = DEFAULT_EXCLUDES):
275 super().__init__()
277 # add excludes to diff excludes
278 self.diff_excludes = set(excludes)
280 def prewrite(self, skel, is_add, change_list=()):
281 if not is_add: # edit
282 old_skel = skel.clone()
283 old_skel.read(skel["key"])
284 self.trigger("edit", old_skel, skel, change_list)
286 def write(self, skel, is_add, change_list=()):
287 if is_add: # add
288 self.trigger("add", None, skel)
290 def delete(self, skel):
291 self.trigger("delete", skel, None)
293 def trigger(
294 self,
295 action: str,
296 old_skel: SkeletonInstance,
297 new_skel: SkeletonInstance,
298 change_list: t.Iterable[str] = (),
299 ) -> str | None:
300 if not (history_module := getattr(conf.main_app, "history", None)):
301 logging.warning(
302 f"{old_skel or new_skel or self!r} uses {self.__class__.__name__}, but no 'history'-module found"
303 )
304 return None
306 # skip excluded actions like login or logout
307 if action in conf.history.excluded_actions:
308 return None
310 # skip when no user is available or provided
311 if not (user := current.user.get()):
312 return None
314 # FIXME: Turn change_list into set, in entire Core...
315 if change_list and not set(change_list).difference(self.diff_excludes):
316 logging.info("change_list is empty, nothing to write")
317 return None
319 # skip excluded kinds and history kind to avoid recursion
320 any_skel = (old_skel or new_skel)
321 if any_skel and (kindname := getattr(any_skel, "kindName", None)):
322 if kindname in conf.history.excluded_kinds:
323 return None
325 if kindname == "viur-history":
326 return None
328 return history_module.log(
329 action, old_skel, new_skel,
330 change_list=change_list,
331 user=user,
332 diff_excludes=self.diff_excludes,
333 )
336class History(List):
337 """
338 ViUR history module
339 """
340 kindName = "viur-history"
342 adminInfo = {
343 "name": "History",
344 "icon": "clock-history",
345 "filter": {
346 "orderby": "timestamp",
347 "orderdir": "desc",
348 },
349 "disabledActions": ["add", "clone", "delete"],
350 }
352 roles = {
353 "admin": "view",
354 }
356 HISTORY_VERSION = 1
357 """
358 History format version.
359 """
361 BigQueryHistoryCls = BigQueryHistory
362 """
363 The connector class used to store entries to BigQuery.
364 """
366 def __init__(self, *args, **kwargs):
367 super().__init__(*args, **kwargs)
369 if self.BigQueryHistoryCls and "bigquery" in conf.history.databases:
370 assert issubclass(self.BigQueryHistoryCls, BigQueryHistory)
371 self.bigquery = self.BigQueryHistoryCls()
372 else:
373 self.bigquery = None
375 def skel(self, **kwargs):
376 # Make all bones readonly!
377 skel = super().skel(**kwargs).clone()
378 skel.readonly()
379 return skel
381 def canEdit(self, skel):
382 return self.canView(skel) # this is needed to open an entry in admin (all bones are readonly!)
384 def canDelete(self, _skel):
385 return False
387 def canAdd(self):
388 return False
390 # Module-specific functions
391 @staticmethod
392 def _create_diff(new: dict, old: dict, diff_excludes: t.Iterable[str] = set()):
393 """
394 Creates a textual diff format string from the contents of two dicts.
395 """
396 diffs = []
398 # Run over union of both dict keys
399 keys = old.keys() | new.keys()
400 keys = set(keys).difference(diff_excludes)
401 keys = sorted(keys)
403 for key in keys:
404 def expand(name, obj):
405 ret = {}
406 if isinstance(obj, list):
407 for i, val in enumerate(obj):
408 ret.update(expand(name + (str(i),), val))
409 elif isinstance(obj, dict):
410 for key, val in obj.items():
411 ret.update(expand(name + (str(key),), val))
412 else:
413 name = ".".join(name)
414 ret[name] = json.dumps(obj, cls=CustomJsonEncoder, ensure_ascii=False)
416 return ret
418 values = tuple(expand((key,), obj.get(key)) for obj in (old, new))
419 assert len(values) == 2
421 for value_key in sorted(set(values[0].keys() | values[1].keys())):
423 diff = "\n".join(
424 difflib.unified_diff(
425 (values[0].get(value_key) or "").splitlines(),
426 (values[1].get(value_key) or "").splitlines(),
427 value_key, value_key,
428 old.get("changedate") or utils.utcNow().isoformat(),
429 new.get("changedate") or utils.utcNow().isoformat(),
430 n=1
431 )
432 )
434 if diff := diff.strip():
435 diffs.append(diff)
437 return "\n".join(diffs).replace("\n\n", "\n")
439 def build_name(self, skel: SkeletonInstance) -> str | None:
440 """
441 Helper function to figure out a name from the skeleton
442 """
444 if not skel:
445 return None
447 if "name" in skel:
448 name = skel.dump()
450 if isinstance(skel["name"], str):
451 return skel["name"]
453 return name
455 return skel["key"].id_or_name
457 def build_descr(self, action: str, skel: SkeletonInstance, change_list: t.Iterable[str]) -> str | None:
458 """
459 Helper function to build a description about the change to the skeleton
460 """
461 if not skel:
462 return action
464 match action:
465 case "add":
466 return (
467 f"""A new entry with the kind {skel.kindName!r}"""
468 f""" and the key {skel["key"].id_or_name!r} was created."""
469 )
470 case "edit":
471 return (
472 f"""The entry {skel["key"].id_or_name!r} of kind {skel.kindName!r} has been modified."""
473 f""" The following fields where changed: {", ".join(change_list)}."""
474 )
475 case "delete":
476 return f"""The entry {skel["key"].id_or_name!r} of kind {skel.kindName!r} has been deleted."""
478 return (
479 f"""The action {action!r} resulted in a change to the entry {skel["key"].id_or_name!r}"""
480 f""" of kind {skel.kindName!r}."""
481 )
483 def create_history_entry(
484 self,
485 action: str,
486 old_skel: t.Optional[SkeletonInstance] = None,
487 new_skel: t.Optional[SkeletonInstance] = None,
488 change_list: t.Iterable[str] = (),
489 descr: t.Optional[str] = None,
490 user: t.Optional[SkeletonInstance] = None,
491 tags: t.Iterable[str] = (),
492 diff_excludes: t.Set[str] = set(),
493 ):
494 """
495 Internal helper function that constructs a JSON-serializable form of the entry
496 that can either be written to datastore or another database.
497 """
498 skel = new_skel or old_skel
499 new_data = skel.dump(bones=change_list) if skel else {}
501 if change_list and old_skel != new_skel:
502 old_data = old_skel.dump(bones=change_list) if old_skel else {}
503 diff = self._create_diff(new_data, old_data, diff_excludes)
504 else:
505 old_data = {}
506 diff = ""
508 # set event tag, in case of an event-action
509 tags = set(tags)
511 # Event tag
512 if action.startswith("event-"):
513 tags.add("is-event")
515 ret = {
516 "action": action,
517 "current_key": skel and str(skel["key"]),
518 "current_kind": skel and getattr(skel, "kindName", None),
519 "current": new_data,
520 "changed_fields": change_list if change_list else [],
521 "descr": descr or self.build_descr(action, skel, change_list),
522 "diff": diff,
523 "name": self.build_name(skel) if skel else ((user and user["name"] or "") + " " + action),
524 "previous": old_data if old_data else None,
525 "tags": tuple(sorted(tags)),
526 "timestamp": utils.utcNow(),
527 "user_firstname": user and user["firstname"],
528 "user_lastname": user and user["lastname"],
529 "user_name": user and user["name"],
530 "user": user and user["key"],
531 "version": self.HISTORY_VERSION,
532 }
534 return ret
536 def log(
537 self,
538 action: str,
539 old_skel: t.Optional[SkeletonInstance] = None,
540 new_skel: t.Optional[SkeletonInstance] = None,
541 change_list: t.Iterable[str] = (),
542 descr: t.Optional[str] = None,
543 user: t.Optional[SkeletonInstance] = None,
544 tags: t.Iterable[str] = (),
545 diff_excludes: t.Set[str] = set(),
546 ) -> str | None:
547 """
548 Creates and persists a history entry for a skeleton change or a standalone event.
550 Builds the entry via :meth:`create_history_entry`, derives a deterministic key
551 from ``action``, ``current_kind``, and the current timestamp, then writes the
552 entry to all configured backends (``"viur"`` datastore and/or ``"bigquery"``)
553 as deferred tasks.
555 Both ``old_skel`` and ``new_skel`` are optional. For skeleton lifecycle actions
556 one of them is typically present (``"add"`` only has ``new_skel``, ``"delete"``
557 only has ``old_skel``, ``"edit"`` has both), but for pure event logging neither
558 is required.
560 :param action: Short identifier for what happened, e.g. ``"add"``, ``"edit"``,
561 ``"delete"``, or a custom ``"event-*"`` string.
562 :param old_skel: Skeleton state before the change, or ``None`` for pure events
563 and ``"add"`` actions.
564 :param new_skel: Skeleton state after the change, or ``None`` for pure events
565 and ``"delete"`` actions.
566 :param change_list: Names of the bones that were modified (used for ``"edit"`` actions).
567 :param descr: Optional human-readable description. Falls back to :meth:`build_descr`
568 when omitted.
569 :param user: Skeleton instance of the user who triggered the action.
570 :param tags: Additional string tags attached to the entry for filtering.
571 Entries with an ``"event-*"`` action automatically receive the ``"is-event"`` tag.
572 :param diff_excludes: Bone names to exclude from the unified diff computation.
573 :returns: The generated history entry key, or ``None`` if no entry was written.
574 """
576 # create entry
577 entry = self.create_history_entry(
578 action, old_skel, new_skel,
579 change_list=change_list,
580 descr=descr,
581 user=user,
582 tags=tags,
583 diff_excludes=diff_excludes,
584 )
586 # generate key from significant properties
587 key = "-".join(
588 part for part in (
589 entry["action"],
590 entry["current_kind"],
591 entry["timestamp"].isoformat()
592 ) if part
593 )
595 # write into datastore via history module
596 if "viur" in conf.history.databases:
597 self.write_to_viur_deferred(key, entry)
599 # write into BigQuery
600 if self.bigquery and "bigquery" in conf.history.databases:
601 # need to do this as biquery functions modifies entry and seems to be called first
602 if conf.instance.is_dev_server:
603 entry = entry.copy() # need to do this as biquery functions modifiy entry
605 self.write_to_bigquery_deferred(key, entry)
607 return key
609 def write_to_viur(self, key: str, entry: dict):
610 """
611 Write a history entry generated from an HistoryAdapter.
612 """
613 skel = self.addSkel()
615 for name, bone in skel.items():
616 if value := entry.get(name):
617 if isinstance(bone, (RelationalBone, RecordBone)):
618 skel.setBoneValue(name, value)
619 else:
620 skel[name] = value
622 skel.write(key=db.Key(skel.kindName, key))
624 logging.info(f"History entry {key=} written to datastore")
626 @tasks.CallDeferred
627 def write_to_viur_deferred(self, key: str, entry: dict):
628 self.write_to_viur(key, entry)
630 def write_to_bigquery(self, key: str, entry: dict):
631 entry["key"] = key
632 entry["timestamp_date"] = entry["timestamp"].strftime("%Y-%m-%d")
633 entry["timestamp_period"] = entry["timestamp"].strftime("%Y-%m")
634 entry["user"] = str(entry["user"]) if entry["user"] else None
636 self.bigquery.write_row(entry)
637 logging.info(f"History entry {key=} written to biquery")
639 @tasks.CallDeferred
640 def write_to_bigquery_deferred(self, key: str, entry: dict):
641 self.write_to_bigquery(key, entry)
644History.json = True
645History.admin = True