Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/modules/history.py: 0%
204 statements
« prev ^ index » next coverage.py v7.16.1, created at 2026-09-25 15:02 +0000
1import difflib
2import json
3import logging
4import typing as t
5from google.cloud import exceptions, bigquery
6from viur.core import db, conf, utils, current, tasks
7from viur.core.bones import *
8from viur.core.prototypes.list import List
9from viur.core.render.json.default import CustomJsonEncoder
10from viur.core.skeleton import SkeletonInstance, Skeleton, DatabaseAdapter
13class HistorySkel(Skeleton):
14 """
15 Skeleton used for a ViUR history entry to log any relevant changes
16 in other Skeletons.
18 The ViurHistorySkel is also used as the base for a biquery logging table,
19 see below.
20 """
22 kindName = "history"
23 creationdate = changedate = None
25 version = NumericBone(
26 descr="Version",
27 )
29 action = StringBone(
30 descr="Action",
31 )
33 tags = StringBone(
34 descr="Tags",
35 multiple=True,
36 )
38 timestamp = DateBone(
39 descr="Timestamp",
40 defaultValue=lambda *args, **kwargs: utils.utcNow(),
41 localize=True,
42 )
44 user = UserBone(
45 updateLevel=RelationalUpdateLevel.OnValueAssignment,
46 searchable=True,
47 )
49 name = StringBone(
50 descr="Name",
51 searchable=True,
52 )
54 descr = StringBone(
55 descr="Description",
56 searchable=True,
57 )
59 current_kind = StringBone(
60 descr="Entry kind",
61 searchable=True,
62 )
64 current_key = KeyBone(
65 descr="Entity key",
66 )
68 current = JsonBone(
69 descr="Entity content",
70 indexed=False,
71 )
73 changed_fields = StringBone(
74 descr="Changed fields",
75 multiple=True
76 )
78 diff = RawBone(
79 type_suffix="code.diff",
80 descr="Human-readable diff",
81 indexed=False,
82 )
85class BigQueryHistory:
86 """
87 Connector for BigQuery history entries.
88 """
90 PATH = f"""{conf.instance.project_id}.history.default"""
91 """
92 Path to the big query table for history entries.
93 """
95 SCHEMA = (
96 {
97 "type": "STRING",
98 "name": "key",
99 "mode": "REQUIRED",
100 "description": "unique identifier, hashed from kindname + timestamp",
101 },
102 {
103 "type": "NUMERIC",
104 "name": "version",
105 "mode": "REQUIRED",
106 "description": "log version",
107 },
108 {
109 "type": "STRING",
110 "name": "action",
111 "mode": "NULLABLE",
112 "description": "logged action",
113 },
114 {
115 "type": "STRING",
116 "name": "tags",
117 "mode": "REPEATED",
118 "description": "Additional tags for filtering",
119 },
120 {
121 "type": "DATETIME",
122 "name": "timestamp",
123 "mode": "REQUIRED",
124 "description": "datetime of logevent",
125 },
126 {
127 "type": "STRING",
128 "name": "timestamp_date",
129 "mode": "REQUIRED",
130 "description": "datetime of logevent: date",
131 },
132 {
133 "type": "STRING",
134 "name": "timestamp_period",
135 "mode": "REQUIRED",
136 "description": "datetime of logevent: period",
137 },
138 {
139 "type": "STRING",
140 "name": "user",
141 "mode": "NULLABLE",
142 "description": "user who trigged log event: key",
143 },
144 {
145 "type": "STRING",
146 "name": "user_name",
147 "mode": "NULLABLE",
148 "description": "user who trigged log event: username",
149 },
150 {
151 "type": "STRING",
152 "name": "user_firstname",
153 "mode": "NULLABLE",
154 "description": "user who trigged log event: firstname",
155 },
156 {
157 "type": "STRING",
158 "name": "user_lastname",
159 "mode": "NULLABLE",
160 "description": "user who trigged log event: lastname",
161 },
162 {
163 "type": "STRING",
164 "name": "name",
165 "mode": "NULLABLE",
166 "description": "readable name of the action",
167 },
168 {
169 "type": "STRING",
170 "name": "descr",
171 "mode": "NULLABLE",
172 "description": "readable event description",
173 },
174 {
175 "type": "STRING",
176 "name": "current_kind",
177 "mode": "NULLABLE",
178 "description": "kindname",
179 },
180 {
181 "type": "STRING",
182 "name": "current_key",
183 "mode": "NULLABLE",
184 "description": "url encoded datastore key",
185 },
186 {
187 "type": "JSON",
188 "name": "current",
189 "mode": "NULLABLE",
190 "description": "full content of the current entry",
191 },
192 {
193 "type": "JSON",
194 "name": "previous",
195 "mode": "NULLABLE",
196 "description": "previous full content of the entry before it changed",
197 },
198 {
199 "type": "STRING",
200 "name": "diff",
201 "mode": "NULLABLE",
202 "description": "diff data",
203 },
204 {
205 "type": "STRING",
206 "name": "changed_fields",
207 "mode": "REPEATED",
208 "description": "Changed fields from old to new",
209 },
210 )
211 """
212 Schema used for the BigQuery table for its initial construction.
213 Keep to the provided format!
214 """
216 def __init__(self):
217 super().__init__()
219 # checks for the table_path
220 if self.PATH.count(".") != 2:
221 raise ValueError("{self.PATH!r} must have exactly 3 parts that separated by a dot.")
223 self.client = bigquery.Client()
224 self.table = self.select_or_create_table()
226 def select_or_create_table(self):
227 try:
228 return self.client.get_table(self.PATH)
230 except exceptions.NotFound:
231 app, dataset, table = self.PATH.split(".")
233 # create dataset if needed
234 try:
235 self.client.get_dataset(dataset)
236 except exceptions.NotFound:
237 logging.info(f"Dataset {dataset!r} does not exist, creating")
238 self.client.create_dataset(dataset)
240 # create table if needed
241 try:
242 return self.client.get_table(self.PATH)
243 except exceptions.NotFound:
244 logging.info(f"Table {self.PATH!r} does not exist, creating")
245 self.client.create_table(
246 bigquery.Table(
247 self.PATH,
248 schema=self.SCHEMA
249 )
250 )
251 return self.client.get_table(self.PATH)
253 def write_row(self, data):
254 if res := self.client.insert_rows(self.table, [data]):
255 raise ValueError(res)
258class HistoryAdapter(DatabaseAdapter):
259 """
260 Generalized adapter for handling history events.
261 """
263 DEFAULT_EXCLUDES = {
264 "key",
265 "changedate",
266 "creationdate",
267 "importdate",
268 "viurCurrentSeoKeys",
269 }
270 """
271 Bones being ignored within history.
272 """
274 def __init__(self, excludes: t.Iterable[str] = DEFAULT_EXCLUDES):
275 super().__init__()
277 # add excludes to diff excludes
278 self.diff_excludes = set(excludes)
280 def prewrite(self, skel, is_add, change_list=()):
281 if not is_add: # edit
282 old_skel = skel.clone()
283 old_skel.read(skel["key"])
284 self.trigger("edit", old_skel, skel, change_list)
286 def write(self, skel, is_add, change_list=()):
287 if is_add: # add
288 self.trigger("add", None, skel)
290 def delete(self, skel):
291 self.trigger("delete", skel, None)
293 def trigger(
294 self,
295 action: str,
296 old_skel: SkeletonInstance,
297 new_skel: SkeletonInstance,
298 change_list: t.Iterable[str] = (),
299 ) -> str | None:
300 if not (history_module := getattr(conf.main_app, "history", None)):
301 logging.warning(
302 f"{old_skel or new_skel or self!r} uses {self.__class__.__name__}, but no 'history'-module found"
303 )
304 return None
306 # skip excluded actions like login or logout
307 if action in conf.history.excluded_actions:
308 return None
310 # skip when no user is available or provided
311 if not (user := current.user.get()):
312 return None
314 # FIXME: Turn change_list into set, in entire Core...
315 if change_list and not set(change_list).difference(self.diff_excludes):
316 logging.info("change_list is empty, nothing to write")
317 return None
319 # skip excluded kinds and history kind to avoid recursion
320 any_skel = (old_skel or new_skel)
321 if any_skel and (kindname := getattr(any_skel, "kindName", None)):
322 if kindname in conf.history.excluded_kinds:
323 return None
325 if kindname == history_module.kindName:
326 return None
328 return history_module.log(
329 action, old_skel, new_skel,
330 change_list=change_list,
331 user=user,
332 diff_excludes=self.diff_excludes,
333 )
336class History(List):
337 """
338 ViUR history module
339 """
340 adminInfo = {
341 "name": "History",
342 "icon": "clock-history",
343 "filter": {
344 "orderby": "timestamp",
345 "orderdir": "desc",
346 },
347 "disabledActions": ["add", "clone", "delete"],
348 }
350 roles = {
351 "admin": "view",
352 }
354 HISTORY_VERSION = 1
355 """
356 History format version.
357 """
359 BigQueryHistoryCls = BigQueryHistory
360 """
361 The connector class used to store entries to BigQuery.
362 """
364 def __init__(self, *args, **kwargs):
365 super().__init__(*args, **kwargs)
367 if self.BigQueryHistoryCls and "bigquery" in conf.history.databases:
368 assert issubclass(self.BigQueryHistoryCls, BigQueryHistory)
369 self.bigquery = self.BigQueryHistoryCls()
370 else:
371 self.bigquery = None
373 def skel(self, **kwargs):
374 # Make all bones readonly!
375 skel = super().skel(**kwargs).clone()
376 skel.readonly()
377 return skel
379 def canEdit(self, skel):
380 return self.canView(skel) # this is needed to open an entry in admin (all bones are readonly!)
382 def canDelete(self, _skel):
383 return False
385 def canAdd(self):
386 return False
388 # Module-specific functions
389 @staticmethod
390 def _create_diff(new: dict, old: dict, diff_excludes: t.Iterable[str] = set()):
391 """
392 Creates a textual diff format string from the contents of two dicts.
393 """
394 diffs = []
396 # Run over union of both dict keys
397 keys = old.keys() | new.keys()
398 keys = set(keys).difference(diff_excludes)
399 keys = sorted(keys)
401 for key in keys:
402 def expand(name, obj):
403 ret = {}
404 if isinstance(obj, list):
405 for i, val in enumerate(obj):
406 ret.update(expand(name + (str(i),), val))
407 elif isinstance(obj, dict):
408 for key, val in obj.items():
409 ret.update(expand(name + (str(key),), val))
410 else:
411 name = ".".join(name)
412 ret[name] = json.dumps(obj, cls=CustomJsonEncoder, ensure_ascii=False)
414 return ret
416 values = tuple(expand((key,), obj.get(key)) for obj in (old, new))
417 assert len(values) == 2
419 for value_key in sorted(set(values[0].keys() | values[1].keys())):
421 diff = "\n".join(
422 difflib.unified_diff(
423 (values[0].get(value_key) or "").splitlines(),
424 (values[1].get(value_key) or "").splitlines(),
425 value_key, value_key,
426 old.get("changedate") or utils.utcNow().isoformat(),
427 new.get("changedate") or utils.utcNow().isoformat(),
428 n=1
429 )
430 )
432 if diff := diff.strip():
433 diffs.append(diff)
435 return "\n".join(diffs).replace("\n\n", "\n")
437 def build_name(self, skel: SkeletonInstance) -> str | None:
438 """
439 Helper function to figure out a name from the skeleton
440 """
442 if not skel:
443 return None
445 if "name" in skel:
446 name = skel.dump()
448 if isinstance(skel["name"], str):
449 return skel["name"]
451 return name
453 return skel["key"].id_or_name
455 def build_descr(self, action: str, skel: SkeletonInstance, change_list: t.Iterable[str]) -> str | None:
456 """
457 Helper function to build a description about the change to the skeleton
458 """
459 if not skel:
460 return action
462 match action:
463 case "add":
464 return (
465 f"""A new entry with the kind {skel.kindName!r}"""
466 f""" and the key {skel["key"].id_or_name!r} was created."""
467 )
468 case "edit":
469 return (
470 f"""The entry {skel["key"].id_or_name!r} of kind {skel.kindName!r} has been modified."""
471 f""" The following fields where changed: {", ".join(change_list)}."""
472 )
473 case "delete":
474 return f"""The entry {skel["key"].id_or_name!r} of kind {skel.kindName!r} has been deleted."""
476 return (
477 f"""The action {action!r} resulted in a change to the entry {skel["key"].id_or_name!r}"""
478 f""" of kind {skel.kindName!r}."""
479 )
481 def create_history_entry(
482 self,
483 action: str,
484 old_skel: t.Optional[SkeletonInstance] = None,
485 new_skel: t.Optional[SkeletonInstance] = None,
486 change_list: t.Iterable[str] = (),
487 descr: t.Optional[str] = None,
488 user: t.Optional[SkeletonInstance] = None,
489 tags: t.Iterable[str] = (),
490 diff_excludes: t.Set[str] = set(),
491 ):
492 """
493 Internal helper function that constructs a JSON-serializable form of the entry
494 that can either be written to datastore or another database.
495 """
496 skel = new_skel or old_skel
497 new_data = skel.dump(bones=change_list) if skel else {}
499 if change_list and old_skel != new_skel:
500 old_data = old_skel.dump(bones=change_list) if old_skel else {}
501 diff = self._create_diff(new_data, old_data, diff_excludes)
502 else:
503 old_data = {}
504 diff = ""
506 # set event tag, in case of an event-action
507 tags = set(tags)
509 # Event tag
510 if action.startswith("event-"):
511 tags.add("is-event")
513 ret = {
514 "action": action,
515 "current_key": skel and str(skel["key"]),
516 "current_kind": skel and getattr(skel, "kindName", None),
517 "current": new_data,
518 "changed_fields": change_list if change_list else [],
519 "descr": descr or self.build_descr(action, skel, change_list),
520 "diff": diff,
521 "name": self.build_name(skel) if skel else ((user and user["name"] or "") + " " + action),
522 "previous": old_data if old_data else None,
523 "tags": tuple(sorted(tags)),
524 "timestamp": utils.utcNow(),
525 "user_firstname": user and user["firstname"],
526 "user_lastname": user and user["lastname"],
527 "user_name": user and user["name"],
528 "user": user and user["key"],
529 "version": self.HISTORY_VERSION,
530 }
532 return ret
534 def log(
535 self,
536 action: str,
537 old_skel: t.Optional[SkeletonInstance] = None,
538 new_skel: t.Optional[SkeletonInstance] = None,
539 change_list: t.Iterable[str] = (),
540 descr: t.Optional[str] = None,
541 user: t.Optional[SkeletonInstance] = None,
542 tags: t.Iterable[str] = (),
543 diff_excludes: t.Set[str] = set(),
544 ) -> str | None:
545 """
546 Creates and persists a history entry for a skeleton change or a standalone event.
548 Builds the entry via :meth:`create_history_entry`, derives a deterministic key
549 from ``action``, ``current_kind``, and the current timestamp, then writes the
550 entry to all configured backends (``"viur"`` datastore and/or ``"bigquery"``)
551 as deferred tasks.
553 Both ``old_skel`` and ``new_skel`` are optional. For skeleton lifecycle actions
554 one of them is typically present (``"add"`` only has ``new_skel``, ``"delete"``
555 only has ``old_skel``, ``"edit"`` has both), but for pure event logging neither
556 is required.
558 :param action: Short identifier for what happened, e.g. ``"add"``, ``"edit"``,
559 ``"delete"``, or a custom ``"event-*"`` string.
560 :param old_skel: Skeleton state before the change, or ``None`` for pure events
561 and ``"add"`` actions.
562 :param new_skel: Skeleton state after the change, or ``None`` for pure events
563 and ``"delete"`` actions.
564 :param change_list: Names of the bones that were modified (used for ``"edit"`` actions).
565 :param descr: Optional human-readable description. Falls back to :meth:`build_descr`
566 when omitted.
567 :param user: Skeleton instance of the user who triggered the action.
568 :param tags: Additional string tags attached to the entry for filtering.
569 Entries with an ``"event-*"`` action automatically receive the ``"is-event"`` tag.
570 :param diff_excludes: Bone names to exclude from the unified diff computation.
571 :returns: The generated history entry key, or ``None`` if no entry was written.
572 """
574 # create entry
575 entry = self.create_history_entry(
576 action, old_skel, new_skel,
577 change_list=change_list,
578 descr=descr,
579 user=user,
580 tags=tags,
581 diff_excludes=diff_excludes,
582 )
584 # generate key from significant properties
585 key = "-".join(
586 part for part in (
587 entry["action"],
588 entry["current_kind"],
589 entry["timestamp"].isoformat()
590 ) if part
591 )
593 # write into datastore via history module
594 if "viur" in conf.history.databases:
595 self.write_to_viur_deferred(key, entry)
597 # write into BigQuery
598 if self.bigquery and "bigquery" in conf.history.databases:
599 # need to do this as biquery functions modifies entry and seems to be called first
600 if conf.instance.is_dev_server:
601 entry = entry.copy() # need to do this as biquery functions modifiy entry
603 self.write_to_bigquery_deferred(key, entry)
605 return key
607 def write_to_viur(self, key: str, entry: dict):
608 """
609 Write a history entry generated from an HistoryAdapter.
610 """
611 skel = self.addSkel()
613 for name, bone in skel.items():
614 if value := entry.get(name):
615 if isinstance(bone, (RelationalBone, RecordBone)):
616 skel.setBoneValue(name, value)
617 else:
618 skel[name] = value
620 skel.write(key=db.Key(skel.kindName, key))
622 logging.info(f"History entry {key=} written to datastore")
624 @tasks.CallDeferred
625 def write_to_viur_deferred(self, key: str, entry: dict):
626 self.write_to_viur(key, entry)
628 def write_to_bigquery(self, key: str, entry: dict):
629 entry["key"] = key
630 entry["timestamp_date"] = entry["timestamp"].strftime("%Y-%m-%d")
631 entry["timestamp_period"] = entry["timestamp"].strftime("%Y-%m")
632 entry["user"] = str(entry["user"]) if entry["user"] else None
634 self.bigquery.write_row(entry)
635 logging.info(f"History entry {key=} written to biquery")
637 @tasks.CallDeferred
638 def write_to_bigquery_deferred(self, key: str, entry: dict):
639 self.write_to_bigquery(key, entry)
642History.json = True
643History.admin = True