Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/modules/history.py: 0%
206 statements
« prev ^ index » next coverage.py v7.10.6, created at 2025-09-13 11:04 +0000
1import difflib
2import json
3import logging
4import typing as t
5from google.cloud import exceptions, bigquery
6from viur.core import db, conf, utils, current, tasks
7from viur.core.bones import *
8from viur.core.prototypes.list import List
9from viur.core.render.json.default import CustomJsonEncoder
10from viur.core.skeleton import SkeletonInstance, Skeleton, DatabaseAdapter
13class HistorySkel(Skeleton):
14 """
15 Skeleton used for a ViUR history entry to log any relevant changes
16 in other Skeletons.
18 The ViurHistorySkel is also used as the base for a biquery logging table,
19 see below.
20 """
22 kindName = "viur-history"
23 creationdate = changedate = None
25 version = NumericBone(
26 descr="Version",
27 )
29 action = StringBone(
30 descr="Action",
31 )
33 tags = StringBone(
34 descr="Tags",
35 multiple=True,
36 )
38 timestamp = DateBone(
39 descr="Timestamp",
40 defaultValue=lambda *args, **kwargs: utils.utcNow(),
41 localize=True,
42 )
44 user = UserBone(
45 updateLevel=RelationalUpdateLevel.OnValueAssignment,
46 searchable=True,
47 )
49 name = StringBone(
50 descr="Name",
51 searchable=True,
52 )
54 descr = StringBone(
55 descr="Description",
56 searchable=True,
57 )
59 current_kind = StringBone(
60 descr="Entry kind",
61 searchable=True,
62 )
64 current_key = KeyBone(
65 descr="Entity key",
66 )
68 current = JsonBone(
69 descr="Entity content",
70 indexed=False,
71 )
73 changed_fields = StringBone(
74 descr="Changed fields",
75 multiple=True
76 )
78 diff = RawBone(
79 descr="Human-readable diff",
80 indexed=False,
81 )
84class BigQueryHistory:
85 """
86 Connector for BigQuery history entries.
87 """
89 PATH = f"""{conf.instance.project_id}.history.default"""
90 """
91 Path to the big query table for history entries.
92 """
94 SCHEMA = (
95 {
96 "type": "STRING",
97 "name": "key",
98 "mode": "REQUIRED",
99 "description": "unique identifier, hashed from kindname + timestamp",
100 },
101 {
102 "type": "NUMERIC",
103 "name": "version",
104 "mode": "REQUIRED",
105 "description": "log version",
106 },
107 {
108 "type": "STRING",
109 "name": "action",
110 "mode": "NULLABLE",
111 "description": "logged action",
112 },
113 {
114 "type": "STRING",
115 "name": "tags",
116 "mode": "REPEATED",
117 "description": "Additional tags for filtering",
118 },
119 {
120 "type": "DATETIME",
121 "name": "timestamp",
122 "mode": "REQUIRED",
123 "description": "datetime of logevent",
124 },
125 {
126 "type": "STRING",
127 "name": "timestamp_date",
128 "mode": "REQUIRED",
129 "description": "datetime of logevent: date",
130 },
131 {
132 "type": "STRING",
133 "name": "timestamp_period",
134 "mode": "REQUIRED",
135 "description": "datetime of logevent: period",
136 },
137 {
138 "type": "STRING",
139 "name": "user",
140 "mode": "NULLABLE",
141 "description": "user who trigged log event: key",
142 },
143 {
144 "type": "STRING",
145 "name": "user_name",
146 "mode": "NULLABLE",
147 "description": "user who trigged log event: username",
148 },
149 {
150 "type": "STRING",
151 "name": "user_firstname",
152 "mode": "NULLABLE",
153 "description": "user who trigged log event: firstname",
154 },
155 {
156 "type": "STRING",
157 "name": "user_lastname",
158 "mode": "NULLABLE",
159 "description": "user who trigged log event: lastname",
160 },
161 {
162 "type": "STRING",
163 "name": "name",
164 "mode": "NULLABLE",
165 "description": "readable name of the action",
166 },
167 {
168 "type": "STRING",
169 "name": "descr",
170 "mode": "NULLABLE",
171 "description": "readable event description",
172 },
173 {
174 "type": "STRING",
175 "name": "current_kind",
176 "mode": "NULLABLE",
177 "description": "kindname",
178 },
179 {
180 "type": "STRING",
181 "name": "current_key",
182 "mode": "NULLABLE",
183 "description": "url encoded datastore key",
184 },
185 {
186 "type": "JSON",
187 "name": "current",
188 "mode": "NULLABLE",
189 "description": "full content of the current entry",
190 },
191 {
192 "type": "JSON",
193 "name": "previous",
194 "mode": "NULLABLE",
195 "description": "previous full content of the entry before it changed",
196 },
197 {
198 "type": "STRING",
199 "name": "diff",
200 "mode": "NULLABLE",
201 "description": "diff data",
202 },
203 {
204 "type": "STRING",
205 "name": "changed_fields",
206 "mode": "REPEATED",
207 "description": "Changed fields from old to new",
208 },
209 )
210 """
211 Schema used for the BigQuery table for its initial construction.
212 Keep to the provided format!
213 """
215 def __init__(self):
216 super().__init__()
218 # checks for the table_path
219 if self.PATH.count(".") != 2:
220 raise ValueError("{self.PATH!r} must have exactly 3 parts that separated by a dot.")
222 self.client = bigquery.Client()
223 self.table = self.select_or_create_table()
225 def select_or_create_table(self):
226 try:
227 return self.client.get_table(self.PATH)
229 except exceptions.NotFound:
230 app, dataset, table = self.PATH.split(".")
231 logging.error(f"{app}:{dataset}:{table}")
232 # create dataset if needed
233 try:
234 self.client.get_dataset(dataset)
235 except exceptions.NotFound:
236 logging.info(f"Dataset {dataset!r} does not exist, creating")
237 self.client.create_dataset(dataset)
239 # create table if needed
240 try:
241 return self.client.get_table(self.PATH)
242 except exceptions.NotFound:
243 logging.info(f"Table {self.PATH!r} does not exist, creating")
244 self.client.create_table(
245 bigquery.Table(
246 self.PATH,
247 schema=self.SCHEMA
248 )
249 )
250 return self.client.get_table(self.PATH)
252 def write_row(self, data):
253 if res := self.client.insert_rows(self.table, [data]):
254 raise ValueError(res)
257class HistoryAdapter(DatabaseAdapter):
258 """
259 Generalized adapter for handling history events.
260 """
262 DEFAULT_EXCLUDES = {
263 "key",
264 "changedate",
265 "creationdate",
266 "importdate",
267 "viurCurrentSeoKeys",
268 }
269 """
270 Bones being ignored within history.
271 """
273 def __init__(self, excludes: t.Iterable[str] = DEFAULT_EXCLUDES):
274 super().__init__()
276 # add excludes to diff excludes
277 self.diff_excludes = set(excludes)
279 def prewrite(self, skel, is_add, change_list=()):
280 if not is_add: # edit
281 old_skel = skel.clone()
282 old_skel.read(skel["key"])
283 self.trigger("edit", old_skel, skel, change_list)
285 def write(self, skel, is_add, change_list=()):
286 if is_add: # add
287 self.trigger("add", None, skel)
289 def delete(self, skel):
290 self.trigger("delete", skel, None)
292 def trigger(
293 self,
294 action: str,
295 old_skel: SkeletonInstance,
296 new_skel: SkeletonInstance,
297 change_list: t.Iterable[str] = (),
298 ) -> str | None:
299 if not (history_module := getattr(conf.main_app, "history", None)):
300 logging.warning(
301 f"{old_skel or new_skel or self!r} uses {self.__class__.__name__}, but no 'history'-module found"
302 )
303 return None
305 # skip excluded actions like login or logout
306 if action in conf.history.excluded_actions:
307 return None
309 # skip when no user is available or provided
310 if not (user := current.user.get()):
311 return None
313 # FIXME: Turn change_list into set, in entire Core...
314 if change_list and not set(change_list).difference(self.diff_excludes):
315 logging.info("change_list is empty, nothing to write")
316 return None
318 # skip excluded kinds and history kind to avoid recursion
319 any_skel = (old_skel or new_skel)
320 if any_skel and (kindname := getattr(any_skel, "kindName", None)):
321 if kindname in conf.history.excluded_kinds:
322 return None
324 if kindname == "viur-history":
325 return None
327 return history_module.write_diff(
328 action, old_skel, new_skel,
329 change_list=change_list,
330 user=user,
331 diff_excludes=self.diff_excludes,
332 )
335class History(List):
336 """
337 ViUR history module
338 """
339 kindName = "viur-history"
341 adminInfo = {
342 "name": "History",
343 "icon": "clock-history",
344 "filter": {
345 "orderby": "timestamp",
346 "orderdir": "desc",
347 },
348 "disabledActions": ["add", "clone", "delete"],
349 }
351 roles = {
352 "admin": "view",
353 }
355 HISTORY_VERSION = 1
356 """
357 History format version.
358 """
360 BigQueryHistoryCls = BigQueryHistory
361 """
362 The connector class used to store entries to BigQuery.
363 """
365 def __init__(self, *args, **kwargs):
366 super().__init__(*args, **kwargs)
368 if self.BigQueryHistoryCls and "bigquery" in conf.history.databases:
369 assert issubclass(self.BigQueryHistoryCls, BigQueryHistory)
370 self.bigquery = self.BigQueryHistoryCls()
371 else:
372 self.bigquery = None
374 def skel(self, **kwargs):
375 # Make all bones readonly!
376 skel = super().skel(**kwargs).clone()
377 skel.readonly()
378 return skel
380 def canEdit(self, skel):
381 return self.canView(skel) # this is needed to open an entry in admin (all bones are readonly!)
383 def canDelete(self, _skel):
384 return False
386 def canAdd(self):
387 return False
389 # Module-specific functions
390 @staticmethod
391 def _create_diff(new: dict, old: dict, diff_excludes: t.Iterable[str] = set()):
392 """
393 Creates a textual diff format string from the contents of two dicts.
394 """
395 diffs = []
397 # Run over union of both dict keys
398 keys = old.keys() | new.keys()
399 keys = set(keys).difference(diff_excludes)
400 keys = sorted(keys)
402 for key in keys:
403 def expand(name, obj):
404 ret = {}
405 if isinstance(obj, list):
406 for i, val in enumerate(obj):
407 ret.update(expand(name + (str(i),), val))
408 elif isinstance(obj, dict):
409 for key, val in obj.items():
410 ret.update(expand(name + (str(key),), val))
411 else:
412 name = ".".join(name)
413 ret[name] = json.dumps(obj, cls=CustomJsonEncoder)
415 return ret
417 values = tuple(expand((key,), obj.get(key)) for obj in (old, new))
418 assert len(values) == 2
420 for value_key in sorted(set(values[0].keys() | values[1].keys())):
422 diff = "\n".join(
423 difflib.unified_diff(
424 (values[0].get(value_key) or "").splitlines(),
425 (values[1].get(value_key) or "").splitlines(),
426 value_key, value_key,
427 old.get("changedate") or utils.utcNow().isoformat(),
428 new.get("changedate") or utils.utcNow().isoformat(),
429 n=1
430 )
431 )
433 if diff := diff.strip():
434 diffs.append(diff)
436 return "\n".join(diffs).replace("\n\n", "\n")
438 def build_name(self, skel: SkeletonInstance) -> str | None:
439 """
440 Helper function to figure out a name from the skeleton
441 """
443 if not skel:
444 return None
446 if "name" in skel:
447 name = skel.dump()
449 if isinstance(skel["name"], str):
450 return skel["name"]
452 return name
454 return skel["key"].id_or_name
456 def build_descr(self, action: str, skel: SkeletonInstance, change_list: t.Iterable[str]) -> str | None:
457 """
458 Helper function to build a description about the change to the skeleton
459 """
460 if not skel:
461 return action
463 match action:
464 case "add":
465 return (
466 f"""A new entry with the kind {skel.kindName!r}"""
467 f""" and the key {skel["key"].id_or_name!r} was created."""
468 )
469 case "edit":
470 return (
471 f"""The entry {skel["key"].id_or_name!r} of kind {skel.kindName!r} has been modified."""
472 f""" The following fields where changed: {", ".join(change_list)}."""
473 )
474 case "delete":
475 return f"""The entry {skel["key"].id_or_name!r} of kind {skel.kindName!r} has been deleted."""
477 return (
478 f"""The action {action!r} resulted in a change to the entry {skel["key"].id_or_name!r}"""
479 f""" of kind {skel.kindName!r}."""
480 )
482 def create_history_entry(
483 self,
484 action: str,
485 old_skel: SkeletonInstance,
486 new_skel: SkeletonInstance,
487 change_list: t.Iterable[str] = (),
488 descr: t.Optional[str] = None,
489 user: t.Optional[SkeletonInstance] = None,
490 tags: t.Iterable[str] = (),
491 diff_excludes: t.Set[str] = set(),
492 ):
493 """
494 Internal helper function that constructs a JSON-serializable form of the entry
495 that can either be written to datastore or another database.
496 """
497 skel = new_skel or old_skel
498 new_data = skel.dump()
500 if change_list and old_skel != new_skel:
501 old_data = old_skel.dump()
502 diff = self._create_diff(new_data, old_data, diff_excludes)
503 else:
504 old_data = {}
505 diff = ""
507 # set event tag, in case of an event-action
508 tags = set(tags)
510 # Event tag
511 if action.startswith("event-"):
512 tags.add("is-event")
514 ret = {
515 "action": action,
516 "current_key": skel and str(skel["key"]),
517 "current_kind": skel and getattr(skel, "kindName", None),
518 "current": new_data,
519 "changed_fields": change_list if change_list else [],
520 "descr": descr or self.build_descr(action, skel, change_list),
521 "diff": diff,
522 "name": self.build_name(skel) if skel else ((user and user["name"] or "") + " " + action),
523 "previous": old_data if old_data else None,
524 "tags": tuple(sorted(tags)),
525 "timestamp": utils.utcNow(),
526 "user_firstname": user and user["firstname"],
527 "user_lastname": user and user["lastname"],
528 "user_name": user and user["name"],
529 "user": user and user["key"],
530 "version": self.HISTORY_VERSION,
531 }
533 return ret
535 def write_diff(
536 self,
537 action: str,
538 old_skel: SkeletonInstance = None,
539 new_skel: SkeletonInstance = None,
540 change_list: t.Iterable[str] = (),
541 descr: t.Optional[str] = None,
542 user: t.Optional[SkeletonInstance] = None,
543 tags: t.Iterable[str] = (),
544 diff_excludes: t.Set[str] = set(),
545 ) -> str | None:
547 # create entry
548 entry = self.create_history_entry(
549 action, old_skel, new_skel,
550 change_list=change_list,
551 descr=descr,
552 user=user,
553 tags=tags,
554 diff_excludes=diff_excludes,
555 )
557 # generate key from significant properties
558 key = "-".join(
559 part for part in (
560 entry["action"],
561 entry["current_kind"],
562 entry["timestamp"].isoformat()
563 ) if part
564 )
566 # write into datastore via history module
567 if "viur" in conf.history.databases:
568 self.write_deferred(key, entry)
570 # write into BigQuery
571 if self.bigquery and "bigquery" in conf.history.databases:
572 # need to do this as biquery functions modifies entry and seems to be called first
573 if conf.instance.is_dev_server:
574 entry = entry.copy() # need to do this as biquery functions modifiy entry
576 self.write_to_bigquery_deferred(key, entry)
578 return key
580 def write(self, key: str, entry: dict):
581 """
582 Write a history entry generated from an HistoryAdapter.
583 """
584 skel = self.addSkel()
586 for name, bone in skel.items():
587 if value := entry.get(name):
588 if isinstance(bone, (RelationalBone, RecordBone)):
589 skel.setBoneValue(name, value)
590 else:
591 skel[name] = value
593 skel.write(key=db.Key(skel.kindName, key))
595 logging.info(f"History entry {key=} written to datastore")
597 @tasks.CallDeferred
598 def write_deferred(self, key: str, entry: dict):
599 self.write(key, entry)
601 def write_to_bigquery(self, key: str, entry: dict):
602 entry["key"] = key
603 entry["timestamp_date"] = entry["timestamp"].strftime("%Y-%m-%d")
604 entry["timestamp_period"] = entry["timestamp"].strftime("%Y-%m")
605 entry["user"] = str(entry["user"]) if entry["user"] else None
607 self.bigquery.write_row(entry)
608 logging.info(f"History entry {key=} written to biquery")
610 @tasks.CallDeferred
611 def write_to_bigquery_deferred(self, key: str, entry: dict):
612 self.write_to_bigquery(key, entry)
615History.json = True
616History.admin = True