Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/bones/file.py: 20%
156 statements
« prev ^ index » next coverage.py v7.16.1, created at 2026-09-25 15:02 +0000
1"""
2The FileBone is a subclass of the TreeLeafBone class, which is a relational bone that can reference
3another entity's fields. FileBone provides additional file-specific properties and methods, such as
4managing file derivatives, handling file size and mime type restrictions, and refreshing file
5metadata.
6"""
7import hashlib
8import warnings
9import time
10import typing as t
11from viur.core import conf, db, current, utils
12from viur.core.bones.treeleaf import TreeLeafBone
13from viur.core.tasks import CallDeferred
14import logging
17@CallDeferred
18def ensureDerived(
19 key: db.Key,
20 src_key: str,
21 derive_map: dict[str, t.Any],
22 refresh_key: db.Key = None,
23 **kwargs
24):
25 r"""
26 The function is a deferred function that ensures all pending thumbnails or other derived files
27 are built. It takes the following parameters:
29 :param db.key key: The database key of the file-object that needs to have its derivation map
30 updated.
31 :param str src_key: A prefix for a stable key to prevent rebuilding derived files repeatedly.
32 :param dict[str,Any] derive_map: A list of DeriveDicts that need to be built or updated.
33 :param db.Key refresh_key: If set, the function fetches and refreshes the skeleton after
34 building new derived files.
36 The function works by fetching the skeleton of the file-object, checking if it has any derived
37 files, and updating the derivation map accordingly. It iterates through the derive_map items and
38 calls the appropriate deriver function. If the deriver function returns a result, the function
39 creates a new or updated resultDict and merges it into the file-object's metadata. Finally,
40 the updated results are written back to the database and the update_relations function is called
41 to ensure proper relations are maintained.
42 """
43 # TODO: Remove in VIUR4
44 for _dep, _new in {
45 "srcKey": "src_key",
46 "deriveMap": "derive_map",
47 "refreshKey": "refresh_key",
48 }.items():
49 if _dep in kwargs:
50 warnings.warn(
51 f"{_dep!r} parameter is deprecated, please use {_new!r} instead",
52 DeprecationWarning, stacklevel=2
53 )
55 locals()[_new] = kwargs.pop(_dep)
57 from viur.core.skeleton.utils import skeletonByKind
58 from viur.core.skeleton.tasks import update_relations
60 skel = skeletonByKind(key.kind)()
61 if not skel.read(key):
62 logging.error(f"{src_key}: File not found, is it gone?")
63 return
65 if not skel["derived"]:
66 logging.info(f"{src_key}: No derives for this file")
67 skel["derived"] = {}
69 skel["derived"] = {"deriveStatus": {}, "files": {}} | skel["derived"]
71 res_status, res_files = {}, {}
72 for call_key, params in derive_map.items():
73 full_src_key = f"{src_key}_{call_key}"
74 params_hash = hashlib.sha256(str(params).encode("UTF-8")).hexdigest() # Hash over given params (dict?)
75 if skel["derived"]["deriveStatus"].get(full_src_key) != params_hash:
76 if not (caller := conf.file_derivations.get(call_key)):
77 logging.warning(f"File-Deriver {call_key} not found - skipping!")
78 continue
80 if call_res := caller(skel, skel["derived"]["files"], params):
81 assert isinstance(call_res, list), "Old (non-list) return value from deriveFunc"
82 res_status[full_src_key] = params_hash
83 for file_name, size, mimetype, custom_data in call_res:
84 res_files[file_name] = {
85 "size": size,
86 "mimetype": mimetype,
87 "customData": custom_data # TODO: Rename in VIUR4
88 }
90 if res_status: # Write updated results back and queue updateRelationsTask
91 def _merge_derives(patch_skel):
92 patch_skel["derived"] = {"deriveStatus": {}, "files": {}} | (patch_skel["derived"] or {})
93 patch_skel["derived"]["deriveStatus"] = patch_skel["derived"]["deriveStatus"] | res_status
94 patch_skel["derived"]["files"] = patch_skel["derived"]["files"] | res_files
96 skel.patch(values=_merge_derives, update_relations=False)
98 # Queue that update_relations call at least 30 seconds into the future, so that other ensureDerived calls from
99 # the same FileBone have the chance to finish, otherwise that update_relations Task will call postSavedHandler
100 # on that FileBone again - re-queueing any ensureDerivedCalls that have not finished yet.
102 if refresh_key:
103 skel = skeletonByKind(refresh_key.kind)()
104 skel.patch(lambda _skel: _skel.refresh(), key=refresh_key, update_relations=False)
106 update_relations(key, min_change_time=int(time.time() + 1), changed_bones=["derived"], _countdown=30)
109class FileBone(TreeLeafBone):
110 r"""
111 A FileBone is a custom bone class that inherits from the TreeLeafBone class, and is used to store and manage
112 file references in a ViUR application.
114 :param format: Hint for the UI how to display a file entry (defaults to it's filename)
115 :param maxFileSize:
116 The maximum filesize accepted by this bone in bytes. None means no limit.
117 This will always be checked against the original file uploaded - not any of it's derivatives.
119 :param derive: A set of functions used to derive other files from the referenced ones. Used fe.
120 to create thumbnails / images for srcmaps from hires uploads. If set, must be a dictionary from string
121 (a key from conf.file_derivations) to the parameters passed to that function. The parameters can be
122 any type (including None) that can be json-serialized.
124 .. code-block:: python
126 # Example
127 derive = { "thumbnail": [{"width": 111}, {"width": 555, "height": 666}]}
129 :param validMimeTypes:
130 A list of Mimetypes that can be selected in this bone (or None for any) Wildcards ("image\/*") are supported.
132 .. code-block:: python
134 # Example
135 validMimeTypes=["application/pdf", "image/*"]
137 """
139 kind = "file"
140 """The kind of this bone is 'file'"""
142 type = "relational.tree.leaf.file"
143 """The type of this bone is 'relational.tree.leaf.file'."""
145 DEFAULT_REFKEYS = (
146 "derived",
147 "dlkey",
148 "height",
149 "mimetype",
150 "name",
151 "public",
152 "serving_url",
153 "size",
154 "width",
155 )
156 """
157 Default RefKeys for FileBone.
158 Use this as extendable reference: extending this tuple, either globally or in a
159 subclass, changes the refKeys of every FileBone that does not pass its own.
160 """
162 def __init__(
163 self,
164 *,
165 derive: None | dict[str, t.Any] = None,
166 maxFileSize: None | int = None,
167 validMimeTypes: None | list[str] = None,
168 refKeys: t.Optional[t.Iterable[str]] = None,
169 public: bool = False,
170 **kwargs
171 ):
172 r"""
173 Initializes a new Filebone. All properties inherited by RelationalBone are supported.
175 :param format: Hint for the UI how to display a file entry (defaults to it's filename)
176 :param maxFileSize: The maximum filesize accepted by this bone in bytes. None means no limit.
177 This will always be checked against the original file uploaded - not any of it's derivatives.
178 :param derive: A set of functions used to derive other files from the referenced ones.
179 Used to create thumbnails and images for srcmaps from hires uploads.
180 If set, must be a dictionary from string (a key from) conf.file_derivations) to the parameters passed to
181 that function. The parameters can be any type (including None) that can be json-serialized.
183 .. code-block:: python
185 # Example
186 derive = {"thumbnail": [{"width": 111}, {"width": 555, "height": 666}]}
188 :param validMimeTypes:
189 A list of Mimetypes that can be selected in this bone (or None for any).
190 Wildcards `('image\*')` are supported.
192 .. code-block:: python
194 #Example
195 validMimeTypes=["application/pdf", "image/*"]
197 :param refKeys:
198 The keys of the referenced file to store in the relation.
199 Defaults to :attr:`DEFAULT_REFKEYS`; unlike in RelationalBone, ``None``
200 selects that default instead of the bare ``key``/``shortkey`` pair, which
201 a FileBone cannot operate with anyway.
202 """
203 # Resolved here rather than in the signature: a default argument is evaluated
204 # once at import time and would ignore any later change to DEFAULT_REFKEYS.
205 if refKeys is None:
206 refKeys = self.DEFAULT_REFKEYS
207 super().__init__(refKeys=refKeys, **kwargs)
209 self.derive = derive
210 self.public = public
211 self.validMimeTypes = validMimeTypes
212 self.maxFileSize = maxFileSize
214 # isInvalid() reads these bones from the RefSkel; a missing one silently
215 # becomes None there, so require them up-front instead.
216 for _required in ("dlkey", "name", "public"):
217 if _required not in self.refKeys:
218 raise ValueError(f"FileBone not operable without refKey {_required!r}")
220 if self.validMimeTypes and "mimetype" not in self.refKeys:
221 raise ValueError("FileBone with validMimeTypes not operable without refKey 'mimetype'")
223 if self.maxFileSize and "size" not in self.refKeys:
224 raise ValueError("FileBone with maxFileSize not operable without refKey 'size'")
226 def isInvalid(self, value):
227 """
228 Checks if the provided value is invalid for this bone based on its MIME type and file size.
230 :param dict value: The value to check for validity.
231 :returns: None if the value is valid, or an error message if it is invalid.
232 """
233 if self.validMimeTypes:
234 mimeType = value["dest"]["mimetype"]
235 for checkMT in self.validMimeTypes:
236 checkMT = checkMT.lower()
237 if checkMT == mimeType or checkMT.endswith("*") and mimeType.startswith(checkMT[:-1]):
238 break
239 else:
240 return "Invalid filetype selected"
241 if self.maxFileSize:
242 if value["dest"]["size"] > self.maxFileSize:
243 return "File too large."
245 if value["dest"]["public"] != self.public:
246 return f"Only files marked public={self.public!r} are allowed."
248 return None
250 def postSavedHandler(self, skel, boneName, key):
251 """
252 Handles post-save processing for the FileBone, including ensuring derived files are built.
254 :param SkeletonInstance skel: The skeleton instance this bone belongs to.
255 :param str boneName: The name of the bone.
256 :param db.Key key: The datastore key of the skeleton.
258 This method first calls the postSavedHandler of its superclass. Then, it checks if the
259 derive attribute is set and if there are any values in the skeleton for the given bone. If
260 so, it handles the creation of derived files based on the provided configuration.
262 If the values are stored as a dictionary without a "dest" key, it assumes a multi-language
263 setup and iterates over each language to handle the derived files. Otherwise, it handles
264 the derived files directly.
265 """
266 super().postSavedHandler(skel, boneName, key)
267 if (
268 current.request.get() and current.request.get().is_deferred
269 and "derived" in (current.request_data.get().get("__update_relations_bones") or ())
270 ):
271 return
273 from viur.core.skeleton import RelSkel, Skeleton
275 if issubclass(skel.skeletonCls, Skeleton):
276 prefix = f"{skel.kindName}_{boneName}"
277 elif issubclass(skel.skeletonCls, RelSkel): # RelSkel is just a container and has no kindname
278 prefix = f"{skel.skeletonCls.__name__}_{boneName}"
279 else:
280 raise NotImplementedError(f"Cannot handle {skel.skeletonCls=}")
282 def handleDerives(values):
283 if isinstance(values, dict):
284 values = [values]
285 for val in (values or ()): # Ensure derives getting build for each file referenced in this relation
286 ensureDerived(val["dest"]["key"], prefix, self.derive, key)
288 values = skel[boneName]
289 if self.derive and values:
290 if isinstance(values, dict) and "dest" not in values: # multi lang
291 for lang in values:
292 handleDerives(values[lang])
293 else:
294 handleDerives(values)
296 def getReferencedBlobs(self, skel: 'viur.core.skeleton.SkeletonInstance', name: str) -> set[str]:
297 r"""
298 Retrieves the referenced blobs in the FileBone.
300 :param SkeletonInstance skel: The skeleton instance this bone belongs to.
301 :param str name: The name of the bone.
302 :return: A set of download keys for the referenced blobs.
303 :rtype: Set[str]
305 This method iterates over the bone values for the given skeleton and bone name. It skips
306 values that are None. For each non-None value, it adds the download key of the referenced
307 blob to a set. Finally, it returns the set of unique download keys for the referenced blobs.
308 """
309 result = set()
310 for idx, lang, value in self.iter_bone_value(skel, name):
311 if value is None:
312 continue
313 result.add(value["dest"]["dlkey"])
314 return result
316 def refresh(self, skel, boneName):
317 r"""
318 Refreshes the FileBone by recreating file entries if needed and importing blobs from ViUR 2.
320 :param SkeletonInstance skel: The skeleton instance this bone belongs to.
321 :param str boneName: The name of the bone.
323 This method defines an inner function, recreateFileEntryIfNeeded(val), which is responsible
324 for recreating the weak file entry referenced by the relation in val if it doesn't exist
325 (e.g., if it was deleted by ViUR 2). It initializes a new skeleton for the "file" kind and
326 checks if the file object already exists. If not, it recreates the file entry with the
327 appropriate properties and saves it to the database.
329 The main part of the refresh method calls the superclass's refresh method and checks if the
330 configuration contains a ViUR 2 import blob source. If it does, it iterates through the file
331 references in the bone value, imports the blobs from ViUR 2, and recreates the file entries if
332 needed using the inner function.
333 """
334 super().refresh(skel, boneName)
336 for _, _, value in self.iter_bone_value(skel, boneName):
337 # Patch any empty serving_url when public file
338 if (
339 value
340 and (value := value["dest"])
341 and value["public"]
342 and value["mimetype"]
343 and value["mimetype"].startswith("image/")
344 and not value["serving_url"]
345 ):
346 logging.info(f"Patching public image with empty serving_url {value['key']!r} ({value['name']!r})")
347 try:
348 file_skel = value.read()
349 except ValueError:
350 continue
352 file_skel.patch(lambda skel: skel.refresh(), update_relations=False)
353 value["serving_url"] = file_skel["serving_url"]
355 # FIXME: REMOVE THIS WITH VIUR4
356 if conf.viur2import_blobsource:
357 from viur.core.modules.file import importBlobFromViur2
358 from viur.core.skeleton import skeletonByKind
360 def recreateFileEntryIfNeeded(val):
361 # Recreate the (weak) filenetry referenced by the relation *val*. (ViUR2 might have deleted them)
362 skel = skeletonByKind("file")()
363 if skel.read(val["key"]): # This file-object exist, no need to recreate it
364 return
365 skel["key"] = val["key"]
366 skel["name"] = val["name"]
367 skel["mimetype"] = val["mimetype"]
368 skel["dlkey"] = val["dlkey"]
369 skel["size"] = val["size"]
370 skel["width"] = val["width"]
371 skel["height"] = val["height"]
372 skel["weak"] = True
373 skel["pending"] = False
374 skel.write()
376 # Just ensure the file get's imported as it may not have an file entry
377 val = skel[boneName]
378 if isinstance(val, list):
379 for x in val:
380 importBlobFromViur2(x["dest"]["dlkey"], x["dest"]["name"])
381 recreateFileEntryIfNeeded(x["dest"])
382 elif isinstance(val, dict):
383 if not "dest" in val:
384 return
385 importBlobFromViur2(val["dest"]["dlkey"], val["dest"]["name"])
386 recreateFileEntryIfNeeded(val["dest"])
388 def structure(self) -> dict:
389 return super().structure() | {
390 "valid_mime_types": self.validMimeTypes,
391 "max_file_size": self.maxFileSize,
392 "public": self.public,
393 }
395 def _atomic_dump(self, value) -> dict | None:
396 value = super()._atomic_dump(value)
397 if value is not None:
398 # VIUR4: Rename "downloadUrl" into "download_url"
399 value["dest"]["downloadUrl"] = conf.main_app.file.create_download_url(
400 value["dest"]["dlkey"],
401 value["dest"]["name"],
402 derived=False,
403 expires=conf.render_json_download_url_expiration
404 )
406 return value