Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/bones/file.py: 11%
150 statements
« prev ^ index » next coverage.py v7.10.7, created at 2025-09-29 09:00 +0000
1"""
2The FileBone is a subclass of the TreeLeafBone class, which is a relational bone that can reference
3another entity's fields. FileBone provides additional file-specific properties and methods, such as
4managing file derivatives, handling file size and mime type restrictions, and refreshing file
5metadata.
6"""
7import hashlib
8import warnings
9import time
10import typing as t
11from viur.core import conf, db, current, utils
12from viur.core.bones.treeleaf import TreeLeafBone
13from viur.core.tasks import CallDeferred
14import logging
17@CallDeferred
18def ensureDerived(
19 key: db.Key,
20 src_key: str,
21 derive_map: dict[str, t.Any],
22 refresh_key: db.Key = None,
23 **kwargs
24):
25 r"""
26 The function is a deferred function that ensures all pending thumbnails or other derived files
27 are built. It takes the following parameters:
29 :param db.key key: The database key of the file-object that needs to have its derivation map
30 updated.
31 :param str src_key: A prefix for a stable key to prevent rebuilding derived files repeatedly.
32 :param dict[str,Any] derive_map: A list of DeriveDicts that need to be built or updated.
33 :param db.Key refresh_key: If set, the function fetches and refreshes the skeleton after
34 building new derived files.
36 The function works by fetching the skeleton of the file-object, checking if it has any derived
37 files, and updating the derivation map accordingly. It iterates through the derive_map items and
38 calls the appropriate deriver function. If the deriver function returns a result, the function
39 creates a new or updated resultDict and merges it into the file-object's metadata. Finally,
40 the updated results are written back to the database and the update_relations function is called
41 to ensure proper relations are maintained.
42 """
43 # TODO: Remove in VIUR4
44 for _dep, _new in {
45 "srcKey": "src_key",
46 "deriveMap": "derive_map",
47 "refreshKey": "refresh_key",
48 }.items():
49 if _dep in kwargs:
50 warnings.warn(
51 f"{_dep!r} parameter is deprecated, please use {_new!r} instead",
52 DeprecationWarning, stacklevel=2
53 )
55 locals()[_new] = kwargs.pop(_dep)
57 from viur.core.skeleton.utils import skeletonByKind
58 from viur.core.skeleton.tasks import update_relations
60 skel = skeletonByKind(key.kind)()
61 if not skel.read(key):
62 logging.error(f"{src_key}: File not found, is it gone?")
63 return
65 if not skel["derived"]:
66 logging.info(f"{src_key}: No derives for this file")
67 skel["derived"] = {}
69 skel["derived"] = {"deriveStatus": {}, "files": {}} | skel["derived"]
71 res_status, res_files = {}, {}
72 for call_key, params in derive_map.items():
73 full_src_key = f"{src_key}_{call_key}"
74 params_hash = hashlib.sha256(str(params).encode("UTF-8")).hexdigest() # Hash over given params (dict?)
75 if skel["derived"]["deriveStatus"].get(full_src_key) != params_hash:
76 if not (caller := conf.file_derivations.get(call_key)):
77 logging.warning(f"File-Deriver {call_key} not found - skipping!")
78 continue
80 if call_res := caller(skel, skel["derived"]["files"], params):
81 assert isinstance(call_res, list), "Old (non-list) return value from deriveFunc"
82 res_status[full_src_key] = params_hash
83 for file_name, size, mimetype, custom_data in call_res:
84 res_files[file_name] = {
85 "size": size,
86 "mimetype": mimetype,
87 "customData": custom_data # TODO: Rename in VIUR4
88 }
90 if res_status: # Write updated results back and queue updateRelationsTask
91 def _merge_derives(patch_skel):
92 patch_skel["derived"] = {"deriveStatus": {}, "files": {}} | (patch_skel["derived"] or {})
93 patch_skel["derived"]["deriveStatus"] = patch_skel["derived"]["deriveStatus"] | res_status
94 patch_skel["derived"]["files"] = patch_skel["derived"]["files"] | res_files
96 skel.patch(values=_merge_derives, update_relations=False)
98 # Queue that update_relations call at least 30 seconds into the future, so that other ensureDerived calls from
99 # the same FileBone have the chance to finish, otherwise that update_relations Task will call postSavedHandler
100 # on that FileBone again - re-queueing any ensureDerivedCalls that have not finished yet.
102 if refresh_key:
103 skel = skeletonByKind(refresh_key.kind)()
104 skel.patch(lambda _skel: _skel.refresh(), key=refresh_key, update_relations=False)
106 update_relations(key, min_change_time=int(time.time() + 1), changed_bones=["derived"], _countdown=30)
109class FileBone(TreeLeafBone):
110 r"""
111 A FileBone is a custom bone class that inherits from the TreeLeafBone class, and is used to store and manage
112 file references in a ViUR application.
114 :param format: Hint for the UI how to display a file entry (defaults to it's filename)
115 :param maxFileSize:
116 The maximum filesize accepted by this bone in bytes. None means no limit.
117 This will always be checked against the original file uploaded - not any of it's derivatives.
119 :param derive: A set of functions used to derive other files from the referenced ones. Used fe.
120 to create thumbnails / images for srcmaps from hires uploads. If set, must be a dictionary from string
121 (a key from conf.file_derivations) to the parameters passed to that function. The parameters can be
122 any type (including None) that can be json-serialized.
124 .. code-block:: python
126 # Example
127 derive = { "thumbnail": [{"width": 111}, {"width": 555, "height": 666}]}
129 :param validMimeTypes:
130 A list of Mimetypes that can be selected in this bone (or None for any) Wildcards ("image\/*") are supported.
132 .. code-block:: python
134 # Example
135 validMimeTypes=["application/pdf", "image/*"]
137 """
139 kind = "file"
140 """The kind of this bone is 'file'"""
142 type = "relational.tree.leaf.file"
143 """The type of this bone is 'relational.tree.leaf.file'."""
145 DEFAULT_REFKEYS = (
146 "derived",
147 "dlkey",
148 "height",
149 "mimetype",
150 "name",
151 "public",
152 "serving_url",
153 "size",
154 "width",
155 )
156 """
157 Default RefKeys for FileBone.
158 Use this as extendable reference.
159 """
161 def __init__(
162 self,
163 *,
164 derive: None | dict[str, t.Any] = None,
165 maxFileSize: None | int = None,
166 validMimeTypes: None | list[str] = None,
167 refKeys: t.Optional[t.Iterable[str]] = DEFAULT_REFKEYS,
168 public: bool = False,
169 **kwargs
170 ):
171 r"""
172 Initializes a new Filebone. All properties inherited by RelationalBone are supported.
174 :param format: Hint for the UI how to display a file entry (defaults to it's filename)
175 :param maxFileSize: The maximum filesize accepted by this bone in bytes. None means no limit.
176 This will always be checked against the original file uploaded - not any of it's derivatives.
177 :param derive: A set of functions used to derive other files from the referenced ones.
178 Used to create thumbnails and images for srcmaps from hires uploads.
179 If set, must be a dictionary from string (a key from) conf.file_derivations) to the parameters passed to
180 that function. The parameters can be any type (including None) that can be json-serialized.
182 .. code-block:: python
184 # Example
185 derive = {"thumbnail": [{"width": 111}, {"width": 555, "height": 666}]}
187 :param validMimeTypes:
188 A list of Mimetypes that can be selected in this bone (or None for any).
189 Wildcards `('image\*')` are supported.
191 .. code-block:: python
193 #Example
194 validMimeTypes=["application/pdf", "image/*"]
196 """
197 super().__init__(refKeys=refKeys, **kwargs)
199 for _required in ("dlkey", "name"):
200 if _required not in self.refKeys:
201 raise ValueError(f"FileBone not operable without refKey {_required!r}")
203 self.derive = derive
204 self.public = public
205 self.validMimeTypes = validMimeTypes
206 self.maxFileSize = maxFileSize
208 def isInvalid(self, value):
209 """
210 Checks if the provided value is invalid for this bone based on its MIME type and file size.
212 :param dict value: The value to check for validity.
213 :returns: None if the value is valid, or an error message if it is invalid.
214 """
215 if self.validMimeTypes:
216 mimeType = value["dest"]["mimetype"]
217 for checkMT in self.validMimeTypes:
218 checkMT = checkMT.lower()
219 if checkMT == mimeType or checkMT.endswith("*") and mimeType.startswith(checkMT[:-1]):
220 break
221 else:
222 return "Invalid filetype selected"
223 if self.maxFileSize:
224 if value["dest"]["size"] > self.maxFileSize:
225 return "File too large."
227 if value["dest"]["public"] != self.public:
228 return f"Only files marked public={self.public!r} are allowed."
230 return None
232 def postSavedHandler(self, skel, boneName, key):
233 """
234 Handles post-save processing for the FileBone, including ensuring derived files are built.
236 :param SkeletonInstance skel: The skeleton instance this bone belongs to.
237 :param str boneName: The name of the bone.
238 :param db.Key key: The datastore key of the skeleton.
240 This method first calls the postSavedHandler of its superclass. Then, it checks if the
241 derive attribute is set and if there are any values in the skeleton for the given bone. If
242 so, it handles the creation of derived files based on the provided configuration.
244 If the values are stored as a dictionary without a "dest" key, it assumes a multi-language
245 setup and iterates over each language to handle the derived files. Otherwise, it handles
246 the derived files directly.
247 """
248 super().postSavedHandler(skel, boneName, key)
249 if (
250 current.request.get().is_deferred
251 and "derived" in (current.request_data.get().get("__update_relations_bones") or ())
252 ):
253 return
255 from viur.core.skeleton import RelSkel, Skeleton
257 if issubclass(skel.skeletonCls, Skeleton):
258 prefix = f"{skel.kindName}_{boneName}"
259 elif issubclass(skel.skeletonCls, RelSkel): # RelSkel is just a container and has no kindname
260 prefix = f"{skel.skeletonCls.__name__}_{boneName}"
261 else:
262 raise NotImplementedError(f"Cannot handle {skel.skeletonCls=}")
264 def handleDerives(values):
265 if isinstance(values, dict):
266 values = [values]
267 for val in (values or ()): # Ensure derives getting build for each file referenced in this relation
268 ensureDerived(val["dest"]["key"], prefix, self.derive, key)
270 values = skel[boneName]
271 if self.derive and values:
272 if isinstance(values, dict) and "dest" not in values: # multi lang
273 for lang in values:
274 handleDerives(values[lang])
275 else:
276 handleDerives(values)
278 def getReferencedBlobs(self, skel: 'viur.core.skeleton.SkeletonInstance', name: str) -> set[str]:
279 r"""
280 Retrieves the referenced blobs in the FileBone.
282 :param SkeletonInstance skel: The skeleton instance this bone belongs to.
283 :param str name: The name of the bone.
284 :return: A set of download keys for the referenced blobs.
285 :rtype: Set[str]
287 This method iterates over the bone values for the given skeleton and bone name. It skips
288 values that are None. For each non-None value, it adds the download key of the referenced
289 blob to a set. Finally, it returns the set of unique download keys for the referenced blobs.
290 """
291 result = set()
292 for idx, lang, value in self.iter_bone_value(skel, name):
293 if value is None:
294 continue
295 result.add(value["dest"]["dlkey"])
296 return result
298 def refresh(self, skel, boneName):
299 r"""
300 Refreshes the FileBone by recreating file entries if needed and importing blobs from ViUR 2.
302 :param SkeletonInstance skel: The skeleton instance this bone belongs to.
303 :param str boneName: The name of the bone.
305 This method defines an inner function, recreateFileEntryIfNeeded(val), which is responsible
306 for recreating the weak file entry referenced by the relation in val if it doesn't exist
307 (e.g., if it was deleted by ViUR 2). It initializes a new skeleton for the "file" kind and
308 checks if the file object already exists. If not, it recreates the file entry with the
309 appropriate properties and saves it to the database.
311 The main part of the refresh method calls the superclass's refresh method and checks if the
312 configuration contains a ViUR 2 import blob source. If it does, it iterates through the file
313 references in the bone value, imports the blobs from ViUR 2, and recreates the file entries if
314 needed using the inner function.
315 """
316 super().refresh(skel, boneName)
318 for _, _, value in self.iter_bone_value(skel, boneName):
319 # Patch any empty serving_url when public file
320 if (
321 value
322 and (value := value["dest"])
323 and value["public"]
324 and value["mimetype"]
325 and value["mimetype"].startswith("image/")
326 and not value["serving_url"]
327 ):
328 logging.info(f"Patching public image with empty serving_url {value['key']!r} ({value['name']!r})")
329 try:
330 file_skel = value.read()
331 except ValueError:
332 continue
334 file_skel.patch(lambda skel: skel.refresh(), update_relations=False)
335 value["serving_url"] = file_skel["serving_url"]
337 # FIXME: REMOVE THIS WITH VIUR4
338 if conf.viur2import_blobsource:
339 from viur.core.modules.file import importBlobFromViur2
340 from viur.core.skeleton import skeletonByKind
342 def recreateFileEntryIfNeeded(val):
343 # Recreate the (weak) filenetry referenced by the relation *val*. (ViUR2 might have deleted them)
344 skel = skeletonByKind("file")()
345 if skel.read(val["key"]): # This file-object exist, no need to recreate it
346 return
347 skel["key"] = val["key"]
348 skel["name"] = val["name"]
349 skel["mimetype"] = val["mimetype"]
350 skel["dlkey"] = val["dlkey"]
351 skel["size"] = val["size"]
352 skel["width"] = val["width"]
353 skel["height"] = val["height"]
354 skel["weak"] = True
355 skel["pending"] = False
356 skel.write()
358 # Just ensure the file get's imported as it may not have an file entry
359 val = skel[boneName]
360 if isinstance(val, list):
361 for x in val:
362 importBlobFromViur2(x["dest"]["dlkey"], x["dest"]["name"])
363 recreateFileEntryIfNeeded(x["dest"])
364 elif isinstance(val, dict):
365 if not "dest" in val:
366 return
367 importBlobFromViur2(val["dest"]["dlkey"], val["dest"]["name"])
368 recreateFileEntryIfNeeded(val["dest"])
370 def structure(self) -> dict:
371 return super().structure() | {
372 "valid_mime_types": self.validMimeTypes,
373 "public": self.public,
374 }
376 def _atomic_dump(self, value) -> dict | None:
377 value = super()._atomic_dump(value)
378 if value is not None:
379 # VIUR4: Rename "downloadUrl" into "download_url"
380 value["dest"]["downloadUrl"] = conf.main_app.file.create_download_url(
381 value["dest"]["dlkey"],
382 value["dest"]["name"],
383 derived=False,
384 expires=conf.render_json_download_url_expiration
385 )
387 return value