Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/bones/file.py: 10%
146 statements
« prev ^ index » next coverage.py v7.10.6, created at 2025-09-16 12:35 +0000
1"""
2The FileBone is a subclass of the TreeLeafBone class, which is a relational bone that can reference
3another entity's fields. FileBone provides additional file-specific properties and methods, such as
4managing file derivatives, handling file size and mime type restrictions, and refreshing file
5metadata.
6"""
7import hashlib
8import warnings
9import time
10import typing as t
11from viur.core import conf, db, current, utils
12from viur.core.bones.treeleaf import TreeLeafBone
13from viur.core.tasks import CallDeferred
14import logging
17@CallDeferred
18def ensureDerived(key: db.Key, src_key, derive_map: dict[str, t.Any], refresh_key: db.Key = None, **kwargs):
19 r"""
20 The function is a deferred function that ensures all pending thumbnails or other derived files
21 are built. It takes the following parameters:
23 :param db.key key: The database key of the file-object that needs to have its derivation map
24 updated.
25 :param str src_key: A prefix for a stable key to prevent rebuilding derived files repeatedly.
26 :param dict[str,Any] derive_map: A list of DeriveDicts that need to be built or updated.
27 :param db.Key refresh_key: If set, the function fetches and refreshes the skeleton after
28 building new derived files.
30 The function works by fetching the skeleton of the file-object, checking if it has any derived
31 files, and updating the derivation map accordingly. It iterates through the derive_map items and
32 calls the appropriate deriver function. If the deriver function returns a result, the function
33 creates a new or updated resultDict and merges it into the file-object's metadata. Finally,
34 the updated results are written back to the database and the update_relations function is called
35 to ensure proper relations are maintained.
36 """
37 # TODO: Remove in VIUR4
38 for _dep, _new in {
39 "srcKey": "src_key",
40 "deriveMap": "derive_map",
41 "refreshKey": "refresh_key",
42 }.items():
43 if _dep in kwargs:
44 warnings.warn(
45 f"{_dep!r} parameter is deprecated, please use {_new!r} instead",
46 DeprecationWarning, stacklevel=2
47 )
49 locals()[_new] = kwargs.pop(_dep)
50 from viur.core.skeleton.utils import skeletonByKind
51 from viur.core.skeleton.tasks import update_relations
53 skel = skeletonByKind(key.kind)()
54 if not skel.read(key):
55 logging.info("File-Entry went missing in ensureDerived")
56 return
57 if not skel["derived"]:
58 logging.info("No Derives for this file")
59 skel["derived"] = {}
60 skel["derived"] = {"deriveStatus": {}, "files": {}} | skel["derived"]
61 res_status, res_files = {}, {}
62 for call_key, params in derive_map.items():
63 full_src_key = f"{src_key}_{call_key}"
64 params_hash = hashlib.sha256(str(params).encode("UTF-8")).hexdigest() # Hash over given params (dict?)
65 if skel["derived"]["deriveStatus"].get(full_src_key) != params_hash:
66 if not (caller := conf.file_derivations.get(call_key)):
67 logging.warning(f"File-Deriver {call_key} not found - skipping!")
68 continue
70 if call_res := caller(skel, skel["derived"]["files"], params):
71 assert isinstance(call_res, list), "Old (non-list) return value from deriveFunc"
72 res_status[full_src_key] = params_hash
73 for file_name, size, mimetype, custom_data in call_res:
74 res_files[file_name] = {
75 "size": size,
76 "mimetype": mimetype,
77 "customData": custom_data # TODO: Rename in VIUR4
78 }
80 if res_status: # Write updated results back and queue updateRelationsTask
81 def _merge_derives(patch_skel):
82 patch_skel["derived"] = {"deriveStatus": {}, "files": {}} | (patch_skel["derived"] or {})
83 patch_skel["derived"]["deriveStatus"] = patch_skel["derived"]["deriveStatus"] | res_status
84 patch_skel["derived"]["files"] = patch_skel["derived"]["files"] | res_files
86 skel.patch(values=_merge_derives, update_relations=False)
88 # Queue that update_relations call at least 30 seconds into the future, so that other ensureDerived calls from
89 # the same FileBone have the chance to finish, otherwise that update_relations Task will call postSavedHandler
90 # on that FileBone again - re-queueing any ensureDerivedCalls that have not finished yet.
92 if refresh_key:
93 skel = skeletonByKind(refresh_key.kind)()
94 skel.patch(lambda _skel: _skel.refresh(), key=refresh_key, update_relations=False)
96 update_relations(key, min_change_time=int(time.time() + 1), changed_bones=["derived"], _countdown=30)
99class FileBone(TreeLeafBone):
100 r"""
101 A FileBone is a custom bone class that inherits from the TreeLeafBone class, and is used to store and manage
102 file references in a ViUR application.
104 :param format: Hint for the UI how to display a file entry (defaults to it's filename)
105 :param maxFileSize:
106 The maximum filesize accepted by this bone in bytes. None means no limit.
107 This will always be checked against the original file uploaded - not any of it's derivatives.
109 :param derive: A set of functions used to derive other files from the referenced ones. Used fe.
110 to create thumbnails / images for srcmaps from hires uploads. If set, must be a dictionary from string
111 (a key from conf.file_derivations) to the parameters passed to that function. The parameters can be
112 any type (including None) that can be json-serialized.
114 .. code-block:: python
116 # Example
117 derive = { "thumbnail": [{"width": 111}, {"width": 555, "height": 666}]}
119 :param validMimeTypes:
120 A list of Mimetypes that can be selected in this bone (or None for any) Wildcards ("image\/*") are supported.
122 .. code-block:: python
124 # Example
125 validMimeTypes=["application/pdf", "image/*"]
127 """
129 kind = "file"
130 """The kind of this bone is 'file'"""
132 type = "relational.tree.leaf.file"
133 """The type of this bone is 'relational.tree.leaf.file'."""
135 def __init__(
136 self,
137 *,
138 derive: None | dict[str, t.Any] = None,
139 maxFileSize: None | int = None,
140 validMimeTypes: None | list[str] = None,
141 refKeys: t.Optional[t.Iterable[str]] = (
142 "name",
143 "mimetype",
144 "size",
145 "width",
146 "height",
147 "derived",
148 "public",
149 "serving_url",
150 ),
151 public: bool = False,
152 **kwargs
153 ):
154 r"""
155 Initializes a new Filebone. All properties inherited by RelationalBone are supported.
157 :param format: Hint for the UI how to display a file entry (defaults to it's filename)
158 :param maxFileSize: The maximum filesize accepted by this bone in bytes. None means no limit.
159 This will always be checked against the original file uploaded - not any of it's derivatives.
160 :param derive: A set of functions used to derive other files from the referenced ones.
161 Used to create thumbnails and images for srcmaps from hires uploads.
162 If set, must be a dictionary from string (a key from) conf.file_derivations) to the parameters passed to
163 that function. The parameters can be any type (including None) that can be json-serialized.
165 .. code-block:: python
167 # Example
168 derive = {"thumbnail": [{"width": 111}, {"width": 555, "height": 666}]}
170 :param validMimeTypes:
171 A list of Mimetypes that can be selected in this bone (or None for any).
172 Wildcards `('image\*')` are supported.
174 .. code-block:: python
176 #Example
177 validMimeTypes=["application/pdf", "image/*"]
179 """
180 super().__init__(refKeys=refKeys, **kwargs)
182 self.refKeys.add("dlkey")
183 self.derive = derive
184 self.public = public
185 self.validMimeTypes = validMimeTypes
186 self.maxFileSize = maxFileSize
188 def isInvalid(self, value):
189 """
190 Checks if the provided value is invalid for this bone based on its MIME type and file size.
192 :param dict value: The value to check for validity.
193 :returns: None if the value is valid, or an error message if it is invalid.
194 """
195 if self.validMimeTypes:
196 mimeType = value["dest"]["mimetype"]
197 for checkMT in self.validMimeTypes:
198 checkMT = checkMT.lower()
199 if checkMT == mimeType or checkMT.endswith("*") and mimeType.startswith(checkMT[:-1]):
200 break
201 else:
202 return "Invalid filetype selected"
203 if self.maxFileSize:
204 if value["dest"]["size"] > self.maxFileSize:
205 return "File too large."
207 if value["dest"]["public"] != self.public:
208 return f"Only files marked public={self.public!r} are allowed."
210 return None
212 def postSavedHandler(self, skel, boneName, key):
213 """
214 Handles post-save processing for the FileBone, including ensuring derived files are built.
216 :param SkeletonInstance skel: The skeleton instance this bone belongs to.
217 :param str boneName: The name of the bone.
218 :param db.Key key: The datastore key of the skeleton.
220 This method first calls the postSavedHandler of its superclass. Then, it checks if the
221 derive attribute is set and if there are any values in the skeleton for the given bone. If
222 so, it handles the creation of derived files based on the provided configuration.
224 If the values are stored as a dictionary without a "dest" key, it assumes a multi-language
225 setup and iterates over each language to handle the derived files. Otherwise, it handles
226 the derived files directly.
227 """
228 super().postSavedHandler(skel, boneName, key)
229 if (
230 current.request.get().is_deferred
231 and "derived" in (current.request_data.get().get("__update_relations_bones") or ())
232 ):
233 return
235 from viur.core.skeleton import RelSkel, Skeleton
237 if issubclass(skel.skeletonCls, Skeleton):
238 prefix = f"{skel.kindName}_{boneName}"
239 elif issubclass(skel.skeletonCls, RelSkel): # RelSkel is just a container and has no kindname
240 prefix = f"{skel.skeletonCls.__name__}_{boneName}"
241 else:
242 raise NotImplementedError(f"Cannot handle {skel.skeletonCls=}")
244 def handleDerives(values):
245 if isinstance(values, dict):
246 values = [values]
247 for val in (values or ()): # Ensure derives getting build for each file referenced in this relation
248 ensureDerived(val["dest"]["key"], prefix, self.derive, key)
250 values = skel[boneName]
251 if self.derive and values:
252 if isinstance(values, dict) and "dest" not in values: # multi lang
253 for lang in values:
254 handleDerives(values[lang])
255 else:
256 handleDerives(values)
258 def getReferencedBlobs(self, skel: 'viur.core.skeleton.SkeletonInstance', name: str) -> set[str]:
259 r"""
260 Retrieves the referenced blobs in the FileBone.
262 :param SkeletonInstance skel: The skeleton instance this bone belongs to.
263 :param str name: The name of the bone.
264 :return: A set of download keys for the referenced blobs.
265 :rtype: Set[str]
267 This method iterates over the bone values for the given skeleton and bone name. It skips
268 values that are None. For each non-None value, it adds the download key of the referenced
269 blob to a set. Finally, it returns the set of unique download keys for the referenced blobs.
270 """
271 result = set()
272 for idx, lang, value in self.iter_bone_value(skel, name):
273 if value is None:
274 continue
275 result.add(value["dest"]["dlkey"])
276 return result
278 def refresh(self, skel, boneName):
279 r"""
280 Refreshes the FileBone by recreating file entries if needed and importing blobs from ViUR 2.
282 :param SkeletonInstance skel: The skeleton instance this bone belongs to.
283 :param str boneName: The name of the bone.
285 This method defines an inner function, recreateFileEntryIfNeeded(val), which is responsible
286 for recreating the weak file entry referenced by the relation in val if it doesn't exist
287 (e.g., if it was deleted by ViUR 2). It initializes a new skeleton for the "file" kind and
288 checks if the file object already exists. If not, it recreates the file entry with the
289 appropriate properties and saves it to the database.
291 The main part of the refresh method calls the superclass's refresh method and checks if the
292 configuration contains a ViUR 2 import blob source. If it does, it iterates through the file
293 references in the bone value, imports the blobs from ViUR 2, and recreates the file entries if
294 needed using the inner function.
295 """
296 super().refresh(skel, boneName)
298 for _, _, value in self.iter_bone_value(skel, boneName):
299 # Patch any empty serving_url when public file
300 if (
301 value
302 and (value := value["dest"])
303 and value["public"]
304 and value["mimetype"]
305 and value["mimetype"].startswith("image/")
306 and not value["serving_url"]
307 ):
308 logging.info(f"Patching public image with empty serving_url {value['key']!r} ({value['name']!r})")
309 try:
310 file_skel = value.read()
311 except ValueError:
312 continue
314 file_skel.patch(lambda skel: skel.refresh(), update_relations=False)
315 value["serving_url"] = file_skel["serving_url"]
317 # FIXME: REMOVE THIS WITH VIUR4
318 if conf.viur2import_blobsource:
319 from viur.core.modules.file import importBlobFromViur2
320 from viur.core.skeleton import skeletonByKind
322 def recreateFileEntryIfNeeded(val):
323 # Recreate the (weak) filenetry referenced by the relation *val*. (ViUR2 might have deleted them)
324 skel = skeletonByKind("file")()
325 if skel.read(val["key"]): # This file-object exist, no need to recreate it
326 return
327 skel["key"] = val["key"]
328 skel["name"] = val["name"]
329 skel["mimetype"] = val["mimetype"]
330 skel["dlkey"] = val["dlkey"]
331 skel["size"] = val["size"]
332 skel["width"] = val["width"]
333 skel["height"] = val["height"]
334 skel["weak"] = True
335 skel["pending"] = False
336 skel.write()
338 # Just ensure the file get's imported as it may not have an file entry
339 val = skel[boneName]
340 if isinstance(val, list):
341 for x in val:
342 importBlobFromViur2(x["dest"]["dlkey"], x["dest"]["name"])
343 recreateFileEntryIfNeeded(x["dest"])
344 elif isinstance(val, dict):
345 if not "dest" in val:
346 return
347 importBlobFromViur2(val["dest"]["dlkey"], val["dest"]["name"])
348 recreateFileEntryIfNeeded(val["dest"])
350 def structure(self) -> dict:
351 return super().structure() | {
352 "valid_mime_types": self.validMimeTypes,
353 "public": self.public,
354 }
356 def _atomic_dump(self, value) -> dict | None:
357 value = super()._atomic_dump(value)
358 if value is not None:
359 value["dest"]["downloadUrl"] = conf.main_app.file.create_download_url(
360 value["dest"]["dlkey"],
361 value["dest"]["name"],
362 derived=False,
363 expires=conf.render_json_download_url_expiration
364 )
366 return value