Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/bones/file.py: 9%
152 statements
« prev ^ index » next coverage.py v7.6.12, created at 2025-02-27 07:59 +0000
1"""
2The FileBone is a subclass of the TreeLeafBone class, which is a relational bone that can reference
3another entity's fields. FileBone provides additional file-specific properties and methods, such as
4managing file derivatives, handling file size and mime type restrictions, and refreshing file
5metadata.
6"""
8from hashlib import sha256
9from time import time
10import typing as t
11from viur.core import conf, db, current
12from viur.core.bones.treeleaf import TreeLeafBone
13from viur.core.tasks import CallDeferred
15import logging
18@CallDeferred
19def ensureDerived(key: db.Key, srcKey, deriveMap: dict[str, t.Any], refreshKey: db.Key = None):
20 r"""
21 The function is a deferred function that ensures all pending thumbnails or other derived files
22 are built. It takes the following parameters:
24 :param db.key key: The database key of the file-object that needs to have its derivation map
25 updated.
26 :param str srcKey: A prefix for a stable key to prevent rebuilding derived files repeatedly.
27 :param dict[str,Any] deriveMap: A list of DeriveDicts that need to be built or updated.
28 :param db.Key refreshKey: If set, the function fetches and refreshes the skeleton after
29 building new derived files.
31 The function works by fetching the skeleton of the file-object, checking if it has any derived
32 files, and updating the derivation map accordingly. It iterates through the deriveMap items and
33 calls the appropriate deriver function. If the deriver function returns a result, the function
34 creates a new or updated resultDict and merges it into the file-object's metadata. Finally,
35 the updated results are written back to the database and the updateRelations function is called
36 to ensure proper relations are maintained.
37 """
38 from viur.core.skeleton import skeletonByKind, updateRelations
39 deriveFuncMap = conf.file_derivations
40 skel = skeletonByKind("file")()
41 if not skel.read(key):
42 logging.info("File-Entry went missing in ensureDerived")
43 return
44 if not skel["derived"]:
45 logging.info("No Derives for this file")
46 skel["derived"] = {}
47 skel["derived"]["deriveStatus"] = skel["derived"].get("deriveStatus") or {}
48 skel["derived"]["files"] = skel["derived"].get("files") or {}
49 resDict = {} # Will contain new or updated resultDicts that will be merged into our file
50 for calleeKey, params in deriveMap.items():
51 fullSrcKey = f"{srcKey}_{calleeKey}"
52 paramsHash = sha256(str(params).encode("UTF-8")).hexdigest() # Hash over given params (dict?)
53 if skel["derived"]["deriveStatus"].get(fullSrcKey) != paramsHash:
54 if calleeKey not in deriveFuncMap:
55 logging.warning(f"File-Deriver {calleeKey} not found - skipping!")
56 continue
57 callee = deriveFuncMap[calleeKey]
58 callRes = callee(skel, skel["derived"]["files"], params)
59 if callRes:
60 assert isinstance(callRes, list), "Old (non-list) return value from deriveFunc"
61 resDict[fullSrcKey] = {"version": paramsHash, "files": {}}
62 for fileName, size, mimetype, customData in callRes:
63 resDict[fullSrcKey]["files"][fileName] = {
64 "size": size,
65 "mimetype": mimetype,
66 "customData": customData
67 }
69 def updateTxn(key, resDict):
70 obj = db.Get(key)
71 if not obj: # File-object got deleted during building of our derives
72 return
73 obj["derived"] = obj.get("derived") or {}
74 obj["derived"]["deriveStatus"] = obj["derived"].get("deriveStatus") or {}
75 obj["derived"]["files"] = obj["derived"].get("files") or {}
76 for k, v in resDict.items():
77 obj["derived"]["deriveStatus"][k] = v["version"]
78 for fileName, fileDict in v["files"].items():
79 obj["derived"]["files"][fileName] = fileDict
80 db.Put(obj)
82 if resDict: # Write updated results back and queue updateRelationsTask
83 db.RunInTransaction(updateTxn, key, resDict)
84 # Queue that updateRelations call at least 30 seconds into the future, so that other ensureDerived calls from
85 # the same FileBone have the chance to finish, otherwise that updateRelations Task will call postSavedHandler
86 # on that FileBone again - re-queueing any ensureDerivedCalls that have not finished yet.
87 updateRelations(key, time() + 1, "derived", _countdown=30)
88 if refreshKey:
89 def refreshTxn():
90 skel = skeletonByKind(refreshKey.kind)()
91 if not skel.read(refreshKey):
92 return
93 skel.refresh()
94 skel.write(update_relations=False)
96 db.RunInTransaction(refreshTxn)
99class FileBone(TreeLeafBone):
100 r"""
101 A FileBone is a custom bone class that inherits from the TreeLeafBone class, and is used to store and manage
102 file references in a ViUR application.
104 :param format: Hint for the UI how to display a file entry (defaults to it's filename)
105 :param maxFileSize:
106 The maximum filesize accepted by this bone in bytes. None means no limit.
107 This will always be checked against the original file uploaded - not any of it's derivatives.
109 :param derive: A set of functions used to derive other files from the referenced ones. Used fe.
110 to create thumbnails / images for srcmaps from hires uploads. If set, must be a dictionary from string
111 (a key from conf.file_derivations) to the parameters passed to that function. The parameters can be
112 any type (including None) that can be json-serialized.
114 .. code-block:: python
116 # Example
117 derive = { "thumbnail": [{"width": 111}, {"width": 555, "height": 666}]}
119 :param validMimeTypes:
120 A list of Mimetypes that can be selected in this bone (or None for any) Wildcards ("image\/*") are supported.
122 .. code-block:: python
124 # Example
125 validMimeTypes=["application/pdf", "image/*"]
127 """
129 kind = "file"
130 """The kind of this bone is 'file'"""
132 type = "relational.tree.leaf.file"
133 """The type of this bone is 'relational.tree.leaf.file'."""
135 def __init__(
136 self,
137 *,
138 derive: None | dict[str, t.Any] = None,
139 maxFileSize: None | int = None,
140 validMimeTypes: None | list[str] = None,
141 refKeys: t.Optional[t.Iterable[str]] = (
142 "name",
143 "mimetype",
144 "size",
145 "width",
146 "height",
147 "derived",
148 "public",
149 "serving_url",
150 ),
151 public: bool = False,
152 **kwargs
153 ):
154 r"""
155 Initializes a new Filebone. All properties inherited by RelationalBone are supported.
157 :param format: Hint for the UI how to display a file entry (defaults to it's filename)
158 :param maxFileSize: The maximum filesize accepted by this bone in bytes. None means no limit.
159 This will always be checked against the original file uploaded - not any of it's derivatives.
160 :param derive: A set of functions used to derive other files from the referenced ones.
161 Used to create thumbnails and images for srcmaps from hires uploads.
162 If set, must be a dictionary from string (a key from) conf.file_derivations) to the parameters passed to
163 that function. The parameters can be any type (including None) that can be json-serialized.
165 .. code-block:: python
167 # Example
168 derive = {"thumbnail": [{"width": 111}, {"width": 555, "height": 666}]}
170 :param validMimeTypes:
171 A list of Mimetypes that can be selected in this bone (or None for any).
172 Wildcards `('image\*')` are supported.
174 .. code-block:: python
176 #Example
177 validMimeTypes=["application/pdf", "image/*"]
179 """
180 super().__init__(refKeys=refKeys, **kwargs)
182 self.refKeys.add("dlkey")
183 self.derive = derive
184 self.public = public
185 self.validMimeTypes = validMimeTypes
186 self.maxFileSize = maxFileSize
188 def isInvalid(self, value):
189 """
190 Checks if the provided value is invalid for this bone based on its MIME type and file size.
192 :param dict value: The value to check for validity.
193 :returns: None if the value is valid, or an error message if it is invalid.
194 """
195 if self.validMimeTypes:
196 mimeType = value["dest"]["mimetype"]
197 for checkMT in self.validMimeTypes:
198 checkMT = checkMT.lower()
199 if checkMT == mimeType or checkMT.endswith("*") and mimeType.startswith(checkMT[:-1]):
200 break
201 else:
202 return "Invalid filetype selected"
203 if self.maxFileSize:
204 if value["dest"]["size"] > self.maxFileSize:
205 return "File too large."
207 if value["dest"]["public"] != self.public:
208 return f"Only files marked public={self.public!r} are allowed."
210 return None
212 def postSavedHandler(self, skel, boneName, key):
213 """
214 Handles post-save processing for the FileBone, including ensuring derived files are built.
216 :param SkeletonInstance skel: The skeleton instance this bone belongs to.
217 :param str boneName: The name of the bone.
218 :param db.Key key: The datastore key of the skeleton.
220 This method first calls the postSavedHandler of its superclass. Then, it checks if the
221 derive attribute is set and if there are any values in the skeleton for the given bone. If
222 so, it handles the creation of derived files based on the provided configuration.
224 If the values are stored as a dictionary without a "dest" key, it assumes a multi-language
225 setup and iterates over each language to handle the derived files. Otherwise, it handles
226 the derived files directly.
227 """
228 super().postSavedHandler(skel, boneName, key)
229 if current.request.get().is_deferred and current.request_data.get().get("__update_relations_bone") == "derived":
230 return
231 from viur.core.skeleton import RelSkel, Skeleton
233 if issubclass(skel.skeletonCls, Skeleton):
234 prefix = f"{skel.kindName}_{boneName}"
235 elif issubclass(skel.skeletonCls, RelSkel): # RelSkel is just a container and has no kindname
236 prefix = f"{skel.skeletonCls.__name__}_{boneName}"
237 else:
238 raise NotImplementedError(f"Cannot handle {skel.skeletonCls=}")
240 def handleDerives(values):
241 if isinstance(values, dict):
242 values = [values]
243 for val in (values or ()): # Ensure derives getting build for each file referenced in this relation
244 ensureDerived(val["dest"]["key"], prefix, self.derive)
246 values = skel[boneName]
247 if self.derive and values:
248 if isinstance(values, dict) and "dest" not in values: # multi lang
249 for lang in values:
250 handleDerives(values[lang])
251 else:
252 handleDerives(values)
254 def getReferencedBlobs(self, skel: 'viur.core.skeleton.SkeletonInstance', name: str) -> set[str]:
255 r"""
256 Retrieves the referenced blobs in the FileBone.
258 :param SkeletonInstance skel: The skeleton instance this bone belongs to.
259 :param str name: The name of the bone.
260 :return: A set of download keys for the referenced blobs.
261 :rtype: Set[str]
263 This method iterates over the bone values for the given skeleton and bone name. It skips
264 values that are None. For each non-None value, it adds the download key of the referenced
265 blob to a set. Finally, it returns the set of unique download keys for the referenced blobs.
266 """
267 result = set()
268 for idx, lang, value in self.iter_bone_value(skel, name):
269 if value is None:
270 continue
271 result.add(value["dest"]["dlkey"])
272 return result
274 def refresh(self, skel, boneName):
275 r"""
276 Refreshes the FileBone by recreating file entries if needed and importing blobs from ViUR 2.
278 :param SkeletonInstance skel: The skeleton instance this bone belongs to.
279 :param str boneName: The name of the bone.
281 This method defines an inner function, recreateFileEntryIfNeeded(val), which is responsible
282 for recreating the weak file entry referenced by the relation in val if it doesn't exist
283 (e.g., if it was deleted by ViUR 2). It initializes a new skeleton for the "file" kind and
284 checks if the file object already exists. If not, it recreates the file entry with the
285 appropriate properties and saves it to the database.
287 The main part of the refresh method calls the superclass's refresh method and checks if the
288 configuration contains a ViUR 2 import blob source. If it does, it iterates through the file
289 references in the bone value, imports the blobs from ViUR 2, and recreates the file entries if
290 needed using the inner function.
291 """
292 super().refresh(skel, boneName)
294 for _, _, value in self.iter_bone_value(skel, boneName):
295 # Patch any empty serving_url when public file
296 if (
297 value
298 and (value := value["dest"])
299 and value["public"]
300 and value["mimetype"]
301 and value["mimetype"].startswith("image/")
302 and not value["serving_url"]
303 ):
304 logging.info(f"Patching public image with empty serving_url {value['key']!r} ({value['name']!r})")
305 try:
306 file_skel = value.read()
307 except ValueError:
308 continue
310 file_skel.patch(lambda skel: skel.refresh(), update_relations=False)
311 value["serving_url"] = file_skel["serving_url"]
313 # FIXME: REMOVE THIS WITH VIUR4
314 if conf.viur2import_blobsource:
315 from viur.core.modules.file import importBlobFromViur2
316 from viur.core.skeleton import skeletonByKind
318 def recreateFileEntryIfNeeded(val):
319 # Recreate the (weak) filenetry referenced by the relation *val*. (ViUR2 might have deleted them)
320 skel = skeletonByKind("file")()
321 if skel.read(val["key"]): # This file-object exist, no need to recreate it
322 return
323 skel["key"] = val["key"]
324 skel["name"] = val["name"]
325 skel["mimetype"] = val["mimetype"]
326 skel["dlkey"] = val["dlkey"]
327 skel["size"] = val["size"]
328 skel["width"] = val["width"]
329 skel["height"] = val["height"]
330 skel["weak"] = True
331 skel["pending"] = False
332 skel.write()
334 # Just ensure the file get's imported as it may not have an file entry
335 val = skel[boneName]
336 if isinstance(val, list):
337 for x in val:
338 importBlobFromViur2(x["dest"]["dlkey"], x["dest"]["name"])
339 recreateFileEntryIfNeeded(x["dest"])
340 elif isinstance(val, dict):
341 if not "dest" in val:
342 return
343 importBlobFromViur2(val["dest"]["dlkey"], val["dest"]["name"])
344 recreateFileEntryIfNeeded(val["dest"])
346 def structure(self) -> dict:
347 return super().structure() | {
348 "valid_mime_types": self.validMimeTypes,
349 "public": self.public,
350 }