Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/bones/file.py: 10%

146 statements  

« prev     ^ index     » next       coverage.py v7.10.6, created at 2025-09-13 11:04 +0000

1""" 

2The FileBone is a subclass of the TreeLeafBone class, which is a relational bone that can reference 

3another entity's fields. FileBone provides additional file-specific properties and methods, such as 

4managing file derivatives, handling file size and mime type restrictions, and refreshing file 

5metadata. 

6""" 

7import hashlib 

8import warnings 

9import time 

10import typing as t 

11from viur.core import conf, db, current, utils 

12from viur.core.bones.treeleaf import TreeLeafBone 

13from viur.core.tasks import CallDeferred 

14import logging 

15 

16 

17@CallDeferred 

18def ensureDerived(key: db.Key, src_key, derive_map: dict[str, t.Any], refresh_key: db.Key = None, **kwargs): 

19 r""" 

20 The function is a deferred function that ensures all pending thumbnails or other derived files 

21 are built. It takes the following parameters: 

22 

23 :param db.key key: The database key of the file-object that needs to have its derivation map 

24 updated. 

25 :param str src_key: A prefix for a stable key to prevent rebuilding derived files repeatedly. 

26 :param dict[str,Any] derive_map: A list of DeriveDicts that need to be built or updated. 

27 :param db.Key refresh_key: If set, the function fetches and refreshes the skeleton after 

28 building new derived files. 

29 

30 The function works by fetching the skeleton of the file-object, checking if it has any derived 

31 files, and updating the derivation map accordingly. It iterates through the derive_map items and 

32 calls the appropriate deriver function. If the deriver function returns a result, the function 

33 creates a new or updated resultDict and merges it into the file-object's metadata. Finally, 

34 the updated results are written back to the database and the update_relations function is called 

35 to ensure proper relations are maintained. 

36 """ 

37 # TODO: Remove in VIUR4 

38 for _dep, _new in { 

39 "srcKey": "src_key", 

40 "deriveMap": "derive_map", 

41 "refreshKey": "refresh_key", 

42 }.items(): 

43 if _dep in kwargs: 

44 warnings.warn( 

45 f"{_dep!r} parameter is deprecated, please use {_new!r} instead", 

46 DeprecationWarning, stacklevel=2 

47 ) 

48 

49 locals()[_new] = kwargs.pop(_dep) 

50 from viur.core.skeleton.utils import skeletonByKind 

51 from viur.core.skeleton.tasks import update_relations 

52 

53 skel = skeletonByKind(key.kind)() 

54 if not skel.read(key): 

55 logging.info("File-Entry went missing in ensureDerived") 

56 return 

57 if not skel["derived"]: 

58 logging.info("No Derives for this file") 

59 skel["derived"] = {} 

60 skel["derived"] = {"deriveStatus": {}, "files": {}} | skel["derived"] 

61 res_status, res_files = {}, {} 

62 for call_key, params in derive_map.items(): 

63 full_src_key = f"{src_key}_{call_key}" 

64 params_hash = hashlib.sha256(str(params).encode("UTF-8")).hexdigest() # Hash over given params (dict?) 

65 if skel["derived"]["deriveStatus"].get(full_src_key) != params_hash: 

66 if not (caller := conf.file_derivations.get(call_key)): 

67 logging.warning(f"File-Deriver {call_key} not found - skipping!") 

68 continue 

69 

70 if call_res := caller(skel, skel["derived"]["files"], params): 

71 assert isinstance(call_res, list), "Old (non-list) return value from deriveFunc" 

72 res_status[full_src_key] = params_hash 

73 for file_name, size, mimetype, custom_data in call_res: 

74 res_files[file_name] = { 

75 "size": size, 

76 "mimetype": mimetype, 

77 "customData": custom_data # TODO: Rename in VIUR4 

78 } 

79 

80 if res_status: # Write updated results back and queue updateRelationsTask 

81 def _merge_derives(patch_skel): 

82 patch_skel["derived"] = {"deriveStatus": {}, "files": {}} | (patch_skel["derived"] or {}) 

83 patch_skel["derived"]["deriveStatus"] = patch_skel["derived"]["deriveStatus"] | res_status 

84 patch_skel["derived"]["files"] = patch_skel["derived"]["files"] | res_files 

85 

86 skel.patch(values=_merge_derives, update_relations=False) 

87 

88 # Queue that update_relations call at least 30 seconds into the future, so that other ensureDerived calls from 

89 # the same FileBone have the chance to finish, otherwise that update_relations Task will call postSavedHandler 

90 # on that FileBone again - re-queueing any ensureDerivedCalls that have not finished yet. 

91 

92 if refresh_key: 

93 skel = skeletonByKind(refresh_key.kind)() 

94 skel.patch(lambda _skel: _skel.refresh(), key=refresh_key, update_relations=False) 

95 

96 update_relations(key, min_change_time=int(time.time() + 1), changed_bones=["derived"], _countdown=30) 

97 

98 

99class FileBone(TreeLeafBone): 

100 r""" 

101 A FileBone is a custom bone class that inherits from the TreeLeafBone class, and is used to store and manage 

102 file references in a ViUR application. 

103 

104 :param format: Hint for the UI how to display a file entry (defaults to it's filename) 

105 :param maxFileSize: 

106 The maximum filesize accepted by this bone in bytes. None means no limit. 

107 This will always be checked against the original file uploaded - not any of it's derivatives. 

108 

109 :param derive: A set of functions used to derive other files from the referenced ones. Used fe. 

110 to create thumbnails / images for srcmaps from hires uploads. If set, must be a dictionary from string 

111 (a key from conf.file_derivations) to the parameters passed to that function. The parameters can be 

112 any type (including None) that can be json-serialized. 

113 

114 .. code-block:: python 

115 

116 # Example 

117 derive = { "thumbnail": [{"width": 111}, {"width": 555, "height": 666}]} 

118 

119 :param validMimeTypes: 

120 A list of Mimetypes that can be selected in this bone (or None for any) Wildcards ("image\/*") are supported. 

121 

122 .. code-block:: python 

123 

124 # Example 

125 validMimeTypes=["application/pdf", "image/*"] 

126 

127 """ 

128 

129 kind = "file" 

130 """The kind of this bone is 'file'""" 

131 

132 type = "relational.tree.leaf.file" 

133 """The type of this bone is 'relational.tree.leaf.file'.""" 

134 

135 def __init__( 

136 self, 

137 *, 

138 derive: None | dict[str, t.Any] = None, 

139 maxFileSize: None | int = None, 

140 validMimeTypes: None | list[str] = None, 

141 refKeys: t.Optional[t.Iterable[str]] = ( 

142 "name", 

143 "mimetype", 

144 "size", 

145 "width", 

146 "height", 

147 "derived", 

148 "public", 

149 "serving_url", 

150 ), 

151 public: bool = False, 

152 **kwargs 

153 ): 

154 r""" 

155 Initializes a new Filebone. All properties inherited by RelationalBone are supported. 

156 

157 :param format: Hint for the UI how to display a file entry (defaults to it's filename) 

158 :param maxFileSize: The maximum filesize accepted by this bone in bytes. None means no limit. 

159 This will always be checked against the original file uploaded - not any of it's derivatives. 

160 :param derive: A set of functions used to derive other files from the referenced ones. 

161 Used to create thumbnails and images for srcmaps from hires uploads. 

162 If set, must be a dictionary from string (a key from) conf.file_derivations) to the parameters passed to 

163 that function. The parameters can be any type (including None) that can be json-serialized. 

164 

165 .. code-block:: python 

166 

167 # Example 

168 derive = {"thumbnail": [{"width": 111}, {"width": 555, "height": 666}]} 

169 

170 :param validMimeTypes: 

171 A list of Mimetypes that can be selected in this bone (or None for any). 

172 Wildcards `('image\*')` are supported. 

173 

174 .. code-block:: python 

175 

176 #Example 

177 validMimeTypes=["application/pdf", "image/*"] 

178 

179 """ 

180 super().__init__(refKeys=refKeys, **kwargs) 

181 

182 self.refKeys.add("dlkey") 

183 self.derive = derive 

184 self.public = public 

185 self.validMimeTypes = validMimeTypes 

186 self.maxFileSize = maxFileSize 

187 

188 def isInvalid(self, value): 

189 """ 

190 Checks if the provided value is invalid for this bone based on its MIME type and file size. 

191 

192 :param dict value: The value to check for validity. 

193 :returns: None if the value is valid, or an error message if it is invalid. 

194 """ 

195 if self.validMimeTypes: 

196 mimeType = value["dest"]["mimetype"] 

197 for checkMT in self.validMimeTypes: 

198 checkMT = checkMT.lower() 

199 if checkMT == mimeType or checkMT.endswith("*") and mimeType.startswith(checkMT[:-1]): 

200 break 

201 else: 

202 return "Invalid filetype selected" 

203 if self.maxFileSize: 

204 if value["dest"]["size"] > self.maxFileSize: 

205 return "File too large." 

206 

207 if value["dest"]["public"] != self.public: 

208 return f"Only files marked public={self.public!r} are allowed." 

209 

210 return None 

211 

212 def postSavedHandler(self, skel, boneName, key): 

213 """ 

214 Handles post-save processing for the FileBone, including ensuring derived files are built. 

215 

216 :param SkeletonInstance skel: The skeleton instance this bone belongs to. 

217 :param str boneName: The name of the bone. 

218 :param db.Key key: The datastore key of the skeleton. 

219 

220 This method first calls the postSavedHandler of its superclass. Then, it checks if the 

221 derive attribute is set and if there are any values in the skeleton for the given bone. If 

222 so, it handles the creation of derived files based on the provided configuration. 

223 

224 If the values are stored as a dictionary without a "dest" key, it assumes a multi-language 

225 setup and iterates over each language to handle the derived files. Otherwise, it handles 

226 the derived files directly. 

227 """ 

228 super().postSavedHandler(skel, boneName, key) 

229 if ( 

230 current.request.get().is_deferred 

231 and "derived" in (current.request_data.get().get("__update_relations_bones") or ()) 

232 ): 

233 return 

234 

235 from viur.core.skeleton import RelSkel, Skeleton 

236 

237 if issubclass(skel.skeletonCls, Skeleton): 

238 prefix = f"{skel.kindName}_{boneName}" 

239 elif issubclass(skel.skeletonCls, RelSkel): # RelSkel is just a container and has no kindname 

240 prefix = f"{skel.skeletonCls.__name__}_{boneName}" 

241 else: 

242 raise NotImplementedError(f"Cannot handle {skel.skeletonCls=}") 

243 

244 def handleDerives(values): 

245 if isinstance(values, dict): 

246 values = [values] 

247 for val in (values or ()): # Ensure derives getting build for each file referenced in this relation 

248 ensureDerived(val["dest"]["key"], prefix, self.derive, key) 

249 

250 values = skel[boneName] 

251 if self.derive and values: 

252 if isinstance(values, dict) and "dest" not in values: # multi lang 

253 for lang in values: 

254 handleDerives(values[lang]) 

255 else: 

256 handleDerives(values) 

257 

258 def getReferencedBlobs(self, skel: 'viur.core.skeleton.SkeletonInstance', name: str) -> set[str]: 

259 r""" 

260 Retrieves the referenced blobs in the FileBone. 

261 

262 :param SkeletonInstance skel: The skeleton instance this bone belongs to. 

263 :param str name: The name of the bone. 

264 :return: A set of download keys for the referenced blobs. 

265 :rtype: Set[str] 

266 

267 This method iterates over the bone values for the given skeleton and bone name. It skips 

268 values that are None. For each non-None value, it adds the download key of the referenced 

269 blob to a set. Finally, it returns the set of unique download keys for the referenced blobs. 

270 """ 

271 result = set() 

272 for idx, lang, value in self.iter_bone_value(skel, name): 

273 if value is None: 

274 continue 

275 result.add(value["dest"]["dlkey"]) 

276 return result 

277 

278 def refresh(self, skel, boneName): 

279 r""" 

280 Refreshes the FileBone by recreating file entries if needed and importing blobs from ViUR 2. 

281 

282 :param SkeletonInstance skel: The skeleton instance this bone belongs to. 

283 :param str boneName: The name of the bone. 

284 

285 This method defines an inner function, recreateFileEntryIfNeeded(val), which is responsible 

286 for recreating the weak file entry referenced by the relation in val if it doesn't exist 

287 (e.g., if it was deleted by ViUR 2). It initializes a new skeleton for the "file" kind and 

288 checks if the file object already exists. If not, it recreates the file entry with the 

289 appropriate properties and saves it to the database. 

290 

291 The main part of the refresh method calls the superclass's refresh method and checks if the 

292 configuration contains a ViUR 2 import blob source. If it does, it iterates through the file 

293 references in the bone value, imports the blobs from ViUR 2, and recreates the file entries if 

294 needed using the inner function. 

295 """ 

296 super().refresh(skel, boneName) 

297 

298 for _, _, value in self.iter_bone_value(skel, boneName): 

299 # Patch any empty serving_url when public file 

300 if ( 

301 value 

302 and (value := value["dest"]) 

303 and value["public"] 

304 and value["mimetype"] 

305 and value["mimetype"].startswith("image/") 

306 and not value["serving_url"] 

307 ): 

308 logging.info(f"Patching public image with empty serving_url {value['key']!r} ({value['name']!r})") 

309 try: 

310 file_skel = value.read() 

311 except ValueError: 

312 continue 

313 

314 file_skel.patch(lambda skel: skel.refresh(), update_relations=False) 

315 value["serving_url"] = file_skel["serving_url"] 

316 

317 # FIXME: REMOVE THIS WITH VIUR4 

318 if conf.viur2import_blobsource: 

319 from viur.core.modules.file import importBlobFromViur2 

320 from viur.core.skeleton import skeletonByKind 

321 

322 def recreateFileEntryIfNeeded(val): 

323 # Recreate the (weak) filenetry referenced by the relation *val*. (ViUR2 might have deleted them) 

324 skel = skeletonByKind("file")() 

325 if skel.read(val["key"]): # This file-object exist, no need to recreate it 

326 return 

327 skel["key"] = val["key"] 

328 skel["name"] = val["name"] 

329 skel["mimetype"] = val["mimetype"] 

330 skel["dlkey"] = val["dlkey"] 

331 skel["size"] = val["size"] 

332 skel["width"] = val["width"] 

333 skel["height"] = val["height"] 

334 skel["weak"] = True 

335 skel["pending"] = False 

336 skel.write() 

337 

338 # Just ensure the file get's imported as it may not have an file entry 

339 val = skel[boneName] 

340 if isinstance(val, list): 

341 for x in val: 

342 importBlobFromViur2(x["dest"]["dlkey"], x["dest"]["name"]) 

343 recreateFileEntryIfNeeded(x["dest"]) 

344 elif isinstance(val, dict): 

345 if not "dest" in val: 

346 return 

347 importBlobFromViur2(val["dest"]["dlkey"], val["dest"]["name"]) 

348 recreateFileEntryIfNeeded(val["dest"]) 

349 

350 def structure(self) -> dict: 

351 return super().structure() | { 

352 "valid_mime_types": self.validMimeTypes, 

353 "public": self.public, 

354 } 

355 

356 def _atomic_dump(self, value) -> dict | None: 

357 value = super()._atomic_dump(value) 

358 if value is not None: 

359 value["dest"]["downloadUrl"] = conf.main_app.file.create_download_url( 

360 value["dest"]["dlkey"], 

361 value["dest"]["name"], 

362 derived=False, 

363 expires=conf.render_json_download_url_expiration 

364 ) 

365 

366 return value