Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/bones/file.py: 11%

150 statements  

« prev     ^ index     » next       coverage.py v7.10.7, created at 2025-09-29 09:00 +0000

1""" 

2The FileBone is a subclass of the TreeLeafBone class, which is a relational bone that can reference 

3another entity's fields. FileBone provides additional file-specific properties and methods, such as 

4managing file derivatives, handling file size and mime type restrictions, and refreshing file 

5metadata. 

6""" 

7import hashlib 

8import warnings 

9import time 

10import typing as t 

11from viur.core import conf, db, current, utils 

12from viur.core.bones.treeleaf import TreeLeafBone 

13from viur.core.tasks import CallDeferred 

14import logging 

15 

16 

17@CallDeferred 

18def ensureDerived( 

19 key: db.Key, 

20 src_key: str, 

21 derive_map: dict[str, t.Any], 

22 refresh_key: db.Key = None, 

23 **kwargs 

24): 

25 r""" 

26 The function is a deferred function that ensures all pending thumbnails or other derived files 

27 are built. It takes the following parameters: 

28 

29 :param db.key key: The database key of the file-object that needs to have its derivation map 

30 updated. 

31 :param str src_key: A prefix for a stable key to prevent rebuilding derived files repeatedly. 

32 :param dict[str,Any] derive_map: A list of DeriveDicts that need to be built or updated. 

33 :param db.Key refresh_key: If set, the function fetches and refreshes the skeleton after 

34 building new derived files. 

35 

36 The function works by fetching the skeleton of the file-object, checking if it has any derived 

37 files, and updating the derivation map accordingly. It iterates through the derive_map items and 

38 calls the appropriate deriver function. If the deriver function returns a result, the function 

39 creates a new or updated resultDict and merges it into the file-object's metadata. Finally, 

40 the updated results are written back to the database and the update_relations function is called 

41 to ensure proper relations are maintained. 

42 """ 

43 # TODO: Remove in VIUR4 

44 for _dep, _new in { 

45 "srcKey": "src_key", 

46 "deriveMap": "derive_map", 

47 "refreshKey": "refresh_key", 

48 }.items(): 

49 if _dep in kwargs: 

50 warnings.warn( 

51 f"{_dep!r} parameter is deprecated, please use {_new!r} instead", 

52 DeprecationWarning, stacklevel=2 

53 ) 

54 

55 locals()[_new] = kwargs.pop(_dep) 

56 

57 from viur.core.skeleton.utils import skeletonByKind 

58 from viur.core.skeleton.tasks import update_relations 

59 

60 skel = skeletonByKind(key.kind)() 

61 if not skel.read(key): 

62 logging.error(f"{src_key}: File not found, is it gone?") 

63 return 

64 

65 if not skel["derived"]: 

66 logging.info(f"{src_key}: No derives for this file") 

67 skel["derived"] = {} 

68 

69 skel["derived"] = {"deriveStatus": {}, "files": {}} | skel["derived"] 

70 

71 res_status, res_files = {}, {} 

72 for call_key, params in derive_map.items(): 

73 full_src_key = f"{src_key}_{call_key}" 

74 params_hash = hashlib.sha256(str(params).encode("UTF-8")).hexdigest() # Hash over given params (dict?) 

75 if skel["derived"]["deriveStatus"].get(full_src_key) != params_hash: 

76 if not (caller := conf.file_derivations.get(call_key)): 

77 logging.warning(f"File-Deriver {call_key} not found - skipping!") 

78 continue 

79 

80 if call_res := caller(skel, skel["derived"]["files"], params): 

81 assert isinstance(call_res, list), "Old (non-list) return value from deriveFunc" 

82 res_status[full_src_key] = params_hash 

83 for file_name, size, mimetype, custom_data in call_res: 

84 res_files[file_name] = { 

85 "size": size, 

86 "mimetype": mimetype, 

87 "customData": custom_data # TODO: Rename in VIUR4 

88 } 

89 

90 if res_status: # Write updated results back and queue updateRelationsTask 

91 def _merge_derives(patch_skel): 

92 patch_skel["derived"] = {"deriveStatus": {}, "files": {}} | (patch_skel["derived"] or {}) 

93 patch_skel["derived"]["deriveStatus"] = patch_skel["derived"]["deriveStatus"] | res_status 

94 patch_skel["derived"]["files"] = patch_skel["derived"]["files"] | res_files 

95 

96 skel.patch(values=_merge_derives, update_relations=False) 

97 

98 # Queue that update_relations call at least 30 seconds into the future, so that other ensureDerived calls from 

99 # the same FileBone have the chance to finish, otherwise that update_relations Task will call postSavedHandler 

100 # on that FileBone again - re-queueing any ensureDerivedCalls that have not finished yet. 

101 

102 if refresh_key: 

103 skel = skeletonByKind(refresh_key.kind)() 

104 skel.patch(lambda _skel: _skel.refresh(), key=refresh_key, update_relations=False) 

105 

106 update_relations(key, min_change_time=int(time.time() + 1), changed_bones=["derived"], _countdown=30) 

107 

108 

109class FileBone(TreeLeafBone): 

110 r""" 

111 A FileBone is a custom bone class that inherits from the TreeLeafBone class, and is used to store and manage 

112 file references in a ViUR application. 

113 

114 :param format: Hint for the UI how to display a file entry (defaults to it's filename) 

115 :param maxFileSize: 

116 The maximum filesize accepted by this bone in bytes. None means no limit. 

117 This will always be checked against the original file uploaded - not any of it's derivatives. 

118 

119 :param derive: A set of functions used to derive other files from the referenced ones. Used fe. 

120 to create thumbnails / images for srcmaps from hires uploads. If set, must be a dictionary from string 

121 (a key from conf.file_derivations) to the parameters passed to that function. The parameters can be 

122 any type (including None) that can be json-serialized. 

123 

124 .. code-block:: python 

125 

126 # Example 

127 derive = { "thumbnail": [{"width": 111}, {"width": 555, "height": 666}]} 

128 

129 :param validMimeTypes: 

130 A list of Mimetypes that can be selected in this bone (or None for any) Wildcards ("image\/*") are supported. 

131 

132 .. code-block:: python 

133 

134 # Example 

135 validMimeTypes=["application/pdf", "image/*"] 

136 

137 """ 

138 

139 kind = "file" 

140 """The kind of this bone is 'file'""" 

141 

142 type = "relational.tree.leaf.file" 

143 """The type of this bone is 'relational.tree.leaf.file'.""" 

144 

145 DEFAULT_REFKEYS = ( 

146 "derived", 

147 "dlkey", 

148 "height", 

149 "mimetype", 

150 "name", 

151 "public", 

152 "serving_url", 

153 "size", 

154 "width", 

155 ) 

156 """ 

157 Default RefKeys for FileBone. 

158 Use this as extendable reference. 

159 """ 

160 

161 def __init__( 

162 self, 

163 *, 

164 derive: None | dict[str, t.Any] = None, 

165 maxFileSize: None | int = None, 

166 validMimeTypes: None | list[str] = None, 

167 refKeys: t.Optional[t.Iterable[str]] = DEFAULT_REFKEYS, 

168 public: bool = False, 

169 **kwargs 

170 ): 

171 r""" 

172 Initializes a new Filebone. All properties inherited by RelationalBone are supported. 

173 

174 :param format: Hint for the UI how to display a file entry (defaults to it's filename) 

175 :param maxFileSize: The maximum filesize accepted by this bone in bytes. None means no limit. 

176 This will always be checked against the original file uploaded - not any of it's derivatives. 

177 :param derive: A set of functions used to derive other files from the referenced ones. 

178 Used to create thumbnails and images for srcmaps from hires uploads. 

179 If set, must be a dictionary from string (a key from) conf.file_derivations) to the parameters passed to 

180 that function. The parameters can be any type (including None) that can be json-serialized. 

181 

182 .. code-block:: python 

183 

184 # Example 

185 derive = {"thumbnail": [{"width": 111}, {"width": 555, "height": 666}]} 

186 

187 :param validMimeTypes: 

188 A list of Mimetypes that can be selected in this bone (or None for any). 

189 Wildcards `('image\*')` are supported. 

190 

191 .. code-block:: python 

192 

193 #Example 

194 validMimeTypes=["application/pdf", "image/*"] 

195 

196 """ 

197 super().__init__(refKeys=refKeys, **kwargs) 

198 

199 for _required in ("dlkey", "name"): 

200 if _required not in self.refKeys: 

201 raise ValueError(f"FileBone not operable without refKey {_required!r}") 

202 

203 self.derive = derive 

204 self.public = public 

205 self.validMimeTypes = validMimeTypes 

206 self.maxFileSize = maxFileSize 

207 

208 def isInvalid(self, value): 

209 """ 

210 Checks if the provided value is invalid for this bone based on its MIME type and file size. 

211 

212 :param dict value: The value to check for validity. 

213 :returns: None if the value is valid, or an error message if it is invalid. 

214 """ 

215 if self.validMimeTypes: 

216 mimeType = value["dest"]["mimetype"] 

217 for checkMT in self.validMimeTypes: 

218 checkMT = checkMT.lower() 

219 if checkMT == mimeType or checkMT.endswith("*") and mimeType.startswith(checkMT[:-1]): 

220 break 

221 else: 

222 return "Invalid filetype selected" 

223 if self.maxFileSize: 

224 if value["dest"]["size"] > self.maxFileSize: 

225 return "File too large." 

226 

227 if value["dest"]["public"] != self.public: 

228 return f"Only files marked public={self.public!r} are allowed." 

229 

230 return None 

231 

232 def postSavedHandler(self, skel, boneName, key): 

233 """ 

234 Handles post-save processing for the FileBone, including ensuring derived files are built. 

235 

236 :param SkeletonInstance skel: The skeleton instance this bone belongs to. 

237 :param str boneName: The name of the bone. 

238 :param db.Key key: The datastore key of the skeleton. 

239 

240 This method first calls the postSavedHandler of its superclass. Then, it checks if the 

241 derive attribute is set and if there are any values in the skeleton for the given bone. If 

242 so, it handles the creation of derived files based on the provided configuration. 

243 

244 If the values are stored as a dictionary without a "dest" key, it assumes a multi-language 

245 setup and iterates over each language to handle the derived files. Otherwise, it handles 

246 the derived files directly. 

247 """ 

248 super().postSavedHandler(skel, boneName, key) 

249 if ( 

250 current.request.get().is_deferred 

251 and "derived" in (current.request_data.get().get("__update_relations_bones") or ()) 

252 ): 

253 return 

254 

255 from viur.core.skeleton import RelSkel, Skeleton 

256 

257 if issubclass(skel.skeletonCls, Skeleton): 

258 prefix = f"{skel.kindName}_{boneName}" 

259 elif issubclass(skel.skeletonCls, RelSkel): # RelSkel is just a container and has no kindname 

260 prefix = f"{skel.skeletonCls.__name__}_{boneName}" 

261 else: 

262 raise NotImplementedError(f"Cannot handle {skel.skeletonCls=}") 

263 

264 def handleDerives(values): 

265 if isinstance(values, dict): 

266 values = [values] 

267 for val in (values or ()): # Ensure derives getting build for each file referenced in this relation 

268 ensureDerived(val["dest"]["key"], prefix, self.derive, key) 

269 

270 values = skel[boneName] 

271 if self.derive and values: 

272 if isinstance(values, dict) and "dest" not in values: # multi lang 

273 for lang in values: 

274 handleDerives(values[lang]) 

275 else: 

276 handleDerives(values) 

277 

278 def getReferencedBlobs(self, skel: 'viur.core.skeleton.SkeletonInstance', name: str) -> set[str]: 

279 r""" 

280 Retrieves the referenced blobs in the FileBone. 

281 

282 :param SkeletonInstance skel: The skeleton instance this bone belongs to. 

283 :param str name: The name of the bone. 

284 :return: A set of download keys for the referenced blobs. 

285 :rtype: Set[str] 

286 

287 This method iterates over the bone values for the given skeleton and bone name. It skips 

288 values that are None. For each non-None value, it adds the download key of the referenced 

289 blob to a set. Finally, it returns the set of unique download keys for the referenced blobs. 

290 """ 

291 result = set() 

292 for idx, lang, value in self.iter_bone_value(skel, name): 

293 if value is None: 

294 continue 

295 result.add(value["dest"]["dlkey"]) 

296 return result 

297 

298 def refresh(self, skel, boneName): 

299 r""" 

300 Refreshes the FileBone by recreating file entries if needed and importing blobs from ViUR 2. 

301 

302 :param SkeletonInstance skel: The skeleton instance this bone belongs to. 

303 :param str boneName: The name of the bone. 

304 

305 This method defines an inner function, recreateFileEntryIfNeeded(val), which is responsible 

306 for recreating the weak file entry referenced by the relation in val if it doesn't exist 

307 (e.g., if it was deleted by ViUR 2). It initializes a new skeleton for the "file" kind and 

308 checks if the file object already exists. If not, it recreates the file entry with the 

309 appropriate properties and saves it to the database. 

310 

311 The main part of the refresh method calls the superclass's refresh method and checks if the 

312 configuration contains a ViUR 2 import blob source. If it does, it iterates through the file 

313 references in the bone value, imports the blobs from ViUR 2, and recreates the file entries if 

314 needed using the inner function. 

315 """ 

316 super().refresh(skel, boneName) 

317 

318 for _, _, value in self.iter_bone_value(skel, boneName): 

319 # Patch any empty serving_url when public file 

320 if ( 

321 value 

322 and (value := value["dest"]) 

323 and value["public"] 

324 and value["mimetype"] 

325 and value["mimetype"].startswith("image/") 

326 and not value["serving_url"] 

327 ): 

328 logging.info(f"Patching public image with empty serving_url {value['key']!r} ({value['name']!r})") 

329 try: 

330 file_skel = value.read() 

331 except ValueError: 

332 continue 

333 

334 file_skel.patch(lambda skel: skel.refresh(), update_relations=False) 

335 value["serving_url"] = file_skel["serving_url"] 

336 

337 # FIXME: REMOVE THIS WITH VIUR4 

338 if conf.viur2import_blobsource: 

339 from viur.core.modules.file import importBlobFromViur2 

340 from viur.core.skeleton import skeletonByKind 

341 

342 def recreateFileEntryIfNeeded(val): 

343 # Recreate the (weak) filenetry referenced by the relation *val*. (ViUR2 might have deleted them) 

344 skel = skeletonByKind("file")() 

345 if skel.read(val["key"]): # This file-object exist, no need to recreate it 

346 return 

347 skel["key"] = val["key"] 

348 skel["name"] = val["name"] 

349 skel["mimetype"] = val["mimetype"] 

350 skel["dlkey"] = val["dlkey"] 

351 skel["size"] = val["size"] 

352 skel["width"] = val["width"] 

353 skel["height"] = val["height"] 

354 skel["weak"] = True 

355 skel["pending"] = False 

356 skel.write() 

357 

358 # Just ensure the file get's imported as it may not have an file entry 

359 val = skel[boneName] 

360 if isinstance(val, list): 

361 for x in val: 

362 importBlobFromViur2(x["dest"]["dlkey"], x["dest"]["name"]) 

363 recreateFileEntryIfNeeded(x["dest"]) 

364 elif isinstance(val, dict): 

365 if not "dest" in val: 

366 return 

367 importBlobFromViur2(val["dest"]["dlkey"], val["dest"]["name"]) 

368 recreateFileEntryIfNeeded(val["dest"]) 

369 

370 def structure(self) -> dict: 

371 return super().structure() | { 

372 "valid_mime_types": self.validMimeTypes, 

373 "public": self.public, 

374 } 

375 

376 def _atomic_dump(self, value) -> dict | None: 

377 value = super()._atomic_dump(value) 

378 if value is not None: 

379 # VIUR4: Rename "downloadUrl" into "download_url" 

380 value["dest"]["downloadUrl"] = conf.main_app.file.create_download_url( 

381 value["dest"]["dlkey"], 

382 value["dest"]["name"], 

383 derived=False, 

384 expires=conf.render_json_download_url_expiration 

385 ) 

386 

387 return value