Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/modules/file.py: 27%
792 statements
« prev ^ index » next coverage.py v7.16.1, created at 2026-09-25 15:02 +0000
1import base64
2import datetime
3import hashlib
4import hmac
5import html
6import io
7import json
8import logging
9import re
10import string
11import typing as t
12import warnings
13from collections import namedtuple
14from urllib.parse import parse_qs, quote as urlquote, unquote as urlunquote, urlencode, urlsplit
15from urllib.request import urlopen
17import PIL
18import PIL.ImageCms
19import google.auth
20import requests
21from PIL import Image
22from google.appengine.api import blobstore, images
23from google.cloud import storage
24from google.oauth2.service_account import Credentials as ServiceAccountCredentials
26from viur.core import conf, current, db, errors, utils, i18n
27from viur.core.bones import BaseBone, BooleanBone, JsonBone, KeyBone, NumericBone, StringBone
29from viur.core.decorators import *
30from viur.core.prototypes.tree import SkelType, Tree, TreeSkel
31from viur.core.skeleton import SkeletonInstance, skeletonByKind
32from viur.core.tasks import CallDeferred, DeleteEntitiesIter, PeriodicTask
34# Globals for connectivity
36VALID_FILENAME_REGEX = re.compile(
37 # || MAY NOT BE THE NAME | MADE OF SPECIAL CHARS | SPECIAL CHARS + `. `|`
38 r"^(?!^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])$)[^\x00-\x1F<>:\"\/\\|?*]*[^\x00-\x1F<>:\"\/\\|?*. ]$",
39 re.IGNORECASE
40)
42_CREDENTIALS, _PROJECT_ID = google.auth.default()
43GOOGLE_STORAGE_CLIENT = storage.Client(_PROJECT_ID, _CREDENTIALS)
45PRIVATE_BUCKET_NAME = f"""{_PROJECT_ID}.appspot.com"""
46PUBLIC_BUCKET_NAME = f"""public-dot-{_PROJECT_ID}"""
47PUBLIC_DLKEY_SUFFIX = "_pub"
49_private_bucket = GOOGLE_STORAGE_CLIENT.lookup_bucket(PRIVATE_BUCKET_NAME)
50_public_bucket = None
52# FilePath is a descriptor for ViUR file components
53FilePath = namedtuple("FilePath", ("dlkey", "is_derived", "filename"))
56def importBlobFromViur2(dlKey, fileName):
57 bucket = conf.main_app.file.get_bucket(dlKey)
59 if not conf.viur2import_blobsource:
60 return False
61 existingImport = db.get(db.Key("viur-viur2-blobimport", dlKey))
62 if existingImport:
63 if existingImport["success"]:
64 return existingImport["dlurl"]
65 return False
66 if conf.viur2import_blobsource["infoURL"]:
67 try:
68 importDataReq = urlopen(conf.viur2import_blobsource["infoURL"] + dlKey)
69 except Exception as e:
70 marker = db.Entity(db.Key("viur-viur2-blobimport", dlKey))
71 marker["success"] = False
72 marker["error"] = "Failed URL-FETCH 1"
73 db.put(marker)
74 return False
75 if importDataReq.status != 200:
76 marker = db.Entity(db.Key("viur-viur2-blobimport", dlKey))
77 marker["success"] = False
78 marker["error"] = "Failed URL-FETCH 2"
79 db.put(marker)
80 return False
81 importData = json.loads(importDataReq.read())
82 oldBlobName = conf.viur2import_blobsource["gsdir"] + "/" + importData["key"]
83 srcBlob = storage.Blob(bucket=bucket,
84 name=conf.viur2import_blobsource["gsdir"] + "/" + importData["key"])
85 else:
86 oldBlobName = conf.viur2import_blobsource["gsdir"] + "/" + dlKey
87 srcBlob = storage.Blob(bucket=bucket, name=conf.viur2import_blobsource["gsdir"] + "/" + dlKey)
88 if not srcBlob.exists():
89 marker = db.Entity(db.Key("viur-viur2-blobimport", dlKey))
90 marker["success"] = False
91 marker["error"] = "Local SRC-Blob missing"
92 marker["oldBlobName"] = oldBlobName
93 db.put(marker)
94 return False
95 bucket.rename_blob(srcBlob, f"{dlKey}/source/{fileName}")
96 marker = db.Entity(db.Key("viur-viur2-blobimport", dlKey))
97 marker["success"] = True
98 marker["old_src_key"] = dlKey
99 marker["old_src_name"] = fileName
100 marker["dlurl"] = conf.main_app.file.create_download_url(dlKey, fileName, False, None)
101 db.put(marker)
102 return marker["dlurl"]
105def thumbnailer(fileSkel, existingFiles, params):
106 file_name = html.unescape(fileSkel["name"])
107 bucket = conf.main_app.file.get_bucket(fileSkel["dlkey"])
109 blob = bucket.get_blob(f"""{fileSkel["dlkey"]}/source/{file_name}""")
110 if not blob:
111 logging.warning(f"""Blob {fileSkel["dlkey"]}/source/{file_name} is missing from cloud storage!""")
112 return
114 source = io.BytesIO()
115 blob.download_to_file(source)
117 result = []
119 for info in params:
120 # Read the image into PIL
121 try:
122 source.seek(0)
123 img = PIL.Image.open(source)
124 except PIL.Image.UnidentifiedImageError: # Can't load this image; so there's no need to try other resolutions
125 break
127 if icc_profile := img.info.get("icc_profile"):
128 # JPEGs might be encoded with a non-standard color-profile; we need to compensate for this if we convert
129 # to WEBp as we'll loose this color-profile information
130 f = io.BytesIO(icc_profile)
131 src_profile = PIL.ImageCms.ImageCmsProfile(f)
132 dst_profile = PIL.ImageCms.createProfile("sRGB")
133 try:
134 img = PIL.ImageCms.profileToProfile(
135 img,
136 inputProfile=src_profile,
137 outputProfile=dst_profile,
138 outputMode="RGBA" if img.has_transparency_data else "RGB")
139 except Exception as e:
140 logging.debug(f"{info=}")
141 logging.exception(e)
142 continue
144 file_extension = info.get("fileExtension", "webp")
145 mimetype = info.get("mimeType", "image/webp")
147 if "width" in info and "height" in info:
148 width = info["width"]
149 height = info["height"]
150 target_filename = f"thumbnail-{width}-{height}.{file_extension}"
152 elif "width" in info:
153 width = info["width"]
154 height = int((float(img.size[1]) * float(width / float(img.size[0]))))
155 target_filename = f"thumbnail-w{width}.{file_extension}"
157 else: # No default fallback - ignore
158 continue
160 # Create resized version of the source
161 target = io.BytesIO()
163 try:
164 img = img.resize((width, height), PIL.Image.LANCZOS)
165 except ValueError as e:
166 # Usually happens to some files, like TIFF-images.
167 logging.debug(f"{info=}")
168 logging.exception(e)
169 break
171 img.save(target, file_extension)
173 # Safe derived target file
174 target_size = target.tell()
175 target.seek(0)
176 target_blob = bucket.blob(f"""{fileSkel["dlkey"]}/derived/{target_filename}""")
177 target_blob.upload_from_file(target, content_type=mimetype)
179 result.append(
180 (target_filename, target_size, mimetype, {"mimetype": mimetype, "width": width, "height": height})
181 )
183 return result
186def cloudfunction_thumbnailer(fileSkel, existingFiles, params):
187 """External Thumbnailer for images.
189 The corresponding cloudfunction can be found here .
190 https://github.com/viur-framework/viur-cloudfunctions/tree/main/thumbnailer
192 You can use it like so:
193 main.py:
195 .. code-block:: python
197 from viur.core.modules.file import cloudfunction_thumbnailer
199 conf.file_thumbnailer_url = "https://xxxxx.cloudfunctions.net/imagerenderer"
200 conf.file_derivations = {"thumbnail": cloudfunction_thumbnailer}
202 conf.derives_pdf = {
203 "thumbnail": [{"width": 1920,"sites":"1,2"}]
204 }
206 skeletons/xxx.py:
207 .. code-block:: python
209 test = FileBone(derive=conf.derives_pdf)
210 """
212 if not conf.file_thumbnailer_url:
213 raise ValueError("conf.file_thumbnailer_url is not set")
215 bucket = conf.main_app.file.get_bucket(fileSkel["dlkey"])
217 def getsignedurl():
218 if conf.instance.is_dev_server:
219 signedUrl = conf.main_app.file.create_download_url(fileSkel["dlkey"], fileSkel["name"])
220 else:
221 path = f"""{fileSkel["dlkey"]}/source/{file_name}"""
222 if not (blob := bucket.get_blob(path)):
223 logging.warning(f"Blob {path} is missing from cloud storage!")
224 return None
225 authRequest = google.auth.transport.requests.Request()
226 expiresAt = datetime.datetime.now() + datetime.timedelta(seconds=60)
227 signing_credentials = google.auth.compute_engine.IDTokenCredentials(authRequest, "")
228 content_disposition = utils.build_content_disposition_header(fileSkel["name"])
229 signedUrl = blob.generate_signed_url(
230 expiresAt,
231 credentials=signing_credentials,
232 response_disposition=content_disposition,
233 version="v4")
234 return signedUrl
236 def make_request():
237 headers = {"Content-Type": "application/json"}
238 data_str = base64.b64encode(json.dumps(dataDict).encode("UTF-8"))
239 sig = conf.main_app.file.hmac_sign(data_str)
240 datadump = json.dumps({"dataStr": data_str.decode('ASCII'), "sign": sig})
241 resp = requests.post(conf.file_thumbnailer_url, data=datadump, headers=headers, allow_redirects=False)
242 if resp.status_code != 200: # Error Handling
243 match resp.status_code:
244 case 302:
245 # The problem is Google resposen 302 to an auth Site when the cloudfunction was not found
246 # https://cloud.google.com/functions/docs/troubleshooting#login
247 logging.error("Cloudfunction not found")
248 case 404:
249 logging.error("Cloudfunction not found")
250 case 403:
251 logging.error("No permission for the Cloudfunction")
252 case _:
253 logging.error(
254 f"cloudfunction_thumbnailer failed with code: {resp.status_code} and data: {resp.content}")
255 return
257 try:
258 response_data = resp.json()
259 except Exception as e:
260 logging.error(f"response could not be converted in json failed with: {e=}")
261 return
262 if "error" in response_data:
263 logging.error(f"cloudfunction_thumbnailer failed with: {response_data.get('error')}")
264 return
266 return response_data
268 file_name = html.unescape(fileSkel["name"])
270 if not (url := getsignedurl()):
271 return
272 dataDict = {
273 "url": url,
274 "name": fileSkel["name"],
275 "params": params,
276 "minetype": fileSkel["mimetype"],
277 "baseUrl": current.request.get().request.host_url.lower(),
278 "targetKey": fileSkel["dlkey"],
279 "nameOnly": True
280 }
281 if not (derivedData := make_request()):
282 return
284 uploadUrls = {}
285 for data in derivedData["values"]:
286 if not conf.main_app.file.is_valid_filename(data["name"]):
287 raise errors.UnprocessableEntity(f"""Invalid derived filename {data["name"]!r} provided""")
289 fileName = urlquote(data["name"])
290 blob = bucket.blob(f"""{fileSkel["dlkey"]}/derived/{fileName}""")
291 uploadUrls[fileSkel["dlkey"] + fileName] = blob.create_resumable_upload_session(timeout=60,
292 content_type=data["mimeType"])
294 if not (url := getsignedurl()):
295 return
297 dataDict["url"] = url
298 dataDict["nameOnly"] = False
299 dataDict["uploadUrls"] = uploadUrls
301 if not (derivedData := make_request()):
302 return
303 reslist = []
304 try:
305 for derived in derivedData["values"]:
306 for key, value in derived.items():
307 reslist.append((key, value["size"], value["mimetype"], value["customData"]))
309 except Exception as e:
310 logging.error(f"cloudfunction_thumbnailer failed with: {e=}")
311 return reslist
314class DownloadUrlBone(BaseBone):
315 """
316 This bone is used to inject a freshly signed download url into a FileSkel.
317 """
319 def unserialize(self, skel, name):
320 if "dlkey" in skel.dbEntity and "name" in skel.dbEntity:
321 skel.accessedValues[name] = conf.main_app.file.create_download_url(
322 skel["dlkey"], skel["name"], expires=conf.render_json_download_url_expiration
323 )
324 return True
326 return False
329class FileLeafSkel(TreeSkel):
330 """
331 Default file leaf skeleton.
332 """
333 kindName = "file"
335 name = StringBone(
336 descr="Filename",
337 caseSensitive=False,
338 searchable=True,
339 vfunc=lambda val: None if File.is_valid_filename(val) else "Invalid filename provided",
340 )
342 alt = StringBone(
343 descr=i18n.translate(
344 "viur.core.image.alt",
345 defaultText="Alternative description",
346 ),
347 searchable=True,
348 languages=conf.i18n.available_languages,
349 )
351 size = NumericBone(
352 descr="Filesize in Bytes",
353 readOnly=True,
354 searchable=True,
355 )
357 dlkey = StringBone(
358 descr="Download-Key",
359 readOnly=True,
360 )
362 mimetype = StringBone(
363 descr="MIME-Type",
364 readOnly=True,
365 )
367 weak = BooleanBone(
368 descr="Weak reference",
369 readOnly=True,
370 visible=False,
371 )
373 pending = BooleanBone(
374 descr="Pending upload",
375 readOnly=True,
376 visible=False,
377 defaultValue=False,
378 )
380 width = NumericBone(
381 descr="Width",
382 readOnly=True,
383 searchable=True,
384 )
386 height = NumericBone(
387 descr="Height",
388 readOnly=True,
389 searchable=True,
390 )
392 downloadUrl = DownloadUrlBone(
393 descr="Download-URL",
394 readOnly=True,
395 visible=False,
396 )
398 derived = JsonBone(
399 descr="Derived Files",
400 readOnly=True,
401 visible=False,
402 )
404 pendingparententry = KeyBone(
405 descr="Pending key Reference",
406 readOnly=True,
407 visible=False,
408 )
410 crc32c_checksum = StringBone(
411 descr="CRC32C checksum",
412 readOnly=True,
413 )
415 md5_checksum = StringBone(
416 descr="MD5 checksum",
417 readOnly=True,
418 )
420 public = BooleanBone(
421 descr="Public File",
422 readOnly=True,
423 defaultValue=False,
424 )
426 serving_url = StringBone(
427 descr="Serving-URL",
428 readOnly=True,
429 params={
430 "tooltip": "The 'serving_url' is only available in public file repositories.",
431 }
432 )
434 @classmethod
435 def _inject_serving_url(cls, skel: SkeletonInstance) -> None:
436 """Inject the serving url for public image files into a FileSkel"""
437 if (
438 skel["public"]
439 and skel["mimetype"]
440 and skel["mimetype"].startswith("image/")
441 and not skel["serving_url"]
442 ):
443 bucket = File.get_bucket(skel["dlkey"])
444 filename = f"/gs/{bucket.name}/{skel['dlkey']}/source/{utils.string.unescape(skel['name'])}"
446 # Trying this on local development server will raise a
447 # `google.appengine.runtime.apiproxy_errors.RPCFailedError`
448 if conf.instance.is_dev_server:
449 logging.warning(f"Can't inject serving_url for {filename!r} on local development server")
450 return
452 try:
453 skel["serving_url"] = images.get_serving_url(None, secure_url=True, filename=filename)
455 except Exception as e:
456 logging.warning(f"Failed to create serving_url for {filename!r} with exception {e!r}")
457 logging.exception(e)
459 def preProcessBlobLocks(self, locks):
460 """
461 Ensure that our dlkey is locked even if we don't have a filebone here
462 """
463 if not self["weak"] and self["dlkey"]:
464 locks.add(self["dlkey"])
465 return locks
467 @classmethod
468 def refresh(cls, skel):
469 super().refresh(skel)
470 if conf.viur2import_blobsource:
471 importData = importBlobFromViur2(skel["dlkey"], skel["name"])
472 if importData:
473 if not skel["downloadUrl"]:
474 skel["downloadUrl"] = importData
475 skel["pendingparententry"] = None
477 cls._inject_serving_url(skel)
479 @classmethod
480 def write(cls, skel, **kwargs):
481 cls._inject_serving_url(skel)
482 return super().write(skel, **kwargs)
485class FileNodeSkel(TreeSkel):
486 """
487 Default file node skeleton.
488 """
489 kindName = "file_rootNode" # FIXME: VIUR4, don't use "_rootNode" kindname
491 name = StringBone(
492 descr="Name",
493 required=True,
494 searchable=True
495 )
497 rootNode = BooleanBone(
498 descr="Is RootNode",
499 defaultValue=False,
500 readOnly=True,
501 visible=False,
502 )
504 public = BooleanBone(
505 descr="Is public?",
506 defaultValue=False,
507 readOnly=True,
508 visible=False,
509 )
511 viurCurrentSeoKeys = None
514class File(Tree):
515 PENDING_POSTFIX = " (pending)"
516 DOWNLOAD_URL_PREFIX = "/file/download/"
517 INTERNAL_SERVING_URL_PREFIX = "/file/serve/"
518 MAX_FILENAME_LEN = 256
519 IMAGE_META_MAX_SIZE: t.Final[int] = 10 * 1024 ** 2
520 """Maximum size of image files that should be analysed in :meth:`set_image_meta`.
521 Default: 10 MiB"""
523 leafSkelCls = FileLeafSkel
524 nodeSkelCls = FileNodeSkel
526 handler = "tree.simple.file"
527 adminInfo = {
528 "icon": "folder-fill",
529 "handler": handler, # fixme: Use static handler; Remove with VIUR4!
530 }
532 roles = {
533 "*": "view",
534 "editor": ("add", "edit"),
535 "admin": "*",
536 }
538 default_order = "name"
540 # Helper functions currently resist here
542 @staticmethod
543 def get_bucket(dlkey: str) -> google.cloud.storage.bucket.Bucket:
544 """
545 Retrieves a Google Cloud Storage bucket for the given dlkey.
546 """
547 global _public_bucket
548 if dlkey and dlkey.endswith(PUBLIC_DLKEY_SUFFIX):
549 if _public_bucket or (_public_bucket := GOOGLE_STORAGE_CLIENT.lookup_bucket(PUBLIC_BUCKET_NAME)):
550 return _public_bucket
552 raise ValueError(
553 f"""The bucket '{PUBLIC_BUCKET_NAME}' does not exist! Please create it with ACL access."""
554 )
556 return _private_bucket
558 @classmethod
559 def is_valid_filename(cls, filename: str) -> bool:
560 """
561 Verifies a valid filename.
563 The filename should be valid on Linux, Mac OS and Windows.
564 It should not be longer than MAX_FILENAME_LEN chars.
566 Rule set: https://stackoverflow.com/a/31976060/3749896
567 Regex test: https://regex101.com/r/iBYpoC/1
568 """
569 if not filename.strip(): 569 ↛ 570line 569 didn't jump to line 570 because the condition on line 569 was never true
570 return False
572 if len(filename) > cls.MAX_FILENAME_LEN: 572 ↛ 573line 572 didn't jump to line 573 because the condition on line 572 was never true
573 return False
575 return bool(re.match(VALID_FILENAME_REGEX, filename))
577 @staticmethod
578 def hmac_sign(data: t.Any) -> str:
579 assert conf.file_hmac_key is not None, "No hmac-key set!"
580 if not isinstance(data, bytes): 580 ↛ 581line 580 didn't jump to line 581 because the condition on line 580 was never true
581 data = str(data).encode("UTF-8")
582 return hmac.new(conf.file_hmac_key, msg=data, digestmod=hashlib.sha3_384).hexdigest()
584 @classmethod
585 def hmac_verify(cls, data: t.Any, signature: str) -> bool:
586 try:
587 return hmac.compare_digest(cls.hmac_sign(data.encode("ASCII")), signature)
588 except (TypeError, UnicodeEncodeError):
589 return False
591 @classmethod
592 def create_internal_serving_url(
593 cls,
594 serving_url: str,
595 size: int = 0,
596 filename: str = "",
597 options: str = "",
598 download: bool = False
599 ) -> str:
600 """
601 Helper function to generate an internal serving url (endpoint: /file/serve) from a Google serving url.
603 This is needed to hide requests to Google as they are internally be routed, and can be the result of a
604 legal requirement like GDPR.
606 :param serving_url: Is the original serving URL as generated from FileLeafSkel._inject_serving_url()
607 :param size: Optional size setting
608 :param filename: Optonal filename setting
609 :param options: Additional options parameter-pass through to /file/serve
610 :param download: Download parameter-pass through to /file/serve
611 """
613 # Split a serving URL into its components, used by serve function.
614 res = re.match(
615 r"^https:\/\/(.*?)\.googleusercontent\.com\/(.*?)$",
616 serving_url
617 )
619 if not res:
620 raise ValueError(f"Invalid {serving_url=!r} provided")
622 # Create internal serving URL
623 serving_url = cls.INTERNAL_SERVING_URL_PREFIX + "/".join(res.groups())
625 # Append additional parameters
626 if params := {
627 k: v for k, v in {
628 "download": download,
629 "filename": filename,
630 "options": options,
631 "size": size,
632 }.items() if v
633 }:
634 serving_url += f"?{urlencode(params)}"
636 return serving_url
638 @classmethod
639 def create_download_url(
640 cls,
641 dlkey: str,
642 filename: str,
643 derived: bool = False,
644 expires: t.Optional[datetime.timedelta | int] = datetime.timedelta(hours=1),
645 download_filename: t.Optional[str] = None
646 ) -> str:
647 """
648 Utility function that creates a signed download-url for the given folder/filename combination
650 :param folder: The GCS-Folder (= the download-key) for that file
651 :param filename: The name of the file. Either the original filename or the name of a derived file.
652 :param derived: True, if it points to a derived file, False if it points to the original uploaded file
653 :param expires:
654 None if the file is supposed to be public (which causes it to be cached on the google ede caches),
655 otherwise a datetime.timedelta of how long that link should be valid
656 :param download_filename: If set, browser is enforced to download this blob with the given alternate
657 filename
658 :return: The signed download-url relative to the current domain (eg /download/...)
659 """
660 if isinstance(expires, int): 660 ↛ 661line 660 didn't jump to line 661 because the condition on line 660 was never true
661 expires = datetime.timedelta(minutes=expires)
663 filename = html.unescape(filename)
664 filepath = f"""{dlkey}/{"derived" if derived else "source"}/{filename}"""
666 if download_filename:
667 if not cls.is_valid_filename(download_filename): 667 ↛ 668line 667 didn't jump to line 668 because the condition on line 667 was never true
668 raise errors.UnprocessableEntity(f"Invalid download_filename {download_filename!r} provided")
670 download_filename = urlquote(download_filename)
672 expires = (datetime.datetime.now() + expires).strftime("%Y%m%d%H%M") if expires else 0
674 data = base64.urlsafe_b64encode(f"""{filepath}\0{expires}\0{download_filename or ""}""".encode("UTF-8"))
675 sig = cls.hmac_sign(data)
677 return f"""{cls.DOWNLOAD_URL_PREFIX}{data.decode("ASCII")}?sig={sig}"""
679 @classmethod
680 def parse_download_url(cls, url) -> t.Optional[FilePath]:
681 """
682 Parses a file download URL in the format `/file/download/xxxx?sig=yyyy` into its FilePath.
684 The URL may be absolute, because admin frontends store it including scheme and host, and
685 it may carry an additional path segment after the payload: :meth:`download` accepts the
686 download file name that way, and a bare trailing slash occurs as well.
688 If the URL cannot be parsed, the function returns None.
690 :param url: The file download URL to be parsed.
691 :return: A FilePath on success, None otherwise.
692 """
693 parsed_url = urlsplit(url)
695 if not parsed_url.path.startswith(cls.DOWNLOAD_URL_PREFIX) or not parsed_url.query:
696 return None
698 # Strip "/file/download/"; the payload is urlsafe-base64 and therefore never contains a
699 # slash, so anything after the first one is the optional download file name.
700 data = parsed_url.path.removeprefix(cls.DOWNLOAD_URL_PREFIX).split("/", 1)[0]
701 sig = parse_qs(parsed_url.query).get("sig", [""])[0]
703 if not cls.hmac_verify(data, sig): 703 ↛ 705line 703 didn't jump to line 705 because the condition on line 703 was never true
704 # Invalid signature
705 return None
707 # Split the blobKey into the individual fields it should contain
708 data = base64.urlsafe_b64decode(data).decode("UTF-8")
710 match data.count("\0"):
711 case 2: 711 ↛ 713line 711 didn't jump to line 713 because the pattern on line 711 always matched
712 dlpath, valid_until, _ = data.split("\0")
713 case 1:
714 # It's the old format, without an downloadFileName
715 dlpath, valid_until = data.split("\0")
716 case _:
717 # Invalid path
718 return None
720 if valid_until != "0" and datetime.datetime.strptime(valid_until, "%Y%m%d%H%M") < datetime.datetime.now():
721 # Signature expired
722 return None
724 if dlpath.count("/") != 2: 724 ↛ 726line 724 didn't jump to line 726 because the condition on line 724 was never true
725 # Invalid path
726 return None
728 dlkey, derived, filename = dlpath.split("/")
729 return FilePath(dlkey, derived != "source", filename)
731 @classmethod
732 def create_src_set(
733 cls,
734 file: t.Union["SkeletonInstance", dict, str],
735 expires: t.Optional[datetime.timedelta | int] = datetime.timedelta(hours=1),
736 width: t.Optional[int] = None,
737 height: t.Optional[int] = None,
738 language: t.Optional[str] = None,
739 ) -> str:
740 """
741 Generates a string suitable for use as the srcset tag in html. This functionality provides the browser
742 with a list of images in different sizes and allows it to choose the smallest file that will fill it's
743 viewport without upscaling.
745 :param file: The file skeleton (or if multiple=True a single value from it) to generate the srcset.
746 :param expires:
747 None if the file is supposed to be public (which causes it to be cached on the google edecaches),
748 otherwise it's lifetime in seconds
749 :param width:
750 A list of widths that should be included in the srcset.
751 If a given width is not available, it will be skipped.
752 :param height: A list of heights that should be included in the srcset. If a given height is not available,
753 it will be skipped.
754 :param language: Language overwrite if file has multiple languages, and we want to explicitly specify one
755 :return: The srctag generated or an empty string if a invalid file object was supplied
756 """
757 if not width and not height: 757 ↛ 758line 757 didn't jump to line 758 because the condition on line 757 was never true
758 logging.error("Neither width or height supplied")
759 return ""
761 if isinstance(file, str): 761 ↛ 762line 761 didn't jump to line 762 because the condition on line 761 was never true
762 file = db.Query("file").filter("dlkey =", file).order(
763 db.QueryOrder("creationdate")).getEntry()
765 if not file: 765 ↛ 766line 765 didn't jump to line 766 because the condition on line 765 was never true
766 return ""
768 if isinstance(file, i18n.LanguageWrapper): 768 ↛ 773line 768 didn't jump to line 773 because the condition on line 768 was always true
769 language = language or current.language.get()
770 if not language or not (file := file.get(language)):
771 return ""
773 if "dlkey" not in file and "dest" in file: 773 ↛ 774line 773 didn't jump to line 774 because the condition on line 773 was never true
774 file = file["dest"]
776 from viur.core.skeleton import SkeletonInstance # avoid circular imports
778 if not ( 778 ↛ 783line 778 didn't jump to line 783 because the condition on line 778 was never true
779 isinstance(file, (SkeletonInstance, dict))
780 and "dlkey" in file
781 and "derived" in file
782 ):
783 logging.error("Invalid file supplied")
784 return ""
786 if not isinstance(file["derived"], dict): 786 ↛ 787line 786 didn't jump to line 787 because the condition on line 786 was never true
787 logging.error("No derives available")
788 return ""
790 src_set = []
791 for filename, derivate in file["derived"]["files"].items():
792 customData = derivate.get("customData", {})
794 if width and customData.get("width") in width: 794 ↛ 799line 794 didn't jump to line 799 because the condition on line 794 was always true
795 src_set.append(
796 f"""{cls.create_download_url(file["dlkey"], filename, True, expires)} {customData["width"]}w"""
797 )
799 if height and customData.get("height") in height: 799 ↛ 800line 799 didn't jump to line 800 because the condition on line 799 was never true
800 src_set.append(
801 f"""{cls.create_download_url(file["dlkey"], filename, True, expires)} {customData["height"]}h"""
802 )
804 return ", ".join(src_set)
806 def write(
807 self,
808 filename: str,
809 content: t.Any,
810 mimetype: str = "text/plain",
811 *,
812 width: int = None,
813 height: int = None,
814 public: bool = False,
815 rootnode: t.Optional[db.Key] = None,
816 folder: t.Iterable[str] | str = (),
817 ) -> db.Key:
818 """
819 Write a file from any bytes-like object into the file module.
821 If *folder* and *rootnode* are both set, the file is added to the repository in that folder.
822 If only *folder* is set, the file is added to the default repository in that folder.
823 If only *rootnode* is set, the file is added to that repository in the root folder.
825 If both are not set, the file is added without a path or repository as a weak file.
826 It will not be visible in admin in this case.
828 :param filename: Filename to be written.
829 :param content: The file content to be written, as bytes-like object.
830 :param mimetype: The file's mimetype.
831 :param width: Optional width information for the file.
832 :param height: Optional height information for the file.
833 :param public: True if the file should be publicly accessible.
834 :param rootnode: Optional root-node of the repository to add the file to
835 :param folder: Optional folder the file should be written into.
837 :return: Returns the key of the file object written. This can be associated e.g. with a FileBone.
838 """
839 # logging.info(f"{filename=} {mimetype=} {width=} {height=} {public=}")
840 if not self.is_valid_filename(filename): 840 ↛ 841line 840 didn't jump to line 841 because the condition on line 840 was never true
841 raise ValueError(f"{filename=} is invalid")
843 # Folder mode?
844 if folder:
845 # Validate correct folder naming
846 if isinstance(folder, str): 846 ↛ 849line 846 didn't jump to line 849 because the condition on line 846 was always true
847 folder = folder, # make it a tuple
849 for foldername in folder:
850 if not self.is_valid_filename(foldername): 850 ↛ 851line 850 didn't jump to line 851 because the condition on line 850 was never true
851 raise ValueError(f"{foldername=} is invalid")
853 # When in folder-mode, a rootnode must exist!
854 if rootnode is None: 854 ↛ 857line 854 didn't jump to line 857 because the condition on line 854 was always true
855 rootnode = self.ensureOwnModuleRootNode()
857 parentrepokey = rootnode.key
858 parentfolderkey = rootnode.key
860 for foldername in folder:
861 query = self.addSkel("node").all()
862 query.filter("parentrepo", parentrepokey)
863 query.filter("parententry", parentfolderkey)
864 query.filter("name", foldername)
866 if folder_skel := query.getSkel(): 866 ↛ 871line 866 didn't jump to line 871 because the condition on line 866 was always true
867 # Skip existing folder
868 parentfolderkey = folder_skel["key"]
869 else:
870 # Create new folder
871 folder_skel = self.addSkel("node")
873 folder_skel["name"] = foldername
874 folder_skel["parentrepo"] = parentrepokey
875 folder_skel["parententry"] = parentfolderkey
876 folder_skel.write()
878 parentfolderkey = folder_skel["key"]
880 else:
881 parentrepokey = None
882 parentfolderkey = None
884 # Write the file
885 dl_key = utils.string.random()
887 if public: 887 ↛ 888line 887 didn't jump to line 888 because the condition on line 887 was never true
888 dl_key += PUBLIC_DLKEY_SUFFIX # mark file as public
890 bucket = self.get_bucket(dl_key)
892 blob = bucket.blob(f"{dl_key}/source/{filename}")
893 blob.upload_from_file(io.BytesIO(content), content_type=mimetype)
895 fileskel = self.addSkel("leaf")
897 fileskel["parentrepo"] = parentrepokey
898 fileskel["parententry"] = parentfolderkey
899 fileskel["name"] = filename
900 fileskel["size"] = blob.size
901 fileskel["mimetype"] = mimetype
902 fileskel["dlkey"] = dl_key
903 fileskel["weak"] = not parentrepokey
904 fileskel["public"] = public
905 fileskel["width"] = width
906 fileskel["height"] = height
907 fileskel["crc32c_checksum"] = base64.b64decode(blob.crc32c).hex()
908 fileskel["md5_checksum"] = base64.b64decode(blob.md5_hash).hex()
909 fileskel["pending"] = False
911 return fileskel.write()["key"]
913 def read(
914 self,
915 key: db.KeyType | None = None,
916 path: str | None = None,
917 ) -> tuple[io.BytesIO, str]:
918 """
919 Read a file from the Cloud Storage.
921 If a key and a path are provided, the key is preferred.
922 This means that the entry in the db is searched first and if this is not found, the path is used.
924 :param key: Key of the LeafSkel that contains the "dlkey" and the "name".
925 :param path: The path of the file in the Cloud Storage Bucket.
927 :return: Returns the file as a io.BytesIO buffer and the content-type
928 """
929 if not key and not path:
930 raise ValueError("Please provide a key or a path")
932 if key:
933 skel = self.viewSkel("leaf")
934 if not skel.read(db.key_helper(key, skel.kindName)):
935 if not path:
936 raise ValueError("This skeleton is not in the database!")
937 else:
938 path = f"""{skel["dlkey"]}/source/{skel["name"]}"""
940 bucket = self.get_bucket(skel["dlkey"])
941 else:
942 bucket = self.get_bucket(path.split("/", 1)[0]) # path's first part is dlkey plus eventual postfix
944 blob = bucket.blob(path)
945 return io.BytesIO(blob.download_as_bytes()), blob.content_type
947 def onDeleteRecursive(self, skelType: SkelType, skel: SkeletonInstance) -> None:
948 """
949 Mark the blob of each cascaded file for deletion.
951 The generic recursive delete of the Tree prototype is inherited as-is
952 (see :meth:`Tree.deleteRecursive`); the only File-specific step is
953 marking a leaf's blob for deletion, which is injected via this hook.
954 Directories (nodes) have no blob and are ignored.
955 """
956 if skelType == "leaf":
957 self.mark_for_deletion(skel["dlkey"])
959 @exposed
960 @skey
961 def getUploadURL(
962 self,
963 fileName: str,
964 mimeType: str,
965 size: t.Optional[int] = None,
966 node: t.Optional[str | db.Key] = None,
967 authData: t.Optional[str] = None,
968 authSig: t.Optional[str] = None,
969 public: bool = False,
970 ):
971 filename = fileName.strip() # VIUR4 FIXME: just for compatiblity of the parameter names
973 if not self.is_valid_filename(filename):
974 raise errors.UnprocessableEntity(f"Invalid filename {filename!r} provided")
976 # Validate the mimetype from the client seems legit
977 mimetype = mimeType.strip().lower()
978 if not (
979 mimetype
980 and mimetype.count("/") == 1
981 and all(ch in string.ascii_letters + string.digits + "/-.+" for ch in mimetype)
982 ):
983 raise errors.UnprocessableEntity(f"Invalid mime-type {mimetype!r} provided")
985 # Validate authentication data
986 if authData and authSig:
987 # First, validate the signature, otherwise we don't need to proceed further
988 if not self.hmac_verify(authData, authSig):
989 raise errors.Unauthorized()
991 authData = json.loads(base64.b64decode(authData.encode("ASCII")).decode("UTF-8"))
993 if datetime.datetime.strptime(authData["validUntil"], "%Y%m%d%H%M") < datetime.datetime.now():
994 raise errors.Gone("The upload URL has expired")
996 if authData["validMimeTypes"]:
997 for validMimeType in authData["validMimeTypes"]:
998 if (
999 validMimeType == mimetype
1000 or (validMimeType.endswith("*") and mimetype.startswith(validMimeType[:-1]))
1001 ):
1002 break
1003 else:
1004 raise errors.UnprocessableEntity(f"Invalid mime-type {mimetype} provided")
1006 node = authData["node"]
1007 maxSize = authData["maxSize"]
1009 else:
1010 rootNode = None
1011 if node and not (rootNode := self.getRootNode(node)):
1012 raise errors.NotFound(f"No valid root node found for {node=}")
1014 if not self.canAdd("leaf", rootNode):
1015 raise errors.Forbidden()
1017 if rootNode and public != bool(rootNode.get("public")):
1018 raise errors.Forbidden("Cannot upload a public file into private repository or vice versa")
1020 maxSize = None # The user has some file/add permissions, don't restrict fileSize
1022 if maxSize:
1023 if size > maxSize:
1024 raise errors.UnprocessableEntity(f"Size {size} exceeds maximum size {maxSize}")
1025 else:
1026 size = None
1028 # Create upload-URL and download key
1029 dlkey = utils.string.random() # let's roll a random key
1031 if public:
1032 dlkey += PUBLIC_DLKEY_SUFFIX # mark file as public
1034 blob = self.get_bucket(dlkey).blob(f"{dlkey}/source/{filename}")
1035 upload_url = blob.create_resumable_upload_session(content_type=mimeType, size=size, timeout=60)
1037 # Create a corresponding file-lock object early, otherwise we would have to ensure that the file-lock object
1038 # the user creates matches the file he had uploaded
1039 file_skel = self.addSkel("leaf")
1041 file_skel["name"] = filename + self.PENDING_POSTFIX
1042 file_skel["size"] = 0
1043 file_skel["mimetype"] = "application/octetstream"
1044 file_skel["dlkey"] = dlkey
1045 file_skel["parentdir"] = None
1046 file_skel["pendingparententry"] = db.key_helper(node, self.addSkel("node").kindName) if node else None
1047 file_skel["pending"] = True
1048 file_skel["weak"] = True
1049 file_skel["public"] = public
1050 file_skel["width"] = 0
1051 file_skel["height"] = 0
1053 file_skel.write()
1054 key = str(file_skel["key"])
1056 # Mark that entry dirty as we might never receive an add
1057 self.mark_for_deletion(dlkey)
1059 # In this case, we'd have to store the key in the users session so he can call add() later on
1060 if authData and authSig:
1061 session = current.session.get()
1063 if "pendingFileUploadKeys" not in session:
1064 session["pendingFileUploadKeys"] = []
1066 session["pendingFileUploadKeys"].append(key)
1068 # Clamp to the latest 50 pending uploads
1069 session["pendingFileUploadKeys"] = session["pendingFileUploadKeys"][-50:]
1070 session.markChanged()
1072 return self.render.view({
1073 "uploadKey": key,
1074 "uploadUrl": upload_url,
1075 })
1077 @exposed
1078 def download(self, blobKey: str, fileName: str = "", download: bool = False, sig: str = "", *args, **kwargs):
1079 """
1080 Download a file.
1081 :param blobKey: The unique blob key of the file.
1082 :param fileName: Optional filename to provide in the header.
1083 :param download: Set header to attachment retrival, set explictly to "1" if download is wanted.
1084 """
1085 if filename := fileName.strip():
1086 if not self.is_valid_filename(filename):
1087 raise errors.UnprocessableEntity(f"The provided filename {filename!r} is invalid!")
1089 try:
1090 values = base64.urlsafe_b64decode(blobKey).decode("UTF-8").split("\0")
1091 except ValueError:
1092 raise errors.BadRequest(f"Invalid encoding of blob key {blobKey!r}!")
1093 try:
1094 dlPath, validUntil, *download_filename = values
1095 # Maybe it's the old format, without a download_filename
1096 download_filename = download_filename[0] if download_filename else ""
1097 except ValueError:
1098 logging.error(f"Encoding of {blobKey=!r} OK. {values=} invalid.")
1099 raise errors.BadRequest(f"The blob key {blobKey!r} has an invalid amount of encoded values!")
1101 bucket = self.get_bucket(dlPath.split("/", 1)[0])
1103 if not sig:
1104 # Check if the current user has the right to download *any* blob present in this application.
1105 # blobKey is then the path inside cloudstore - not a base64 encoded tuple
1106 if not (usr := current.user.get()):
1107 raise errors.Unauthorized()
1108 if "root" not in usr["access"] and "file-view" not in usr["access"]:
1109 raise errors.Forbidden()
1110 validUntil = "-1" # Prevent this from being cached down below
1111 blob = bucket.get_blob(blobKey)
1113 else:
1114 # We got an request including a signature (probably a guest or a user without file-view access)
1115 # First, validate the signature, otherwise we don't need to proceed any further
1116 if not self.hmac_verify(blobKey, sig):
1117 raise errors.Forbidden()
1119 if validUntil != "0" and datetime.datetime.strptime(validUntil, "%Y%m%d%H%M") < datetime.datetime.now():
1120 blob = None
1121 else:
1122 blob = bucket.get_blob(dlPath)
1124 if not blob:
1125 raise errors.Gone("The requested blob has expired.")
1127 if not filename:
1128 filename = urlunquote(download_filename) if download_filename else blob.name.rsplit("/", 1)[-1]
1130 content_disposition = utils.build_content_disposition_header(filename, attachment=download)
1132 if isinstance(_CREDENTIALS, ServiceAccountCredentials):
1133 expiresAt = datetime.datetime.now() + datetime.timedelta(seconds=60)
1134 signedUrl = blob.generate_signed_url(expiresAt, response_disposition=content_disposition, version="v4")
1135 raise errors.Redirect(signedUrl)
1137 elif conf.instance.is_dev_server: # No Service-Account to sign with - Serve everything directly
1138 response = current.request.get().response
1139 response.headers["Content-Type"] = blob.content_type
1140 if content_disposition:
1141 response.headers["Content-Disposition"] = content_disposition
1142 return blob.download_as_bytes()
1144 if validUntil == "0" or blobKey.endswith(PUBLIC_DLKEY_SUFFIX): # Its an indefinitely valid URL
1145 if blob.size < 5 * 1024 * 1024: # Less than 5 MB - Serve directly and push it into the ede caches
1146 response = current.request.get().response
1147 response.headers["Content-Type"] = blob.content_type
1148 response.headers["Cache-Control"] = "public, max-age=604800" # 7 Days
1149 if content_disposition:
1150 response.headers["Content-Disposition"] = content_disposition
1151 return blob.download_as_bytes()
1153 # Default fallback - create a signed URL and redirect
1154 authRequest = google.auth.transport.requests.Request()
1155 expiresAt = datetime.datetime.now() + datetime.timedelta(seconds=60)
1156 signing_credentials = google.auth.compute_engine.IDTokenCredentials(authRequest, "")
1157 signedUrl = blob.generate_signed_url(
1158 expiresAt,
1159 credentials=signing_credentials,
1160 response_disposition=content_disposition,
1161 version="v4")
1163 raise errors.Redirect(signedUrl)
1165 SERVE_VALID_OPTIONS = {
1166 "c",
1167 "p",
1168 "fv",
1169 "fh",
1170 "r90",
1171 "r180",
1172 "r270",
1173 "nu",
1174 }
1175 """
1176 Valid modification option shorts for the serve-function.
1177 This is passed-through to the Google UserContent API, and hast to be supported there.
1178 """
1180 SERVE_VALID_FORMATS = {
1181 "jpg": "rj",
1182 "jpeg": "rj",
1183 "png": "rp",
1184 "webp": "rw",
1185 }
1186 """
1187 Valid file-formats to the serve-function.
1188 This is passed-through to the Google UserContent API, and hast to be supported there.
1189 """
1191 @exposed
1192 def serve(
1193 self,
1194 host: str,
1195 key: str,
1196 size: t.Optional[int] = None,
1197 filename: t.Optional[str] = None,
1198 options: str = "",
1199 download: bool = False,
1200 ):
1201 """
1202 Requests an image using the serving url to bypass direct Google requests.
1204 :param host: the google host prefix i.e. lh3
1205 :param key: the serving url key
1206 :param size: the target image size
1207 :param filename: a random string with an extention, valid extentions are (defined in File.SERVE_VALID_FORMATS).
1208 :param options: - seperated options (defined in File.SERVE_VALID_OPTIONS).
1209 c - crop
1210 p - face crop
1211 fv - vertrical flip
1212 fh - horizontal flip
1213 rXXX - rotate 90, 180, 270
1214 nu - no upscale
1215 :param download: Serves the content as download (Content-Disposition) or not.
1217 :return: Returns the requested content on success, raises a proper HTTP exception otherwise.
1218 """
1220 if any(c not in conf.search_valid_chars for c in host):
1221 raise errors.BadRequest("key contains invalid characters")
1223 # extract format from filename
1224 file_fmt = "webp"
1226 if filename:
1227 fmt = filename.rsplit(".", 1)[-1].lower()
1228 if fmt in self.SERVE_VALID_FORMATS:
1229 file_fmt = fmt
1230 else:
1231 raise errors.UnprocessableEntity(f"Unsupported filetype {fmt}")
1233 url = f"https://{host}.googleusercontent.com/{key}"
1235 if options and not all(param in self.SERVE_VALID_OPTIONS for param in options.split("-")):
1236 raise errors.BadRequest("Invalid options provided")
1238 options += f"-{self.SERVE_VALID_FORMATS[file_fmt]}"
1240 if size:
1241 options = f"s{size}-" + options
1243 url += "=" + options
1245 response = current.request.get().response
1246 response.headers["Content-Type"] = f"image/{file_fmt}"
1247 response.headers["Cache-Control"] = "public, max-age=604800" # 7 Days
1248 response.headers["Content-Disposition"] = utils.build_content_disposition_header(filename, attachment=download)
1250 answ = requests.get(url, timeout=20)
1251 if not answ.ok:
1252 logging.error(f"{answ.status_code} {answ.text}")
1253 raise errors.BadRequest("Unable to fetch a file with these parameters")
1255 return answ.content
1257 @exposed
1258 @force_ssl
1259 @force_post
1260 @skey(allow_empty=True)
1261 def add(self, skelType: SkelType, node: db.KeyType | None = None, *args, **kwargs):
1262 # We can't add files directly (they need to be uploaded
1263 if skelType == "leaf": # We need to handle leafs separately here
1264 targetKey = kwargs.get("key")
1265 skel = self.addSkel("leaf")
1267 if not skel.read(targetKey):
1268 raise errors.NotFound()
1270 if not skel["pending"]:
1271 raise errors.PreconditionFailed()
1273 skel["pending"] = False
1274 skel["parententry"] = skel["pendingparententry"]
1276 if skel["parententry"]:
1277 rootNode = self.getRootNode(skel["parententry"])
1278 else:
1279 rootNode = None
1281 if not self.canAdd("leaf", rootNode):
1282 # Check for a marker in this session (created if using a signed upload URL)
1283 session = current.session.get()
1284 if targetKey not in (session.get("pendingFileUploadKeys") or []):
1285 raise errors.Forbidden()
1286 session["pendingFileUploadKeys"].remove(targetKey)
1287 session.markChanged()
1289 # Now read the blob from the dlkey folder
1290 bucket = self.get_bucket(skel["dlkey"])
1292 blobs = list(bucket.list_blobs(prefix=f"""{skel["dlkey"]}/"""))
1293 if len(blobs) != 1:
1294 logging.error("Invalid number of blobs in folder")
1295 logging.error(targetKey)
1296 raise errors.PreconditionFailed()
1298 # only one item is allowed here!
1299 blob = blobs[0]
1301 # update the corresponding file skeleton
1302 skel["name"] = skel["name"].removesuffix(self.PENDING_POSTFIX)
1303 skel["mimetype"] = utils.string.escape(blob.content_type)
1304 skel["size"] = blob.size
1305 skel["parentrepo"] = rootNode["key"] if rootNode else None
1306 skel["weak"] = rootNode is None
1307 skel["crc32c_checksum"] = base64.b64decode(blob.crc32c).hex()
1308 skel["md5_checksum"] = base64.b64decode(blob.md5_hash).hex()
1309 self.onAdd("leaf", skel)
1310 skel.write()
1311 self.onAdded("leaf", skel)
1313 # Add updated download-URL as the auto-generated isn't valid yet.
1314 # Same lifetime as DownloadUrlBone, which this replaces.
1315 skel["downloadUrl"] = self.create_download_url(
1316 skel["dlkey"], skel["name"], expires=conf.render_json_download_url_expiration
1317 )
1319 return self.render.addSuccess(skel)
1321 return super().add(skelType, node, *args, **kwargs)
1323 @exposed
1324 def get_download_url(
1325 self,
1326 key: t.Optional[db.Key] = None,
1327 dlkey: t.Optional[str] = None,
1328 filename: t.Optional[str] = None,
1329 derived: bool = False,
1330 ):
1331 """
1332 Request a download url for a given file
1333 :param key: The key of the file
1334 :param dlkey: The download key of the file
1335 :param filename: The filename to be given. If no filename is provided
1336 downloadUrls for all derived files are returned in case of `derived=True`.
1337 :param derived: True, if a derived file download URL is being requested.
1338 """
1339 skel = self.viewSkel("leaf")
1340 if dlkey is not None:
1341 skel = skel.all().filter("dlkey", dlkey).getSkel()
1342 elif key is None and dlkey is None:
1343 raise errors.BadRequest("No key or dlkey provided")
1345 if not (skel and skel.read(key)):
1346 raise errors.NotFound()
1348 if not self.canView("leaf", skel):
1349 raise errors.Unauthorized()
1351 dlkey = skel["dlkey"]
1353 if derived and filename is None:
1354 res = {}
1355 for filename in skel["derived"]["files"]:
1356 res[filename] = self.create_download_url(dlkey, filename, derived)
1357 else:
1358 if derived:
1359 # Check if Filename exist in the Derives. We sign nothing that not exist.
1360 if filename not in skel["derived"]["files"]:
1361 raise errors.NotFound("File not in derives")
1362 else:
1363 if filename is None:
1364 filename = skel["name"]
1365 elif filename != skel["name"]:
1366 raise errors.NotFound("Filename not match")
1368 res = self.create_download_url(dlkey, filename, derived)
1370 return self.render.view(res)
1372 def onEdit(self, skelType: SkelType, skel: SkeletonInstance):
1373 super().onEdit(skelType, skel)
1375 if skelType == "leaf":
1376 old_skel = self.editSkel(skelType)
1377 old_skel.setEntity(skel.dbEntity)
1379 if old_skel["name"] == skel["name"]: # name not changed we can return
1380 return
1382 # Move Blob to new name
1383 # https://cloud.google.com/storage/docs/copying-renaming-moving-objects
1384 old_path = f"""{skel["dlkey"]}/source/{html.unescape(old_skel["name"])}"""
1385 new_path = f"""{skel["dlkey"]}/source/{html.unescape(skel["name"])}"""
1387 bucket = self.get_bucket(skel["dlkey"])
1389 if not (old_blob := bucket.get_blob(old_path)):
1390 raise errors.Gone()
1392 bucket.copy_blob(old_blob, bucket, new_path, if_generation_match=0)
1393 bucket.delete_blob(old_path)
1395 def onAdded(self, skelType: SkelType, skel: SkeletonInstance) -> None:
1396 if skelType == "leaf" and skel["mimetype"].startswith("image/"):
1397 if skel["size"] > self.IMAGE_META_MAX_SIZE:
1398 logging.warning(f"File size {skel['size']} exceeds limit {self.IMAGE_META_MAX_SIZE=}")
1399 return
1400 self.set_image_meta(skel["key"])
1402 super().onAdded(skelType, skel)
1404 @CallDeferred
1405 def set_image_meta(self, key: db.Key) -> None:
1406 """Write image metadata (height and width) to FileSkel"""
1407 skel = self.editSkel("leaf", key)
1408 if not skel.read(key):
1409 logging.error(f"File {key} does not exist")
1410 return
1411 if skel["width"] and skel["height"]:
1412 logging.info(f'File {skel["key"]} has already {skel["width"]=} and {skel["height"]=}')
1413 return
1414 file_name = html.unescape(skel["name"])
1415 blob = self.get_bucket(skel["dlkey"]).get_blob(f"""{skel["dlkey"]}/source/{file_name}""")
1416 if not blob:
1417 logging.error(f'Blob {skel["dlkey"]}/source/{file_name} is missing in Cloud Storage!')
1418 return
1420 file_obj = io.BytesIO()
1421 blob.download_to_file(file_obj)
1422 file_obj.seek(0)
1423 try:
1424 img = Image.open(file_obj)
1425 except Image.UnidentifiedImageError as e: # Can't load this image
1426 logging.exception(f'Cannot open {skel["key"]} | {skel["name"]} to set image meta data: {e}')
1427 return
1429 skel.patch(
1430 values={
1431 "width": img.width,
1432 "height": img.height,
1433 },
1434 )
1436 def mark_for_deletion(self, dlkey: str) -> None:
1437 """
1438 Adds a marker to the datastore that the file specified as *dlkey* can be deleted.
1440 Once the mark has been set, the data store is checked four times (default: every 4 hours)
1441 if the file is in use somewhere. If it is still in use, the mark goes away, otherwise
1442 the mark and the file are removed from the datastore. These delayed checks are necessary
1443 due to database inconsistency.
1445 :param dlkey: Unique download-key of the file that shall be marked for deletion.
1446 """
1447 fileObj = db.Query("viur-deleted-files").filter("dlkey", dlkey).getEntry()
1449 if fileObj: # Its allready marked
1450 return
1452 fileObj = db.Entity(db.Key("viur-deleted-files"))
1453 fileObj["itercount"] = 0
1454 fileObj["dlkey"] = str(dlkey)
1456 db.put(fileObj)
1459@PeriodicTask(interval=datetime.timedelta(hours=4))
1460def startCheckForUnreferencedBlobs():
1461 """
1462 Start searching for blob locks that have been recently freed
1463 """
1464 doCheckForUnreferencedBlobs()
1467@CallDeferred
1468def doCheckForUnreferencedBlobs(cursor=None):
1469 def getOldBlobKeysTxn(dbKey):
1470 obj = db.get(dbKey)
1471 if obj is None:
1472 # The lock was already processed and removed by a concurrent run
1473 return []
1474 res = obj["old_blob_references"] or []
1475 if obj["is_stale"]:
1476 db.delete(dbKey)
1477 else:
1478 obj["has_old_blob_references"] = False
1479 obj["old_blob_references"] = []
1480 db.put(obj)
1481 return res
1483 query = db.Query("viur-blob-locks").filter("has_old_blob_references", True).setCursor(cursor)
1484 for lockObj in query.run(100):
1485 oldBlobKeys = db.run_in_transaction(getOldBlobKeysTxn, lockObj.key)
1486 for blobKey in oldBlobKeys:
1487 if db.Query("viur-blob-locks").filter("active_blob_references =", blobKey).getEntry(): 1487 ↛ 1489line 1487 didn't jump to line 1489 because the condition on line 1487 was never true
1488 # This blob is referenced elsewhere
1489 logging.info(f"Stale blob is still referenced, {blobKey}")
1490 continue
1491 # Add a marker and schedule it for deletion
1492 fileObj = db.Query("viur-deleted-files").filter("dlkey", blobKey).getEntry()
1493 if fileObj: # Its already marked
1494 logging.info(f"Stale blob already marked for deletion, {blobKey}")
1495 continue
1496 fileObj = db.Entity(db.Key("viur-deleted-files"))
1497 fileObj["itercount"] = 0
1498 fileObj["dlkey"] = str(blobKey)
1499 logging.info(f"Stale blob marked dirty, {blobKey}")
1500 db.put(fileObj)
1501 newCursor = query.getCursor()
1502 if newCursor: 1502 ↛ 1503line 1502 didn't jump to line 1503 because the condition on line 1502 was never true
1503 doCheckForUnreferencedBlobs(newCursor)
1506@PeriodicTask(interval=datetime.timedelta(hours=4))
1507def startCleanupDeletedFiles():
1508 """
1509 Increase deletion counter on each blob currently not referenced and delete
1510 it if that counter reaches maxIterCount
1511 """
1512 doCleanupDeletedFiles()
1515@CallDeferred
1516def doCleanupDeletedFiles(cursor=None):
1517 maxIterCount = 2 # How often a file will be checked for deletion
1518 query = db.Query("viur-deleted-files")
1519 if cursor:
1520 query.setCursor(cursor)
1521 for file in query.run(100):
1522 if "dlkey" not in file:
1523 db.delete(file.key)
1524 elif db.Query("viur-blob-locks").filter("active_blob_references =", file["dlkey"]).getEntry():
1525 logging.info(f"""is referenced, {file["dlkey"]}""")
1526 db.delete(file.key)
1527 else:
1528 if file["itercount"] > maxIterCount:
1529 logging.info(f"""Finally deleting, {file["dlkey"]}""")
1530 bucket = conf.main_app.file.get_bucket(file["dlkey"])
1531 blobs = bucket.list_blobs(prefix=f"""{file["dlkey"]}/""")
1532 for blob in blobs:
1533 blob.delete()
1534 db.delete(file.key)
1535 # There should be exactly 1 or 0 of these
1536 for f in skeletonByKind("file")().all().filter("dlkey =", file["dlkey"]).fetch(99):
1537 f.delete()
1539 if f["serving_url"]:
1540 bucket = conf.main_app.file.get_bucket(f["dlkey"])
1541 blob_key = blobstore.create_gs_key(
1542 f"/gs/{bucket.name}/{f['dlkey']}/source/{f['name']}"
1543 )
1544 images.delete_serving_url(blob_key) # delete serving url
1545 else:
1546 logging.debug(f"""Increasing count, {file["dlkey"]}""")
1547 file["itercount"] += 1
1548 db.put(file)
1549 newCursor = query.getCursor()
1550 if newCursor:
1551 doCleanupDeletedFiles(newCursor)
1554@PeriodicTask(interval=datetime.timedelta(hours=4))
1555def start_delete_pending_files():
1556 """
1557 Start deletion of pending FileSkels that are older than 7 days.
1558 """
1559 DeleteEntitiesIter.startIterOnQuery(
1560 FileLeafSkel().all()
1561 .filter("pending =", True)
1562 .filter("creationdate <", utils.utcNow() - datetime.timedelta(days=7))
1563 )
1566# DEPRECATED ATTRIBUTES HANDLING
1568def __getattr__(attr: str) -> object:
1569 if entry := { 1569 ↛ 1573line 1569 didn't jump to line 1573 because the condition on line 1569 was never true
1570 # stuff prior viur-core < 3.7
1571 "GOOGLE_STORAGE_BUCKET": ("conf.main_app.file.get_bucket()", _private_bucket),
1572 }.get(attr):
1573 msg = f"{attr} was replaced by {entry[0]}"
1574 warnings.warn(msg, DeprecationWarning, stacklevel=2)
1575 logging.warning(msg, stacklevel=2)
1576 return entry[1]
1578 return super(__import__(__name__).__class__).__getattribute__(attr)