Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/i18n.py: 20%
256 statements
« prev ^ index » next coverage.py v7.9.1, created at 2025-06-26 11:31 +0000
1"""
2This module provides translation, also known as internationalization -- short: i18n.
4Project translations must be stored in the datastore. There are only some
5static translation tables in the viur-core to have some basic ones.
7The viur-core's own "translation" module (routed as _translation) provides
8an API to manage these translations, for example in the vi-admin.
10How to use translations?
11First, make sure that the languages are configured:
12.. code-block:: python
13 from viur.core.config import conf
14 # These are the main languages (for which translated values exist)
15 # that should be available for the project.
16 conf.i18n.available_languages = = ["en", "de", "fr"]
18 # These are some aliases for languages that should use the translated
19 # values of a particular main language, but don't have their own values.
20 conf.i18n.language_alias_map = {
21 "at": "de", # Austria uses German
22 "ch": "de", # Switzerland uses German
23 "be": "fr", # Belgian uses France
24 "us": "en", # US uses English
25 }
27Now translations can be used
291. In python
30.. code-block:: python
31 from viur.core.i18n import translate
32 # Just the translation key, the minimal case
33 print(translate("translation-key"))
34 # also provide a default value to use if there's no value in the datastore
35 # set and a hint to provide some context.
36 print(translate("translation-key", "the default value", "a hint"))
37 # Use string interpolation with variables
38 print(translate("hello", "Hello {{name}}!", "greeting a user")(name=current.user.get()["firstname"]))
402. In jinja
41.. code-block:: jinja
42 {# Use the ViUR translation extension, it can be compiled with the template,
43 caches the translation values and is therefore efficient #}
44 {% do translate "hello", "Hello {{name}}!", "greet a user", name="ViUR" %}
46 {# But in some cases the key or interpolation variables are dynamic and
47 not available during template compilation.
48 For this you can use the translate function: #}
49 {{ translate("hello", "Hello {{name}}!", "greet a user", name=skel["firstname"]) }}
52How to add translations
53There are two ways to add translations:
541. Manually
55With the vi-admin. Entries can be added manually by creating a new skeleton
56and filling in of the key and values.
582. Automatically
59The add_missing_translations option must be enabled for this.
60.. code-block:: python
62 from viur.core.config import conf
63 conf.i18n.add_missing_translations = True
66If a translation is now printed and the key is unknown (because someone has
67just added the related print code), an entry is added in the datastore kind.
68In addition, the default text and the hint are filled in and the filename
69and the line from the call from the code are set in the skeleton.
70This is the recommended way, as ViUR collects all the information you need
71and you only have to enter the translated values.
72(3. own way
73Of course you can create skeletons / entries in the datastore in your project
74on your own. Just use the TranslateSkel).
75""" # FIXME: grammar, rst syntax
76import datetime
77import enum
78import fnmatch
79import logging
80import sys
81import traceback
82import typing as t
83from pathlib import Path
85import jinja2.ext as jinja2
86from viur.core import current, db, languages, tasks
87from viur.core.config import conf
89systemTranslations = {}
90"""Memory storage for translation methods"""
92KINDNAME = "viur-translations"
93"""Kindname for the translations"""
96class AddMissing(enum.IntEnum):
97 """
98 An indicator flag for the `add_missing` parameter of the `translate`
99 to make this decision final and ignore the `conf.i18n.add_missing_translations` configuration.
100 """
102 NEVER = enum.auto()
103 """This translation will never be added, regardless of any config.
104 It's like a final `False`.
105 """
107 ALWAYS = enum.auto()
108 """This translation will be always be added, regardless of any config.
109 It's like a final `True`.
110 """
113class LanguageWrapper(dict):
114 """
115 Wrapper-class for a multi-language value.
117 It's a dictionary, allowing accessing each stored language,
118 but can also be used as a string, in which case it tries to
119 guess the correct language.
120 Used by the HTML renderer to provide multi-lang bones values for templates.
121 """
123 def __init__(self, languages: list[str] | tuple[str]):
124 """
125 :param languages: Languages which are set in the bone.
126 """
127 super(LanguageWrapper, self).__init__()
128 self.languages = languages
130 def __str__(self) -> str:
131 return str(self.resolve())
133 def __bool__(self) -> bool:
134 # Overridden to support if skel["bone"] tests in html render
135 # (otherwise that test is always true as this dict contains keys)
136 return bool(str(self))
138 def resolve(self) -> str:
139 """
140 Causes this wrapper to evaluate to the best language available for the current request.
142 :returns: An item stored inside this instance or the empty string.
143 """
144 lang = current.language.get()
145 if lang:
146 lang = conf.i18n.language_alias_map.get(lang, lang)
147 else:
148 logging.warning(f"No lang set to current! {lang = }")
149 lang = self.languages[0]
150 if (value := self.get(lang)) and str(value).strip():
151 # The site language is available and not empty
152 return value
153 else: # Choose the first not-empty value as alternative
154 for lang in self.languages:
155 if (value := self.get(lang)) and str(value).strip():
156 return value
157 return "" # TODO: maybe we should better use sth like None or N/A
160class translate:
161 """
162 Translate class which chooses the correct translation according to the request language
164 This class is the replacement for the old translate() function provided by ViUR2. This classes __init__
165 takes the unique translation key (a string usually something like "user.auth_user_password.loginfailed" which
166 uniquely defines this text fragment), a default text that will be used if no translation for this key has been
167 added yet (in the projects default language) and a hint (an optional text that can convey context information
168 for the persons translating these texts - they are not shown to the end-user). This class will resolve its
169 translations upfront, so the actual resolving (by casting this class to string) is fast. This resolves most
170 translation issues with bones, which can now take an instance of this class as it's description/hints.
171 """
173 __slots__ = (
174 "add_missing",
175 "default_variables",
176 "defaultText",
177 "filename",
178 "force_lang",
179 "hint",
180 "key",
181 "lineno",
182 "public",
183 "translationCache",
184 )
186 def __init__(
187 self,
188 key: str,
189 defaultText: str = None,
190 hint: str = None,
191 force_lang: str = None,
192 public: bool = False,
193 add_missing: bool | AddMissing = False,
194 default_variables: dict[str, t.Any] | None = None,
195 caller_is_jinja: bool = False,
196 ):
197 """
198 :param key: The unique key defining this text fragment.
199 Usually it's a path/filename and a unique descriptor in that file
200 :param defaultText: The text to use if no translation has been added yet.
201 While optional, it's recommended to set this, as the key is used
202 instead if neither are available.
203 :param hint: A text only shown to the person translating this text,
204 as the key/defaultText may have different meanings in the
205 target language.
206 :param force_lang: Use this language instead the one of the request.
207 :param public: Flag for public translations, which can be obtained via /json/_translate/get_public.
208 :param default_variables: Default values for variable substitution.
209 :param caller_is_jinja: Is the call caused by our jinja method?
210 """
211 super().__init__()
213 if not isinstance(key, str): 213 ↛ 215line 213 didn't jump to line 215 because the condition on line 213 was never true
214 # TODO: ViUR4: raise a ValueError instead of the warning
215 logging.warning(f"Got non-string (type {type(key)}) as {key=}!", exc_info=True)
216 if isinstance(key, translate):
217 # Because of the string cast below, we would otherwise have a translated string as key
218 key = key.key
220 if force_lang is not None and force_lang not in conf.i18n.available_dialects: 220 ↛ 221line 220 didn't jump to line 221 because the condition on line 220 was never true
221 raise ValueError(f"The language {force_lang=} is not available")
223 key = str(key) # ensure key is a str
224 self.key = key.lower()
225 self.defaultText = defaultText or key
226 self.hint = hint
227 self.translationCache = None
228 self.force_lang = force_lang
229 self.public = public
230 self.add_missing = add_missing
231 self.default_variables = default_variables or {}
232 self.filename, self.lineno = None, None
234 if ( 234 ↛ 240line 234 didn't jump to line 240 because the condition on line 234 was never true
235 add_missing is not AddMissing.NEVER
236 and (add_missing or conf.i18n.add_missing_translations)
237 and self.key not in systemTranslations
238 ):
239 # This translation seems to be new and should be added
240 for frame, line in traceback.walk_stack(sys._getframe(0).f_back):
241 if self.filename is None:
242 # Use the first frame as fallback.
243 # In case of calling this class directly,
244 # this is anyway the caller we're looking for.
245 self.filename = frame.f_code.co_filename
246 self.lineno = frame.f_lineno
247 if not caller_is_jinja:
248 break
249 if caller_is_jinja and not frame.f_code.co_filename.endswith(".py"):
250 # Look for the latest html, macro (not py) where the
251 # translate method has been used, that's our caller
252 self.filename = frame.f_code.co_filename
253 self.lineno = line
254 break
256 def __repr__(self) -> str:
257 return f"<translate object for {self.key} with force_lang={self.force_lang}>"
259 def __str__(self) -> str:
260 if self.translationCache is None:
261 global systemTranslations
263 if self.key not in systemTranslations:
264 # either the translate()-object has add_missing set
265 if not (add_missing := self.add_missing) and not isinstance(add_missing, AddMissing):
266 # otherwise, use configuration flag
267 add_missing = conf.i18n.add_missing_translations
269 # match against fnmatch pattern, when given
270 if isinstance(add_missing, str):
271 add_missing = fnmatch.fnmatch(self.key, add_missing)
272 elif isinstance(add_missing, t.Iterable):
273 add_missing = bool(any(fnmatch.fnmatch(self.key, pat) for pat in add_missing))
274 elif callable(add_missing):
275 add_missing = add_missing(self)
276 else:
277 add_missing = bool(add_missing)
279 if add_missing is True or add_missing is AddMissing.ALWAYS:
280 # This translation seems to be new and should be added
281 add_missing_translation(
282 key=self.key,
283 hint=self.hint,
284 default_text=self.defaultText,
285 filename=self.filename,
286 lineno=self.lineno,
287 variables=list(self.default_variables.keys()),
288 public=self.public,
289 )
291 self.translationCache = self.merge_alias(systemTranslations.get(self.key, {}))
293 if (lang := self.force_lang) is None:
294 # The default case: use the request language
295 lang = current.language.get()
297 if value := self.translationCache.get(lang):
298 return self.substitute_vars(value, **self.default_variables)
300 # Use the default text from datastore or from the caller arguments
301 return self.substitute_vars(
302 self.translationCache.get("_default_text_") or self.defaultText,
303 **self.default_variables
304 )
306 def translate(self, **kwargs) -> str:
307 """Substitute the given kwargs in the translated or default text."""
308 return self.substitute_vars(str(self), **(self.default_variables | kwargs))
310 def __call__(self, **kwargs) -> str:
311 """Just an alias for translate"""
312 return self.translate(**kwargs)
314 @staticmethod
315 def substitute_vars(value: str, **kwargs) -> str:
316 """Substitute vars in a translation
318 Variables has to start with two braces (`{{`), followed by the variable
319 name and end with two braces (`}}`).
320 Values can be anything, they are cast to string anyway.
321 "Hello {{name}}!" becomes with name="Bob": "Hello Bob!"
322 """
323 res = str(value)
324 for k, v in kwargs.items():
325 # 2 braces * (escape + real brace) + 1 for variable = 5
326 res = res.replace(f"{{{{{k}}}}}", str(v))
327 return res
329 @staticmethod
330 def merge_alias(translations: dict[str, str]):
331 """Make sure each aliased language has a value
333 If an aliased language does not have a value in the translation dict,
334 the value of the main language is copied.
335 """
336 for alias, main in conf.i18n.language_alias_map.items():
337 if not (value := translations.get(alias)) or not value.strip():
338 if main_value := translations.get(main):
339 # Use only not empty value
340 translations[alias] = main_value
341 return translations
344class TranslationExtension(jinja2.Extension):
345 """
346 Default translation extension for jinja2 render.
347 Use like {% translate "translationKey", "defaultText", "translationHint", replaceValue1="replacedText1" %}
348 All except translationKey is optional. translationKey is the same Key supplied to _() before.
349 defaultText will be printed if no translation is available.
350 translationHint is an optional hint for anyone adding a now translation how/where that translation is used.
351 `force_lang` can be used as a keyword argument (the only allowed way) to
352 force the use of a specific language, not the language of the request.
353 """
355 tags = {
356 "translate",
357 }
359 def parse(self, parser):
360 # Parse the translate tag
361 global systemTranslations
363 args = [] # positional args for the `_translate()` method
364 kwargs = {} # keyword args (force_lang + substitute vars) for the `_translate()` method
365 lineno = parser.stream.current.lineno
366 filename = parser.stream.filename
368 # Parse arguments (args and kwargs) until the current block ends
369 lastToken = None
370 while parser.stream.current.type != 'block_end':
371 lastToken = parser.parse_expression()
372 if parser.stream.current.type == "comma": # It's a positional arg
373 args.append(lastToken.value)
374 next(parser.stream) # Advance pointer
375 lastToken = None
376 elif parser.stream.current.type == "assign":
377 next(parser.stream) # Advance beyond =
378 expr = parser.parse_expression()
379 kwargs[lastToken.name] = expr.value
380 if parser.stream.current.type == "comma":
381 next(parser.stream)
382 elif parser.stream.current.type == "block_end":
383 lastToken = None
384 break
385 else:
386 raise SyntaxError()
387 lastToken = None
389 if lastToken: # TODO: what's this? what it is doing?
390 # logging.debug(f"final append {lastToken = }")
391 args.append(lastToken.value)
393 if not 0 < len(args) <= 3:
394 raise SyntaxError("Translation-Key missing or excess parameters!")
396 args += [""] * (3 - len(args))
397 args += [kwargs]
398 name = args[0].lower()
399 public = kwargs.pop("_public_", False) or False
401 if conf.i18n.add_missing_translations and name not in systemTranslations:
402 add_missing_translation(
403 key=name,
404 hint=args[1],
405 default_text=args[2],
406 filename=filename,
407 lineno=lineno,
408 variables=list(kwargs.keys()),
409 public=public,
410 )
412 translations = translate.merge_alias(systemTranslations.get(name, {}))
413 args[1] = translations.get("_default_text_") or args[1]
414 args = [jinja2.nodes.Const(x) for x in args]
415 args.append(jinja2.nodes.Const(translations))
416 return jinja2.nodes.CallBlock(self.call_method("_translate", args), [], [], []).set_lineno(lineno)
418 def _translate(
419 self, key: str, default_text: str, hint: str, kwargs: dict[str, t.Any],
420 translations: dict[str, str], caller
421 ) -> str:
422 """Perform the actual translation during render"""
423 lang = kwargs.pop("force_lang", current.language.get())
424 res = str(translations.get(lang, default_text))
425 return translate.substitute_vars(res, **kwargs)
428def initializeTranslations() -> None:
429 """
430 Fetches all translations from the datastore and populates the *systemTranslations* dictionary of this module.
431 Currently, the translate-class will resolve using that dictionary; but as we expect projects to grow and
432 accumulate translations that are no longer/not yet used, we plan to made the translation-class fetch it's
433 translations directly from the datastore, so we don't have to allocate memory for unused translations.
434 """
435 # Load translations from static languages module into systemTranslations
436 # If they're in the datastore, they will be overwritten below.
437 for lang in dir(languages):
438 if lang.startswith("__"):
439 continue
440 for name, tr_value in getattr(languages, lang).items():
441 systemTranslations.setdefault(name, {})[lang] = tr_value
443 # Load translations from datastore into systemTranslations
444 # TODO: iter() would be more memory efficient, but unfortunately takes much longer than run()
445 # for entity in db.Query(KINDNAME).iter():
446 for entity in db.Query(KINDNAME).run(10_000):
447 if "name" not in entity:
448 logging.warning(f"translations entity {entity.key} has no name set --> Call migration")
449 migrate_translation(entity.key)
450 # Before the migration has run do a quick modification to get it loaded as is
451 entity["name"] = entity["key"] or entity.key.name
452 if not entity.get("name"):
453 logging.error(f'translations entity {entity.key} has an empty {entity["name"]=} set. Skipping.')
454 continue
455 if entity and not isinstance(entity["translations"], dict):
456 logging.error(f'translations entity {entity.key} has invalid '
457 f'translations set: {entity["translations"]}. Skipping.')
458 continue
460 translations = {
461 "_default_text_": entity.get("default_text") or None,
462 "_public_": entity.get("public") or False,
463 }
465 for lang, translation in entity["translations"].items():
466 if lang not in conf.i18n.available_dialects:
467 # Don't store unknown languages in the memory
468 continue
469 if not translation or not str(translation).strip():
470 # Skip empty values
471 continue
472 translations[lang] = translation
474 systemTranslations[entity["name"]] = translations
477@tasks.CallDeferred
478@tasks.retry_n_times(20)
479def add_missing_translation(
480 key: str,
481 hint: str | None = None,
482 default_text: str | None = None,
483 filename: str | None = None,
484 lineno: int | None = None,
485 variables: list[str] = None,
486 public: bool = False,
487) -> None:
488 """Add missing translations to datastore"""
490 logging.info(f"add_missing_translation {key=} {hint=} {default_text=} {filename=} {lineno=} {variables=} {public=}")
492 try:
493 from viur.core.modules.translation import TranslationSkel, Creator
494 except ImportError as exc:
495 # We use translate inside the TranslationSkel, this causes circular dependencies which can be ignored
496 logging.warning(f"ImportError (probably during warmup), "
497 f"cannot add translation {key}: {exc}", exc_info=True)
498 return
500 # Ensure lowercase key
501 key = key.lower()
503 # Check if key already exists
504 # if db.Get(db.Key(KINDNAME, key)): # FIXME ViUR4 should only use named keys
505 entity = db.Query(KINDNAME).filter("name =", key).getEntry()
506 if entity is not None:
507 # Ensure it doesn't exist to avoid datastore conflicts
508 logging.warning(f"Found an entity with {key=}. Probably an other instance was faster.")
509 return
511 if isinstance(filename, str):
512 try:
513 filename = str(Path(filename)
514 .relative_to(conf.instance.project_base_path,
515 conf.instance.core_base_path))
516 except ValueError:
517 pass # not a subpath
519 logging.info(f"Add missing translation {key}")
520 skel = TranslationSkel()
521 skel["name"] = key
522 skel["default_text"] = default_text or None
523 skel["hint"] = hint or None
524 skel["usage_filename"] = filename
525 skel["usage_lineno"] = lineno
526 skel["usage_variables"] = variables or []
527 skel["creator"] = Creator.VIUR
528 skel["public"] = public
529 skel.write()
531 # Add to system translation to avoid triggering this method again
532 systemTranslations[key] = {
533 "_default_text_": default_text or None,
534 "_public_": public,
535 }
538@tasks.CallDeferred
539@tasks.retry_n_times(20)
540def migrate_translation(
541 key: db.Key,
542) -> None:
543 """Migrate entities, if required.
545 With viur-core 3.6 translations are now managed as Skeletons and require
546 some changes, which are performed in this method.
547 """
548 from viur.core.modules.translation import TranslationSkel
549 logging.info(f"Migrate translation {key}")
550 entity: db.Entity = db.Get(key)
551 if "name" not in entity:
552 entity["name"] = entity["key"] or key.name
553 if "translation" in entity:
554 if not isinstance(dict, entity["translation"]):
555 logging.error("translation is not a dict?")
556 entity["translation"]["_viurLanguageWrapper_"] = True
557 skel = TranslationSkel()
558 skel.setEntity(entity)
559 skel["key"] = key
560 try:
561 skel.write()
562 except ValueError as exc:
563 logging.exception(exc)
564 if "unique value" in exc.args[0] and "recently claimed" in exc.args[0]:
565 logging.info(f"Delete duplicate entry {key}: {entity}")
566 db.Delete(key)
567 else:
568 raise exc
571localizedDateTime = translate("const_datetimeformat", "%a %b %d %H:%M:%S %Y", "Localized Time and Date format string")
572localizedDate = translate("const_dateformat", "%m/%d/%Y", "Localized Date only format string")
573localizedTime = translate("const_timeformat", "%H:%M:%S", "Localized Time only format string")
574localizedAbbrevDayNames = {
575 0: translate("const_day_0_short", "Sun", "Abbreviation for Sunday"),
576 1: translate("const_day_1_short", "Mon", "Abbreviation for Monday"),
577 2: translate("const_day_2_short", "Tue", "Abbreviation for Tuesday"),
578 3: translate("const_day_3_short", "Wed", "Abbreviation for Wednesday"),
579 4: translate("const_day_4_short", "Thu", "Abbreviation for Thursday"),
580 5: translate("const_day_5_short", "Fri", "Abbreviation for Friday"),
581 6: translate("const_day_6_short", "Sat", "Abbreviation for Saturday"),
582}
583localizedDayNames = {
584 0: translate("const_day_0_long", "Sunday", "Sunday"),
585 1: translate("const_day_1_long", "Monday", "Monday"),
586 2: translate("const_day_2_long", "Tuesday", "Tuesday"),
587 3: translate("const_day_3_long", "Wednesday", "Wednesday"),
588 4: translate("const_day_4_long", "Thursday", "Thursday"),
589 5: translate("const_day_5_long", "Friday", "Friday"),
590 6: translate("const_day_6_long", "Saturday", "Saturday"),
591}
592localizedAbbrevMonthNames = {
593 1: translate("const_month_1_short", "Jan", "Abbreviation for January"),
594 2: translate("const_month_2_short", "Feb", "Abbreviation for February"),
595 3: translate("const_month_3_short", "Mar", "Abbreviation for March"),
596 4: translate("const_month_4_short", "Apr", "Abbreviation for April"),
597 5: translate("const_month_5_short", "May", "Abbreviation for May"),
598 6: translate("const_month_6_short", "Jun", "Abbreviation for June"),
599 7: translate("const_month_7_short", "Jul", "Abbreviation for July"),
600 8: translate("const_month_8_short", "Aug", "Abbreviation for August"),
601 9: translate("const_month_9_short", "Sep", "Abbreviation for September"),
602 10: translate("const_month_10_short", "Oct", "Abbreviation for October"),
603 11: translate("const_month_11_short", "Nov", "Abbreviation for November"),
604 12: translate("const_month_12_short", "Dec", "Abbreviation for December"),
605}
606localizedMonthNames = {
607 1: translate("const_month_1_long", "January", "January"),
608 2: translate("const_month_2_long", "February", "February"),
609 3: translate("const_month_3_long", "March", "March"),
610 4: translate("const_month_4_long", "April", "April"),
611 5: translate("const_month_5_long", "May", "May"),
612 6: translate("const_month_6_long", "June", "June"),
613 7: translate("const_month_7_long", "July", "July"),
614 8: translate("const_month_8_long", "August", "August"),
615 9: translate("const_month_9_long", "September", "September"),
616 10: translate("const_month_10_long", "October", "October"),
617 11: translate("const_month_11_long", "November", "November"),
618 12: translate("const_month_12_long", "December", "December"),
619}
622def localizedStrfTime(datetimeObj: datetime.datetime, format: str) -> str:
623 """
624 Provides correct localized names for directives like %a which don't get translated on GAE properly as we can't
625 set the locale (for each request).
626 This currently replaces %a, %A, %b, %B, %c, %x and %X.
628 :param datetimeObj: Datetime-instance to call strftime on
629 :param format: String containing the Format to apply.
630 :returns: Date and time formatted according to format with correct localization
631 """
632 if "%c" in format:
633 format = format.replace("%c", str(localizedDateTime))
634 if "%x" in format:
635 format = format.replace("%x", str(localizedDate))
636 if "%X" in format:
637 format = format.replace("%X", str(localizedTime))
638 if "%a" in format:
639 format = format.replace("%a", str(localizedAbbrevDayNames[int(datetimeObj.strftime("%w"))]))
640 if "%A" in format:
641 format = format.replace("%A", str(localizedDayNames[int(datetimeObj.strftime("%w"))]))
642 if "%b" in format:
643 format = format.replace("%b", str(localizedAbbrevMonthNames[int(datetimeObj.strftime("%m"))]))
644 if "%B" in format:
645 format = format.replace("%B", str(localizedMonthNames[int(datetimeObj.strftime("%m"))]))
646 return datetimeObj.strftime(format)