Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/i18n.py: 19%

1"""

2This module provides translation, also known as internationalization -- short: i18n.

4Project translations must be stored in the datastore. There are only some

5static translation tables in the viur-core to have some basic ones.

7The viur-core's own "translation" module (routed as _translation) provides

8an API to manage these translations, for example in the vi-admin.

10How to use translations?

11First, make sure that the languages are configured:

12.. code-block:: python

13 from viur.core.config import conf

14 # These are the main languages (for which translated values exist)

15 # that should be available for the project.

16 conf.i18n.available_languages = = ["en", "de", "fr"]

18 # These are some aliases for languages that should use the translated

19 # values of a particular main language, but don't have their own values.

20 conf.i18n.language_alias_map = {

21 "at": "de", # Austria uses German

22 "ch": "de", # Switzerland uses German

23 "be": "fr", # Belgian uses France

24 "us": "en", # US uses English

25 }

27Now translations can be used

291. In python

30.. code-block:: python

31 from viur.core.i18n import translate

32 # Just the translation key, the minimal case

33 print(translate("translation-key"))

34 # also provide a default value to use if there's no value in the datastore

35 # set and a hint to provide some context.

36 print(translate("translation-key", "the default value", "a hint"))

37 # Use string interpolation with variables

38 print(translate("hello", "Hello {{name}}!", "greeting a user")(name=current.user.get()["firstname"]))

402. In jinja

41.. code-block:: jinja

42 {# Use the ViUR translation extension, it can be compiled with the template,

43 caches the translation values and is therefore efficient #}

44 {% do translate "hello", "Hello {{name}}!", "greet a user", name="ViUR" %}

46 {# But in some cases the key or interpolation variables are dynamic and

47 not available during template compilation.

48 For this you can use the translate function: #}

49 {{ translate("hello", "Hello {{name}}!", "greet a user", name=skel["firstname"]) }}

52How to add translations

53There are two ways to add translations:

541. Manually

55With the vi-admin. Entries can be added manually by creating a new skeleton

56and filling in of the key and values.

582. Automatically

59The add_missing_translations option must be enabled for this.

60.. code-block:: python

62 from viur.core.config import conf

63 conf.i18n.add_missing_translations = True

66If a translation is now printed and the key is unknown (because someone has

67just added the related print code), an entry is added in the datastore kind.

68In addition, the default text and the hint are filled in and the filename

69and the line from the call from the code are set in the skeleton.

70This is the recommended way, as ViUR collects all the information you need

71and you only have to enter the translated values.

72(3. own way

73Of course you can create skeletons / entries in the datastore in your project

74on your own. Just use the TranslateSkel).

75""" # FIXME: grammar, rst syntax

76import datetime

77import logging

78import sys

79import traceback

80import typing as t

81from pathlib import Path

83import jinja2.ext as jinja2

85from viur.core import current, db, languages, tasks

86from viur.core.config import conf

88systemTranslations = {}

89"""Memory storage for translation methods"""

91KINDNAME = "viur-translations"

92"""Kindname for the translations"""

95class LanguageWrapper(dict):

96 """

97 Wrapper-class for a multi-language value.

99 It's a dictionary, allowing accessing each stored language,

100 but can also be used as a string, in which case it tries to

101 guess the correct language.

102 Used by the HTML renderer to provide multi-lang bones values for templates.

103 """

104

105 def __init__(self, languages: list[str] | tuple[str]):

106 """

107 :param languages: Languages which are set in the bone.

108 """

109 super(LanguageWrapper, self).__init__()

110 self.languages = languages

111

112 def __str__(self) -> str:

113 return str(self.resolve())

114

115 def __bool__(self) -> bool:

116 # Overridden to support if skel["bone"] tests in html render

117 # (otherwise that test is always true as this dict contains keys)

118 return bool(str(self))

119

120 def resolve(self) -> str:

121 """

122 Causes this wrapper to evaluate to the best language available for the current request.

123

124 :returns: An item stored inside this instance or the empty string.

125 """

126 lang = current.language.get()

127 if lang:

128 lang = conf.i18n.language_alias_map.get(lang, lang)

129 else:

130 logging.warning(f"No lang set to current! {lang = }")

131 lang = self.languages[0]

132 if (value := self.get(lang)) and str(value).strip():

133 # The site language is available and not empty

134 return value

135 else: # Choose the first not-empty value as alternative

136 for lang in self.languages:

137 if (value := self.get(lang)) and str(value).strip():

138 return value

139 return "" # TODO: maybe we should better use sth like None or N/A

140

141

142class translate:

143 """

144 Translate class which chooses the correct translation according to the request language

145

146 This class is the replacement for the old translate() function provided by ViUR2. This classes __init__

147 takes the unique translation key (a string usually something like "user.auth_user_password.loginfailed" which

148 uniquely defines this text fragment), a default text that will be used if no translation for this key has been

149 added yet (in the projects default language) and a hint (an optional text that can convey context information

150 for the persons translating these texts - they are not shown to the end-user). This class will resolve its

151 translations upfront, so the actual resolving (by casting this class to string) is fast. This resolves most

152 translation issues with bones, which can now take an instance of this class as it's description/hints.

153 """

154

155 __slots__ = (

156 "key",

157 "defaultText",

158 "hint",

159 "translationCache",

160 "force_lang",

161 "public",

162 "filename",

163 "lineno",

164 )

165

166 def __init__(

167 self,

168 key: str,

169 defaultText: str = None,

170 hint: str = None,

171 force_lang: str = None,

172 public: bool = False,

173 caller_is_jinja: bool = False,

174 ):

175 """

176 :param key: The unique key defining this text fragment.

177 Usually it's a path/filename and a unique descriptor in that file

178 :param defaultText: The text to use if no translation has been added yet.

179 While optional, it's recommended to set this, as the key is used

180 instead if neither are available.

181 :param hint: A text only shown to the person translating this text,

182 as the key/defaultText may have different meanings in the

183 target language.

184 :param force_lang: Use this language instead the one of the request.

185 :param public: Flag for public translations, which can be obtained via /json/_translate/get_public.

186 :param caller_is_jinja: Is the call caused by our jinja method?

187 """

188 super().__init__()

189

190 if not isinstance(key, str): 190 ↛ 191line 190 didn't jump to line 191 because the condition on line 190 was never true

191 logging.warning(f"Got non-string (type {type(key)}) as {key=}!", exc_info=True)

192

193 if force_lang is not None and force_lang not in conf.i18n.available_dialects: 193 ↛ 194line 193 didn't jump to line 194 because the condition on line 193 was never true

194 raise ValueError(f"The language {force_lang=} is not available")

195

196 key = str(key) # ensure key is a str

197 self.key = key.lower()

198 self.defaultText = defaultText or key

199 self.hint = hint

200 self.translationCache = None

201 self.force_lang = force_lang

202 self.public = public

203 self.filename, self.lineno = None, None

204

205 if conf.i18n.add_missing_translations and self.key not in systemTranslations: 205 ↛ 207line 205 didn't jump to line 207 because the condition on line 205 was never true

206 # This translation seems to be new and should be added

207 for frame, line in traceback.walk_stack(sys._getframe(0).f_back):

208 if self.filename is None:

209 # Use the first frame as fallback.

210 # In case of calling this class directly,

211 # this is anyway the caller we're looking for.

212 self.filename = frame.f_code.co_filename

213 self.lineno = frame.f_lineno

214 if not caller_is_jinja:

215 break

216 if caller_is_jinja and not frame.f_code.co_filename.endswith(".py"):

217 # Look for the latest html, macro (not py) where the

218 # translate method has been used, that's our caller

219 self.filename = frame.f_code.co_filename

220 self.lineno = line

221 break

222

223 def __repr__(self) -> str:

224 return f"<translate object for {self.key} with force_lang={self.force_lang}>"

225

226 def __str__(self) -> str:

227 if self.translationCache is None:

228 global systemTranslations

229

230 if conf.i18n.add_missing_translations and self.key not in systemTranslations:

231 # This translation seems to be new and should be added

232

233 add_missing_translation(

234 key=self.key,

235 hint=self.hint,

236 default_text=self.defaultText,

237 filename=self.filename,

238 lineno=self.lineno,

239 public=self.public,

240 )

241

242 self.translationCache = self.merge_alias(systemTranslations.get(self.key, {}))

243

244 if (lang := self.force_lang) is None:

245 # The default case: use the request language

246 lang = current.language.get()

247 if value := self.translationCache.get(lang):

248 return value

249 # Use the default text from datastore or from the caller arguments

250 return self.translationCache.get("_default_text_") or self.defaultText

251

252 def translate(self, **kwargs) -> str:

253 """Substitute the given kwargs in the translated or default text."""

254 return self.substitute_vars(str(self), **kwargs)

255

256 def __call__(self, **kwargs):

257 """Just an alias for translate"""

258 return self.translate(**kwargs)

259

260 @staticmethod

261 def substitute_vars(value: str, **kwargs):

262 """Substitute vars in a translation

263

264 Variables has to start with two braces (`{{`), followed by the variable

265 name and end with two braces (`}}`).

266 Values can be anything, they are cast to string anyway.

267 "Hello {{name}}!" becomes with name="Bob": "Hello Bob!"

268 """

269 res = str(value)

270 for k, v in kwargs.items():

271 # 2 braces * (escape + real brace) + 1 for variable = 5

272 res = res.replace(f"{ { {k}} } ", str(v))

273 return res

274

275 @staticmethod

276 def merge_alias(translations: dict[str, str]):

277 """Make sure each aliased language has a value

278

279 If an aliased language does not have a value in the translation dict,

280 the value of the main language is copied.

281 """

282 for alias, main in conf.i18n.language_alias_map.items():

283 if not (value := translations.get(alias)) or not value.strip():

284 if main_value := translations.get(main):

285 # Use only not empty value

286 translations[alias] = main_value

287 return translations

288

289

290class TranslationExtension(jinja2.Extension):

291 """

292 Default translation extension for jinja2 render.

293 Use like {% translate "translationKey", "defaultText", "translationHint", replaceValue1="replacedText1" %}

294 All except translationKey is optional. translationKey is the same Key supplied to _() before.

295 defaultText will be printed if no translation is available.

296 translationHint is an optional hint for anyone adding a now translation how/where that translation is used.

297 `force_lang` can be used as a keyword argument (the only allowed way) to

298 force the use of a specific language, not the language of the request.

299 """

300

301 tags = {

302 "translate",

303 }

304

305 def parse(self, parser):

306 # Parse the translate tag

307 global systemTranslations

308

309 args = [] # positional args for the `_translate()` method

310 kwargs = {} # keyword args (force_lang + substitute vars) for the `_translate()` method

311 lineno = parser.stream.current.lineno

312 filename = parser.stream.filename

313

314 # Parse arguments (args and kwargs) until the current block ends

315 lastToken = None

316 while parser.stream.current.type != 'block_end':

317 lastToken = parser.parse_expression()

318 if parser.stream.current.type == "comma": # It's a positional arg

319 args.append(lastToken.value)

320 next(parser.stream) # Advance pointer

321 lastToken = None

322 elif parser.stream.current.type == "assign":

323 next(parser.stream) # Advance beyond =

324 expr = parser.parse_expression()

325 kwargs[lastToken.name] = expr.value

326 if parser.stream.current.type == "comma":

327 next(parser.stream)

328 elif parser.stream.current.type == "block_end":

329 lastToken = None

330 break

331 else:

332 raise SyntaxError()

333 lastToken = None

334

335 if lastToken: # TODO: what's this? what it is doing?

336 # logging.debug(f"final append {lastToken = }")

337 args.append(lastToken.value)

338

339 if not 0 < len(args) <= 3:

340 raise SyntaxError("Translation-Key missing or excess parameters!")

341

342 args += [""] * (3 - len(args))

343 args += [kwargs]

344 tr_key = args[0].lower()

345 public = kwargs.pop("_public_", False) or False

346

347 if conf.i18n.add_missing_translations and tr_key not in systemTranslations:

348 add_missing_translation(

349 key=tr_key,

350 hint=args[1],

351 default_text=args[2],

352 filename=filename,

353 lineno=lineno,

354 variables=list(kwargs.keys()),

355 public=public,

356 )

357

358 translations = translate.merge_alias(systemTranslations.get(tr_key, {}))

359 args[1] = translations.get("_default_text_") or args[1]

360 args = [jinja2.nodes.Const(x) for x in args]

361 args.append(jinja2.nodes.Const(translations))

362 return jinja2.nodes.CallBlock(self.call_method("_translate", args), [], [], []).set_lineno(lineno)

363

364 def _translate(

365 self, key: str, default_text: str, hint: str, kwargs: dict[str, t.Any],

366 translations: dict[str, str], caller

367 ) -> str:

368 """Perform the actual translation during render"""

369 lang = kwargs.pop("force_lang", current.language.get())

370 res = str(translations.get(lang, default_text))

371 return translate.substitute_vars(res, **kwargs)

372

373

374def initializeTranslations() -> None:

375 """

376 Fetches all translations from the datastore and populates the *systemTranslations* dictionary of this module.

377 Currently, the translate-class will resolve using that dictionary; but as we expect projects to grow and

378 accumulate translations that are no longer/not yet used, we plan to made the translation-class fetch it's

379 translations directly from the datastore, so we don't have to allocate memory for unused translations.

380 """

381 # Load translations from static languages module into systemTranslations

382 # If they're in the datastore, they will be overwritten below.

383 for lang in dir(languages):

384 if lang.startswith("__"):

385 continue

386 for tr_key, tr_value in getattr(languages, lang).items():

387 systemTranslations.setdefault(tr_key, {})[lang] = tr_value

388

389 # Load translations from datastore into systemTranslations

390 # TODO: iter() would be more memory efficient, but unfortunately takes much longer than run()

391 # for entity in db.Query(KINDNAME).iter():

392 for entity in db.Query(KINDNAME).run(10_000):

393 if "tr_key" not in entity:

394 logging.warning(f"translations entity {entity.key} has no tr_key set --> Call migration")

395 migrate_translation(entity.key)

396 # Before the migration has run do a quick modification to get it loaded as is

397 entity["tr_key"] = entity["key"] or entity.key.name

398 if not entity.get("tr_key"):

399 logging.error(f'translations entity {entity.key} has an empty {entity["tr_key"]=} set. Skipping.')

400 continue

401 if entity and not isinstance(entity["translations"], dict):

402 logging.error(f'translations entity {entity.key} has invalid '

403 f'translations set: {entity["translations"]}. Skipping.')

404 continue

405

406 translations = {

407 "_default_text_": entity.get("default_text") or None,

408 "_public_": entity.get("public") or False,

409 }

410

411 for lang, translation in entity["translations"].items():

412 if lang not in conf.i18n.available_dialects:

413 # Don't store unknown languages in the memory

414 continue

415 if not translation or not str(translation).strip():

416 # Skip empty values

417 continue

418 translations[lang] = translation

419

420 systemTranslations[entity["tr_key"]] = translations

421

422

423@tasks.CallDeferred

424@tasks.retry_n_times(20)

425def add_missing_translation(

426 key: str,

427 hint: str | None = None,

428 default_text: str | None = None,

429 filename: str | None = None,

430 lineno: int | None = None,

431 variables: list[str] = None,

432 public: bool = False,

433) -> None:

434 """Add missing translations to datastore"""

435 try:

436 from viur.core.modules.translation import TranslationSkel, Creator

437 except ImportError as exc:

438 # We use translate inside the TranslationSkel, this causes circular dependencies which can be ignored

439 logging.warning(f"ImportError (probably during warmup), "

440 f"cannot add translation {key}: {exc}", exc_info=True)

441 return

442

443 # Ensure lowercase key

444 key = key.lower()

445 entity = db.Query(KINDNAME).filter("tr_key =", key).getEntry()

446 if entity is not None:

447 # Ensure it doesn't exist to avoid datastore conflicts

448 logging.warning(f"Found an entity with tr_key={key}. "

449 f"Probably an other instance was faster.")

450 return

451

452 if isinstance(filename, str):

453 try:

454 filename = str(Path(filename)

455 .relative_to(conf.instance.project_base_path,

456 conf.instance.core_base_path))

457 except ValueError:

458 pass # not a subpath

459

460 logging.info(f"Add missing translation {key}")

461 skel = TranslationSkel()

462 skel["tr_key"] = key

463 skel["default_text"] = default_text or None

464 skel["hint"] = hint or None

465 skel["usage_filename"] = filename

466 skel["usage_lineno"] = lineno

467 skel["usage_variables"] = variables or []

468 skel["creator"] = Creator.VIUR

469 skel["public"] = public

470 skel.write()

471

472 # Add to system translation to avoid triggering this method again

473 systemTranslations[key] = {

474 "_default_text_": default_text or None,

475 "_public_": public,

476 }

477

478

479@tasks.CallDeferred

480@tasks.retry_n_times(20)

481def migrate_translation(

482 key: db.Key,

483) -> None:

484 """Migrate entities, if required.

485

486 With viur-core 3.6 translations are now managed as Skeletons and require

487 some changes, which are performed in this method.

488 """

489 from viur.core.modules.translation import TranslationSkel

490 logging.info(f"Migrate translation {key}")

491 entity: db.Entity = db.Get(key)

492 if "tr_key" not in entity:

493 entity["tr_key"] = entity["key"] or key.name

494 if "translation" in entity:

495 if not isinstance(dict, entity["translation"]):

496 logging.error("translation is not a dict?")

497 entity["translation"]["_viurLanguageWrapper_"] = True

498 skel = TranslationSkel()

499 skel.setEntity(entity)

500 skel["key"] = key

501 try:

502 skel.write()

503 except ValueError as exc:

504 logging.exception(exc)

505 if "unique value" in exc.args[0] and "recently claimed" in exc.args[0]:

506 logging.info(f"Delete duplicate entry {key}: {entity}")

507 db.Delete(key)

508 else:

509 raise exc

510

511

512localizedDateTime = translate("const_datetimeformat", "%a %b %d %H:%M:%S %Y", "Localized Time and Date format string")

513localizedDate = translate("const_dateformat", "%m/%d/%Y", "Localized Date only format string")

514localizedTime = translate("const_timeformat", "%H:%M:%S", "Localized Time only format string")

515localizedAbbrevDayNames = {

516 0: translate("const_day_0_short", "Sun", "Abbreviation for Sunday"),

517 1: translate("const_day_1_short", "Mon", "Abbreviation for Monday"),

518 2: translate("const_day_2_short", "Tue", "Abbreviation for Tuesday"),

519 3: translate("const_day_3_short", "Wed", "Abbreviation for Wednesday"),

520 4: translate("const_day_4_short", "Thu", "Abbreviation for Thursday"),

521 5: translate("const_day_5_short", "Fri", "Abbreviation for Friday"),

522 6: translate("const_day_6_short", "Sat", "Abbreviation for Saturday"),

523}

524localizedDayNames = {

525 0: translate("const_day_0_long", "Sunday", "Sunday"),

526 1: translate("const_day_1_long", "Monday", "Monday"),

527 2: translate("const_day_2_long", "Tuesday", "Tuesday"),

528 3: translate("const_day_3_long", "Wednesday", "Wednesday"),

529 4: translate("const_day_4_long", "Thursday", "Thursday"),

530 5: translate("const_day_5_long", "Friday", "Friday"),

531 6: translate("const_day_6_long", "Saturday", "Saturday"),

532}

533localizedAbbrevMonthNames = {

534 1: translate("const_month_1_short", "Jan", "Abbreviation for January"),

535 2: translate("const_month_2_short", "Feb", "Abbreviation for February"),

536 3: translate("const_month_3_short", "Mar", "Abbreviation for March"),

537 4: translate("const_month_4_short", "Apr", "Abbreviation for April"),

538 5: translate("const_month_5_short", "May", "Abbreviation for May"),

539 6: translate("const_month_6_short", "Jun", "Abbreviation for June"),

540 7: translate("const_month_7_short", "Jul", "Abbreviation for July"),

541 8: translate("const_month_8_short", "Aug", "Abbreviation for August"),

542 9: translate("const_month_9_short", "Sep", "Abbreviation for September"),

543 10: translate("const_month_10_short", "Oct", "Abbreviation for October"),

544 11: translate("const_month_11_short", "Nov", "Abbreviation for November"),

545 12: translate("const_month_12_short", "Dec", "Abbreviation for December"),

546}

547localizedMonthNames = {

548 1: translate("const_month_1_long", "January", "January"),

549 2: translate("const_month_2_long", "February", "February"),

550 3: translate("const_month_3_long", "March", "March"),

551 4: translate("const_month_4_long", "April", "April"),

552 5: translate("const_month_5_long", "May", "May"),

553 6: translate("const_month_6_long", "June", "June"),

554 7: translate("const_month_7_long", "July", "July"),

555 8: translate("const_month_8_long", "August", "August"),

556 9: translate("const_month_9_long", "September", "September"),

557 10: translate("const_month_10_long", "October", "October"),

558 11: translate("const_month_11_long", "November", "November"),

559 12: translate("const_month_12_long", "December", "December"),

560}

561

562

563def localizedStrfTime(datetimeObj: datetime.datetime, format: str) -> str:

564 """

565 Provides correct localized names for directives like %a which don't get translated on GAE properly as we can't

566 set the locale (for each request).

567 This currently replaces %a, %A, %b, %B, %c, %x and %X.

568

569 :param datetimeObj: Datetime-instance to call strftime on

570 :param format: String containing the Format to apply.

571 :returns: Date and time formatted according to format with correct localization

572 """

573 if "%c" in format:

574 format = format.replace("%c", str(localizedDateTime))

575 if "%x" in format:

576 format = format.replace("%x", str(localizedDate))

577 if "%X" in format:

578 format = format.replace("%X", str(localizedTime))

579 if "%a" in format:

580 format = format.replace("%a", str(localizedAbbrevDayNames[int(datetimeObj.strftime("%w"))]))

581 if "%A" in format:

582 format = format.replace("%A", str(localizedDayNames[int(datetimeObj.strftime("%w"))]))

583 if "%b" in format:

584 format = format.replace("%b", str(localizedAbbrevMonthNames[int(datetimeObj.strftime("%m"))]))

585 if "%B" in format:

586 format = format.replace("%B", str(localizedMonthNames[int(datetimeObj.strftime("%m"))]))

587 return datetimeObj.strftime(format)