Coverage for app / backend / src / couchers / i18n / i18next.py: 89%

1""" 

2Implements localizing strings stored in the i18next json format. 

3""" 

4 

5import re 

6from collections.abc import Mapping 

7from dataclasses import dataclass, field 

8from html import escape, unescape 

9from typing import Any 

10 

11from babel import Locale, UnknownLocaleError 

12from markupsafe import Markup 

13 

14PLURALIZABLE_VARIABLE_NAME = "count" 

15"""Special variable name for which i18next supports pluralization forms.""" 

16 

17 

18# A dictionary of values to substitute placeholders with. 

19# str: Default case, will be escaped if localizing with markup. 

20# int: Supports plurals. 

21# Markup: Will be unescaped if localizing without markup. 

22SubstitutionDict = Mapping[str, str | int | Markup] 

23 

24 

25@dataclass 

26class I18Next: 

27 """Retrieves translated strings from their keys based on the i18next format.""" 

28 

29 translations_by_locale: dict[str, Translation] = field(default_factory=dict) 

30 default_translation: Translation | None = None 

31 """The translation used to look up strings in unsupported locales.""" 

32 

33 def add_translation(self, locale: str, *, json_dict: dict[str, Any] | None = None) -> Translation: 

34 translation = Translation(locale) 

35 self.translations_by_locale[locale] = translation 

36 if json_dict: 

37 translation.load_json_dict(json_dict) 

38 return translation 

39 

40 def find_string(self, key: str, locale: str, substitutions: SubstitutionDict | None = None) -> String | None: 

41 """Find the string that will be localized, applying fallbacks and variant selection.""" 

42 translation = self.translations_by_locale.get(locale, self.default_translation) 

43 candidate_translations = [translation] + translation.fallbacks if translation else [] 

44 for candidate_translation in candidate_translations: 

45 if string := candidate_translation.find_string(key, substitutions): 

46 if string.template.can_render(substitutions): 

47 return string 

48 

49 raise LocalizationError(locale, key) 

50 

51 def localize(self, string_key: str, locale: str, substitutions: SubstitutionDict | None = None) -> str: 

52 """Finds a translated string in the best matching locale and performs substitutions.""" 

53 if string := self.find_string(string_key, locale, substitutions): 53 ↛ 56line 53 didn't jump to line 56 because the condition on line 53 was always true

54 return string.render(substitutions) 

55 else: 

56 raise LocalizationError(locale, string_key) 

57 

58 def localize_with_markup( 

59 self, string_key: str, locale: str, substitutions: SubstitutionDict | None = None 

60 ) -> Markup: 

61 """ 

62 Finds a translated string that might contain markup in the best matching locale, 

63 and performs substitutions in a way that results in a markup-safe string. 

64 """ 

65 if string := self.find_string(string_key, locale, substitutions): 65 ↛ 68line 65 didn't jump to line 68 because the condition on line 65 was always true

66 return string.render_with_markup(substitutions) 

67 else: 

68 raise LocalizationError(locale, string_key) 

69 

70 

71@dataclass 

72class Translation: 

73 """A set of translated strings for a locale.""" 

74 

75 locale: str 

76 """The locale, e.g. 'en' or 'fr-CA'""" 

77 strings_by_key: dict[str, String] = field(default_factory=dict) 

78 fallbacks: list[Translation] = field(default_factory=list) 

79 

80 def load_json_dict(self, json_dict: Mapping[str, Any]) -> None: 

81 def add_strings(json_dict: Mapping[str, Any], key_prefix: str | None) -> None: 

82 for k, v in json_dict.items(): 

83 full_key = f"{key_prefix}.{k}" if key_prefix else k 

84 if isinstance(v, str): 

85 self.add_string(full_key, v) 

86 elif isinstance(v, dict): 86 ↛ 89line 86 didn't jump to line 89 because the condition on line 86 was always true

87 add_strings(v, key_prefix=full_key) 

88 else: 

89 raise ValueError(f"Unexpected value type in locale JSON: {type(v)}") 

90 

91 add_strings(json_dict, key_prefix=None) 

92 

93 def add_string(self, key: str, value: str) -> None: 

94 # Weblate and i18next consider an empty string as the absence of a translation. 

95 if value: 

96 self.strings_by_key[key] = String(key, StringTemplate.parse(value)) 

97 

98 def find_string(self, key: str, substitutions: SubstitutionDict | None = None) -> String | None: 

99 # if we have a numerical "count" substitution, 

100 # i18next will first search for a key with a suffix 

101 # based on the plural category suggested by the count 

102 # according to the current locale's rules. 

103 if substitutions: 

104 if count := substitutions.get(PLURALIZABLE_VARIABLE_NAME): 

105 if isinstance(count, int): 105 ↛ 114line 105 didn't jump to line 114 because the condition on line 105 was always true

106 try: 

107 plural_form = Locale(self.locale).plural_form(count) 

108 except UnknownLocaleError: 

109 # Fallback to English-style plural rule. 

110 plural_form = "one" if 1 else "other" 

111 plural_key = key + "_" + plural_form 

112 if string := self.strings_by_key.get(plural_key): 

113 return string 

114 return self.strings_by_key.get(key) 

115 

116 def localize(self, string_key: str, substitutions: SubstitutionDict | None = None) -> str: 

117 string = self.find_string(string_key, substitutions) 

118 if string is None: 

119 raise LocalizationError(locale=self.locale, string_key=string_key) 

120 return string.render(substitutions) 

121 

122 def localize_with_markup(self, string_key: str, substitutions: SubstitutionDict | None = None) -> Markup: 

123 string = self.find_string(string_key, substitutions) 

124 if string is None: 

125 raise LocalizationError(locale=self.locale, string_key=string_key) 

126 return string.render_with_markup(substitutions) 

127 

128 

129@dataclass(frozen=True, slots=True) 

130class String: 

131 """An i18next string key + template pair.""" 

132 

133 key: str 

134 template: StringTemplate 

135 

136 def render(self, substitutions: SubstitutionDict | None) -> str: 

137 return self.template.render(substitutions) 

138 

139 def render_with_markup(self, substitutions: SubstitutionDict | None) -> Markup: 

140 return self.template.render_with_markup(substitutions) 

141 

142 

143@dataclass(frozen=True, slots=True) 

144class StringTemplate: 

145 """A string value which may contain variable placeholders.""" 

146 

147 segments: list[StringSegment] 

148 

149 def can_render(self, substitutions: SubstitutionDict | None) -> bool: 

150 for segment in self.segments: 

151 if segment.is_variable: 

152 if not substitutions or substitutions.get(segment.text) is None: 

153 return False 

154 return True 

155 

156 def render(self, substitutions: SubstitutionDict | None) -> str: 

157 return self._render(substitutions, with_markup=False) 

158 

159 def render_with_markup(self, substitutions: SubstitutionDict | None) -> Markup: 

160 return Markup(self._render(substitutions, with_markup=True)) 

161 

162 def _render(self, substitutions: SubstitutionDict | None, with_markup: bool) -> str: 

163 substrings: list[str] = [] 

164 for segment in self.segments: 

165 if segment.is_variable: 

166 if substitutions is not None and segment.text in substitutions: 166 ↛ 178line 166 didn't jump to line 178 because the condition on line 166 was always true

167 value = substitutions[segment.text] 

168 if with_markup and not isinstance(value, Markup) and not isinstance(value, int): 

169 # Auto-escape since we're producing markup 

170 substrings.append(escape(value)) 

171 elif isinstance(value, Markup) and not with_markup: 

172 # We're producing a string where markup is not evaluated 

173 # so resolve escape sequences like " 

174 substrings.append(unescape(value)) 

175 else: 

176 substrings.append(str(value)) 

177 else: 

178 raise ValueError(f"Missing substitution for variable '{segment.text}'") 

179 else: 

180 substrings.append(segment.text) 

181 return "".join(substrings) 

182 

183 @staticmethod 

184 def parse(value: str) -> StringTemplate: 

185 last_index = 0 

186 segments: list[StringSegment] = [] 

187 for match in re.finditer(r"\{\{\s*([^\}]+?)\s*\}\}", value): 

188 if match.start() > last_index: 

189 segments.append(StringSegment(text=value[last_index : match.start()], is_variable=False)) 

190 segments.append(StringSegment(text=match.group(1), is_variable=True)) 

191 last_index = match.end() 

192 if last_index < len(value): 

193 segments.append(StringSegment(text=value[last_index:], is_variable=False)) 

194 return StringTemplate(segments) 

195 

196 

197@dataclass(frozen=True, slots=True) 

198class StringSegment: 

199 """Either a literal text segment or a variable placeholder.""" 

200 

201 text: str 

202 is_variable: bool = False 

203 

204 

205class LocalizationError(Exception): 

206 """Raised failing to localize a string, e.g. if it is not found in any fallback locale.""" 

207 

208 def __init__(self, locale: str, string_key: str): 

209 self.locale = locale 

210 self.string_key = string_key 

211 super().__init__(f"Could not localize string {string_key} for locale {locale}")