Coverage for app / backend / src / couchers / i18n / i18next.py: 89%

1""" 

2Implements localizing strings stored in the i18next json format. 

3""" 

4 

5import re 

6from collections.abc import Mapping 

7from dataclasses import dataclass, field 

8from html import escape, unescape 

9from typing import Any 

10 

11from babel import Locale, UnknownLocaleError 

12from markupsafe import Markup 

13 

14PLURALIZABLE_VARIABLE_NAME = "count" 

15"""Special variable name for which i18next supports pluralization forms.""" 

16 

17 

18# A dictionary of values to substitute placeholders with. 

19# str: Default case, will be escaped if localizing with markup. 

20# int: Supports plurals. 

21# Markup: Will be unescaped if localizing without markup. 

22SubstitutionDict = Mapping[str, str | int | Markup] 

23 

24 

25@dataclass 

26class I18Next: 

27 """Retrieves translated strings from their keys based on the i18next format.""" 

28 

29 translations_by_locale: dict[str, Translation] = field(default_factory=dict) 

30 default_translation: Translation | None = None 

31 """The translation used to look up strings in unsupported locales.""" 

32 

33 def add_translation(self, locale: str, *, json_dict: dict[str, Any] | None = None) -> Translation: 

34 translation = Translation(locale) 

35 self.translations_by_locale[locale] = translation 

36 if json_dict: 

37 translation.load_json_dict(json_dict) 

38 return translation 

39 

40 def find_string(self, key: str, locale: str, substitutions: SubstitutionDict | None = None) -> String | None: 

41 """Find the string that will be localized, applying fallbacks and variant selection.""" 

42 translation = self.translations_by_locale.get(locale, self.default_translation) 

43 candidate_translations = [translation] + translation.fallbacks if translation else [] 

44 for candidate_translation in candidate_translations: 

45 if string := candidate_translation.find_string(key, substitutions): 

46 if string.template.can_render(substitutions): 

47 return string 

48 

49 raise LocalizationError(locale, key) 

50 

51 def localize(self, string_key: str, locale: str, substitutions: SubstitutionDict | None = None) -> str: 

52 """Finds a translated string in the best matching locale and performs substitutions.""" 

53 if string := self.find_string(string_key, locale, substitutions): 53 ↛ 56line 53 didn't jump to line 56 because the condition on line 53 was always true

54 return string.render(substitutions) 

55 else: 

56 raise LocalizationError(locale, string_key) 

57 

58 def localize_with_markup( 

59 self, string_key: str, locale: str, substitutions: SubstitutionDict | None = None 

60 ) -> Markup: 

61 """ 

62 Finds a translated string that might contain markup in the best matching locale, 

63 and performs substitutions in a way that results in a markup-safe string. 

64 """ 

65 if string := self.find_string(string_key, locale, substitutions): 65 ↛ 68line 65 didn't jump to line 68 because the condition on line 65 was always true

66 return string.render_with_markup(substitutions) 

67 else: 

68 raise LocalizationError(locale, string_key) 

69 

70 

71@dataclass 

72class Translation: 

73 """A set of translated strings for a locale.""" 

74 

75 locale: str 

76 """The locale, e.g. 'en' or 'fr-CA'""" 

77 strings_by_key: dict[str, String] = field(default_factory=dict) 

78 fallbacks: list[Translation] = field(default_factory=list) 

79 

80 def load_json_dict(self, json_dict: Mapping[str, Any]) -> None: 

81 def add_strings(json_dict: Mapping[str, Any], key_prefix: str | None) -> None: 

82 for k, v in json_dict.items(): 

83 full_key = f"{key_prefix}.{k}" if key_prefix else k 

84 if isinstance(v, str): 

85 self.add_string(full_key, v) 

86 elif isinstance(v, dict): 86 ↛ 89line 86 didn't jump to line 89 because the condition on line 86 was always true

87 add_strings(v, key_prefix=full_key) 

88 else: 

89 raise ValueError(f"Unexpected value type in locale JSON: {type(v)}") 

90 

91 add_strings(json_dict, key_prefix=None) 

92 

93 def add_string(self, key: str, value: str) -> None: 

94 # Weblate and i18next consider an empty string as the absence of a translation. 

95 if value: 

96 self.strings_by_key[key] = String(key, StringTemplate.parse(value)) 

97 

98 def find_string(self, key: str, substitutions: SubstitutionDict | None = None) -> String | None: 

99 # if we have a numerical "count" substitution, 

100 # i18next will first search for a key with a suffix 

101 # based on the plural category suggested by the count 

102 # according to the current locale's rules. 

103 if substitutions: 

104 if count := substitutions.get(PLURALIZABLE_VARIABLE_NAME): 

105 if isinstance(count, int): 105 ↛ 116line 105 didn't jump to line 116 because the condition on line 105 was always true

106 try: 

107 # Babel resolves the locale name into a filename that uses underscores 

108 babel_locale_name = self.locale.replace("-", "_") 

109 plural_form = Locale(babel_locale_name).plural_form(count) 

110 except UnknownLocaleError: 

111 # Fallback to English-style plural rule. 

112 plural_form = "one" if 1 else "other" 

113 plural_key = key + "_" + plural_form 

114 if string := self.strings_by_key.get(plural_key): 

115 return string 

116 return self.strings_by_key.get(key) 

117 

118 def localize(self, string_key: str, substitutions: SubstitutionDict | None = None) -> str: 

119 string = self.find_string(string_key, substitutions) 

120 if string is None: 

121 raise LocalizationError(locale=self.locale, string_key=string_key) 

122 return string.render(substitutions) 

123 

124 def localize_with_markup(self, string_key: str, substitutions: SubstitutionDict | None = None) -> Markup: 

125 string = self.find_string(string_key, substitutions) 

126 if string is None: 

127 raise LocalizationError(locale=self.locale, string_key=string_key) 

128 return string.render_with_markup(substitutions) 

129 

130 

131@dataclass(frozen=True, slots=True) 

132class String: 

133 """An i18next string key + template pair.""" 

134 

135 key: str 

136 template: StringTemplate 

137 

138 def render(self, substitutions: SubstitutionDict | None) -> str: 

139 return self.template.render(substitutions) 

140 

141 def render_with_markup(self, substitutions: SubstitutionDict | None) -> Markup: 

142 return self.template.render_with_markup(substitutions) 

143 

144 

145@dataclass(frozen=True, slots=True) 

146class StringTemplate: 

147 """A string value which may contain variable placeholders.""" 

148 

149 segments: list[StringSegment] 

150 

151 def can_render(self, substitutions: SubstitutionDict | None) -> bool: 

152 for segment in self.segments: 

153 if segment.is_variable: 

154 if not substitutions or substitutions.get(segment.text) is None: 

155 return False 

156 return True 

157 

158 def render(self, substitutions: SubstitutionDict | None) -> str: 

159 return self._render(substitutions, with_markup=False) 

160 

161 def render_with_markup(self, substitutions: SubstitutionDict | None) -> Markup: 

162 return Markup(self._render(substitutions, with_markup=True)) 

163 

164 def _render(self, substitutions: SubstitutionDict | None, with_markup: bool) -> str: 

165 substrings: list[str] = [] 

166 for segment in self.segments: 

167 if segment.is_variable: 

168 if substitutions is not None and segment.text in substitutions: 168 ↛ 180line 168 didn't jump to line 180 because the condition on line 168 was always true

169 value = substitutions[segment.text] 

170 if with_markup and not isinstance(value, Markup) and not isinstance(value, int): 

171 # Auto-escape since we're producing markup 

172 substrings.append(escape(value)) 

173 elif isinstance(value, Markup) and not with_markup: 

174 # We're producing a string where markup is not evaluated 

175 # so resolve escape sequences like " 

176 substrings.append(unescape(value)) 

177 else: 

178 substrings.append(str(value)) 

179 else: 

180 raise ValueError(f"Missing substitution for variable '{segment.text}'") 

181 else: 

182 substrings.append(segment.text) 

183 return "".join(substrings) 

184 

185 @staticmethod 

186 def parse(value: str) -> StringTemplate: 

187 last_index = 0 

188 segments: list[StringSegment] = [] 

189 for match in re.finditer(r"\{\{\s*([^\}]+?)\s*\}\}", value): 

190 if match.start() > last_index: 

191 segments.append(StringSegment(text=value[last_index : match.start()], is_variable=False)) 

192 segments.append(StringSegment(text=match.group(1), is_variable=True)) 

193 last_index = match.end() 

194 if last_index < len(value): 

195 segments.append(StringSegment(text=value[last_index:], is_variable=False)) 

196 return StringTemplate(segments) 

197 

198 

199@dataclass(frozen=True, slots=True) 

200class StringSegment: 

201 """Either a literal text segment or a variable placeholder.""" 

202 

203 text: str 

204 is_variable: bool = False 

205 

206 

207class LocalizationError(Exception): 

208 """Raised failing to localize a string, e.g. if it is not found in any fallback locale.""" 

209 

210 def __init__(self, locale: str, string_key: str): 

211 self.locale = locale 

212 self.string_key = string_key 

213 super().__init__(f"Could not localize string {string_key} for locale {locale}")