Coverage for app/backend/src/tests/test_experimentation.py: 100%

220 statements  

« prev     ^ index     » next       coverage.py v7.17.4, created at 2026-11-21 14:45 +0000

1import json 

2from datetime import date 

3 

4import pytest 

5from google.protobuf import empty_pb2 

6from growthbook.common_types import FeatureResult 

7from sqlalchemy import select 

8 

9from couchers import experimentation 

10from couchers.config import config 

11from couchers.context import make_background_user_context, make_logged_out_context 

12from couchers.db import session_scope 

13from couchers.experimentation import ( 

14 GrowthBookUnavailableError, 

15 _load_local_flags, 

16 _record_feature_usage, 

17 get_targeting_attributes, 

18 setup_experimentation, 

19) 

20from couchers.i18n import LocalizationContext 

21from couchers.metrics import feature_flag_evaluations_counter 

22from couchers.models.logging import ExperimentExposure, ExposureSource, FeatureUsage 

23from couchers.proto import account_pb2, bugs_pb2 

24from tests.fixtures.db import generate_user, make_volunteer 

25from tests.fixtures.feature_flags import FeatureFlags, Flags 

26from tests.fixtures.sessions import account_session, bugs_session 

27 

28 

29def _flag_eval_count(flag_key: str, source: str, value: str) -> float: 

30 return sum( 

31 s.value 

32 for m in feature_flag_evaluations_counter.collect() 

33 for s in m.samples 

34 if s.name == "couchers_feature_flag_evaluations_total" 

35 and s.labels.get("flag_key") == flag_key 

36 and s.labels.get("source") == source 

37 and s.labels.get("value") == value 

38 ) 

39 

40 

41# Raw GrowthBook feature definitions for exercising the framework's own bucketing/exposure mechanics. 

42# Most tests just need feature_flags.set(key, value); these go through feature_flags.set_definition(). 

43# A rollout with explicit coverage: bucketing needs a hash, so anonymous (logged-out) users are 

44# excluded even at 100% coverage and get the feature's default value instead. 

45_ROLLOUT_FLAG = {"defaultValue": "control", "rules": [{"force": "treatment", "coverage": 1.0}]} 

46# A global force with no coverage: applies to everyone, including anonymous users. 

47_GLOBAL_FORCE_FLAG = {"defaultValue": False, "rules": [{"force": True}]} 

48# An actual experiment: a logged-in user gets bucketed (coverage 1), which fires the exposure callback. 

49_EXPERIMENT_FLAG = { 

50 "defaultValue": "control", 

51 "rules": [{"key": "my_experiment", "variations": ["control", "treatment"], "coverage": 1.0}], 

52} 

53 

54# Forced on only for current volunteers, via the is_volunteer targeting attribute. 

55_VOLUNTEERS_FLAG = {"defaultValue": False, "rules": [{"condition": {"is_volunteer": True}, "force": True}]} 

56 

57 

58def test_logged_in_user_is_bucketed_into_rollout(db, feature_flags: FeatureFlags): 

59 feature_flags.set_definition("rollout_flag", _ROLLOUT_FLAG) 

60 user, _ = generate_user() 

61 context = make_background_user_context(user.id) 

62 assert context.get_string_value("rollout_flag", "fallback") == "treatment" 

63 

64 

65def test_anonymous_user_excluded_from_rollout_gets_feature_default(feature_flags: FeatureFlags): 

66 feature_flags.set_definition("rollout_flag", _ROLLOUT_FLAG) 

67 context = make_logged_out_context(LocalizationContext.en_utc()) 

68 # Previously this raised NotLoggedInContextException via context.user_id. 

69 assert context.get_string_value("rollout_flag", "fallback") == "control" 

70 

71 

72def test_anonymous_user_still_gets_global_force_on_flag(feature_flags: FeatureFlags): 

73 feature_flags.set_definition("global_flag", _GLOBAL_FORCE_FLAG) 

74 context = make_logged_out_context(LocalizationContext.en_utc()) 

75 assert context.get_boolean_value("global_flag", default=False) is True 

76 

77 

78def test_unknown_feature_returns_in_code_default(feature_flags: FeatureFlags): 

79 context = make_logged_out_context(LocalizationContext.en_utc()) 

80 assert context.get_string_value("does_not_exist", "my_default") == "my_default" 

81 

82 

83def test_evaluating_an_experiment_flag_records_exactly_one_exposure(db, feature_flags: FeatureFlags): 

84 # Evaluating an experiment-backed flag for a bucketed user records exactly one exposure - this is 

85 # the whole point of per-flag evaluation: exposure is logged only for flags the user actually hits. 

86 feature_flags.set_definition("experiment_flag", _EXPERIMENT_FLAG) 

87 user, _ = generate_user() 

88 context = make_background_user_context(user.id) 

89 assert context.get_object_value("experiment_flag", "control") in {"control", "treatment"} 

90 

91 with session_scope() as session: 

92 rows = session.execute(select(ExperimentExposure).where(ExperimentExposure.user_id == user.id)).scalars().all() 

93 assert len(rows) == 1 

94 assert rows[0].experiment_key == "my_experiment" 

95 assert rows[0].source == ExposureSource.backend 

96 

97 

98def test_targeting_attributes(db): 

99 user, _ = generate_user() 

100 volunteer, _ = generate_user() 

101 past_volunteer, _ = generate_user() 

102 with session_scope() as session: 

103 session.add(make_volunteer(user_id=volunteer.id, role="Tester", started_volunteering=date(2020, 1, 1))) 

104 session.add( 

105 make_volunteer( 

106 user_id=past_volunteer.id, 

107 role="Tester", 

108 started_volunteering=date(2020, 1, 1), 

109 stopped_volunteering=date(2021, 1, 1), 

110 ) 

111 ) 

112 

113 assert get_targeting_attributes(None) == {"logged_in": False} 

114 assert get_targeting_attributes(user.id) == { 

115 "id": str(user.id), 

116 "logged_in": True, 

117 "is_volunteer": False, 

118 "is_beta": False, 

119 } 

120 assert get_targeting_attributes(volunteer.id) == { 

121 "id": str(volunteer.id), 

122 "logged_in": True, 

123 "is_volunteer": True, 

124 "is_beta": False, 

125 } 

126 assert get_targeting_attributes(past_volunteer.id) == { 

127 "id": str(past_volunteer.id), 

128 "logged_in": True, 

129 "is_volunteer": False, 

130 "is_beta": False, 

131 } 

132 

133 

134def test_targeting_attributes_is_beta(db): 

135 user, token = generate_user() 

136 with account_session(token) as account: 

137 account.SetBetaFeaturesOptIn(account_pb2.SetBetaFeaturesOptInReq(beta_features_opt_in=True)) 

138 assert get_targeting_attributes(user.id)["is_beta"] is True 

139 

140 

141def test_flag_targeted_at_beta_users(db, feature_flags: FeatureFlags): 

142 feature_flags.set_definition( 

143 "beta_flag", {"defaultValue": False, "rules": [{"condition": {"is_beta": True}, "force": True}]} 

144 ) 

145 user, token = generate_user() 

146 assert make_background_user_context(user.id).get_boolean_value("beta_flag", default=False) is False 

147 with account_session(token) as account: 

148 account.SetBetaFeaturesOptIn(account_pb2.SetBetaFeaturesOptInReq(beta_features_opt_in=True)) 

149 assert make_background_user_context(user.id).get_boolean_value("beta_flag", default=False) is True 

150 

151 

152def test_flag_targeted_at_volunteers(db, feature_flags: FeatureFlags): 

153 feature_flags.set_definition("volunteers_flag", _VOLUNTEERS_FLAG) 

154 user, _ = generate_user() 

155 volunteer, _ = generate_user() 

156 with session_scope() as session: 

157 session.add(make_volunteer(user_id=volunteer.id, role="Tester", started_volunteering=date(2020, 1, 1))) 

158 

159 assert make_background_user_context(volunteer.id).get_boolean_value("volunteers_flag", default=False) is True 

160 assert make_background_user_context(user.id).get_boolean_value("volunteers_flag", default=False) is False 

161 logged_out = make_logged_out_context(LocalizationContext.en_utc()) 

162 assert logged_out.get_boolean_value("volunteers_flag", default=False) is False 

163 

164 

165def test_get_feature_flag_attributes_servicer(db): 

166 user, token = generate_user() 

167 with bugs_session(token) as bugs: 

168 res = bugs.GetFeatureFlagAttributes(empty_pb2.Empty()) 

169 assert json.loads(res.attributes_json) == { 

170 "id": str(user.id), 

171 "logged_in": True, 

172 "is_volunteer": False, 

173 "is_beta": False, 

174 } 

175 

176 with bugs_session() as bugs: 

177 res = bugs.GetFeatureFlagAttributes(empty_pb2.Empty()) 

178 assert json.loads(res.attributes_json) == {"logged_in": False} 

179 

180 

181def test_evaluate_feature_flag_servicer_returns_value(feature_flags: FeatureFlags, db): 

182 feature_flags.set("global_flag", True) 

183 with bugs_session() as bugs: 

184 res = bugs.EvaluateFeatureFlag(bugs_pb2.EvaluateFeatureFlagReq(flag_key="global_flag")) 

185 assert res.value.bool_value is True 

186 

187 

188def test_evaluate_feature_flag_servicer_unknown_leaves_value_unset(feature_flags: FeatureFlags, db): 

189 with bugs_session() as bugs: 

190 res = bugs.EvaluateFeatureFlag(bugs_pb2.EvaluateFeatureFlagReq(flag_key="does_not_exist")) 

191 assert not res.HasField("value") 

192 

193 

194def _get_usage(session, user_id): 

195 return ( 

196 session.execute(select(FeatureUsage).where(FeatureUsage.user_id == user_id).order_by(FeatureUsage.id)) 

197 .scalars() 

198 .all() 

199 ) 

200 

201 

202def test_record_feature_usage_appends_a_row(db): 

203 _record_feature_usage(1, "my_feature", FeatureResult(value=True, source="defaultValue")) 

204 

205 with session_scope() as session: 

206 rows = _get_usage(session, 1) 

207 assert len(rows) == 1 

208 assert rows[0].feature_key == "my_feature" 

209 assert rows[0].value is True 

210 assert rows[0].time is not None 

211 

212 

213def test_record_feature_usage_appends_a_row_per_check(db): 

214 # every check appends - the log is append-only, not deduplicated per (user, feature) 

215 _record_feature_usage(1, "my_feature", FeatureResult(value="first", source="force")) 

216 _record_feature_usage(1, "my_feature", FeatureResult(value="second", source="force")) 

217 

218 with session_scope() as session: 

219 rows = _get_usage(session, 1) 

220 assert len(rows) == 2 

221 assert [row.value for row in rows] == ["first", "second"] 

222 

223 

224def test_record_feature_usage_records_each_user_and_feature(db): 

225 _record_feature_usage(1, "feature_a", FeatureResult(value=1, source="force")) 

226 _record_feature_usage(1, "feature_b", FeatureResult(value=2, source="force")) 

227 _record_feature_usage(2, "feature_a", FeatureResult(value=3, source="force")) 

228 

229 with session_scope() as session: 

230 assert {row.feature_key for row in _get_usage(session, 1)} == {"feature_a", "feature_b"} 

231 assert len(_get_usage(session, 2)) == 1 

232 

233 

234def test_record_feature_usage_none_value(db): 

235 # unknown features evaluate to a None value - must persist without violating NOT NULL 

236 _record_feature_usage(1, "unknown_feature", FeatureResult(value=None, source="unknownFeature")) 

237 

238 with session_scope() as session: 

239 rows = _get_usage(session, 1) 

240 assert len(rows) == 1 

241 assert rows[0].value is None 

242 

243 

244def test_global_evaluation_excluded_from_rollout_gets_feature_default(feature_flags: FeatureFlags): 

245 # global (no-user) evaluation can't bucket into a rollout, so it gets the feature default 

246 feature_flags.set_definition("rollout_flag", _ROLLOUT_FLAG) 

247 assert experimentation.get_global_string_value("rollout_flag", "fallback") == "control" 

248 

249 

250def test_global_evaluation_gets_global_force_on_flag(feature_flags: FeatureFlags): 

251 feature_flags.set_definition("global_flag", _GLOBAL_FORCE_FLAG) 

252 assert experimentation.get_global_boolean_value("global_flag", default=False) is True 

253 

254 

255def test_global_evaluation_unknown_feature_returns_in_code_default(feature_flags: FeatureFlags): 

256 assert experimentation.get_global_string_value("does_not_exist", "my_default") == "my_default" 

257 

258 

259def test_evaluation_increments_metric_with_source_and_value(feature_flags: FeatureFlags): 

260 feature_flags.set("metric_flag", "yes") 

261 before = _flag_eval_count("metric_flag", "defaultValue", "yes") 

262 assert experimentation.get_global_string_value("metric_flag", "no") == "yes" 

263 assert _flag_eval_count("metric_flag", "defaultValue", "yes") == before + 1 

264 

265 

266def test_unknown_feature_increments_metric_with_unknown_source(feature_flags: FeatureFlags): 

267 before = _flag_eval_count("metric_unknown_flag", "unknownFeature", "fallback") 

268 assert experimentation.get_global_string_value("metric_unknown_flag", "fallback") == "fallback" 

269 assert _flag_eval_count("metric_unknown_flag", "unknownFeature", "fallback") == before + 1 

270 

271 

272@pytest.fixture 

273def setup_isolation(monkeypatch, tmp_path): 

274 """Run setup_experimentation() against a clean module state and a tmp cache path, and make sure the 

275 background refresh thread it starts is stopped afterwards.""" 

276 monkeypatch.setattr(experimentation, "_initialized", False) 

277 monkeypatch.setattr(experimentation, "_last_fetch_time", None) 

278 monkeypatch.setattr(experimentation, "_state", {"features": {}, "savedGroups": {}}) 

279 monkeypatch.setitem(config, "FEATURE_FLAGS_FILE_OVERRIDE_PATH", "") 

280 monkeypatch.setitem(config, "GROWTHBOOK_CACHE_PATH", str(tmp_path / "cache.json")) 

281 yield tmp_path / "cache.json" 

282 experimentation._refresh_stop.set() 

283 if experimentation._refresh_thread is not None: 

284 experimentation._refresh_thread.join(timeout=5) 

285 experimentation._refresh_stop.clear() 

286 experimentation._refresh_thread = None 

287 

288 

289def test_setup_writes_cache_and_records_fetch_time(setup_isolation, monkeypatch): 

290 cache = setup_isolation 

291 payload = {"features": {"f": {"defaultValue": True}}, "savedGroups": {}} 

292 monkeypatch.setattr(experimentation, "_fetch_features", lambda: payload) 

293 

294 setup_experimentation() 

295 

296 assert experimentation._state["features"] == {"f": {"defaultValue": True}} 

297 assert experimentation.seconds_since_last_fetch() is not None 

298 written = json.loads(cache.read_text()) 

299 assert written["response"] == payload 

300 assert "fetched_at" in written 

301 

302 

303def test_setup_falls_back_to_disk_cache_when_fetch_fails(setup_isolation, monkeypatch): 

304 cache = setup_isolation 

305 cached_payload = {"features": {"cached": {"defaultValue": "x"}}, "savedGroups": {}} 

306 cache.write_text(json.dumps({"fetched_at": 1000.0, "response": cached_payload})) 

307 monkeypatch.setattr(experimentation, "_fetch_features", lambda: None) 

308 

309 setup_experimentation() 

310 

311 assert experimentation._state["features"] == {"cached": {"defaultValue": "x"}} 

312 # fetch time reflects the cached pull time, so staleness is large immediately 

313 staleness = experimentation.seconds_since_last_fetch() 

314 assert staleness is not None and staleness > 0 

315 

316 

317def test_setup_raises_when_fetch_fails_and_no_cache(setup_isolation, monkeypatch): 

318 monkeypatch.setattr(experimentation, "_fetch_features", lambda: None) 

319 

320 with pytest.raises(GrowthBookUnavailableError): 

321 setup_experimentation() 

322 

323 

324def test_setup_raises_on_corrupt_cache(setup_isolation, monkeypatch): 

325 cache = setup_isolation 

326 cache.write_text("this is not json") 

327 monkeypatch.setattr(experimentation, "_fetch_features", lambda: None) 

328 

329 with pytest.raises(json.JSONDecodeError): 

330 setup_experimentation() 

331 

332 

333def test_seconds_since_last_fetch_none_when_never_fetched(setup_isolation): 

334 assert experimentation.seconds_since_last_fetch() is None 

335 

336 

337def test_flags_value_returned(flags: Flags): 

338 flags.set_string("my_flag", "from_file") 

339 context = make_logged_out_context(LocalizationContext.en_utc()) 

340 assert context.get_string_value("my_flag", "fallback") == "from_file" 

341 

342 

343def test_flags_missing_key_returns_in_code_default(flags: Flags): 

344 context = make_logged_out_context(LocalizationContext.en_utc()) 

345 assert context.get_string_value("missing_flag", "fallback") == "fallback" 

346 

347 

348def test_flags_boolean_value(flags: Flags): 

349 flags.set_boolean("bool_flag", False) 

350 context = make_logged_out_context(LocalizationContext.en_utc()) 

351 assert context.get_boolean_value("bool_flag", default=True) is False 

352 

353 

354def test_load_local_flags_from_file(tmp_path): 

355 path = tmp_path / "flags.json" 

356 path.write_text(json.dumps({"flag_a": True, "flag_b": "hello", "flag_c": 42})) 

357 assert _load_local_flags(str(path)) == {"flag_a": True, "flag_b": "hello", "flag_c": 42} 

358 

359 

360def test_load_local_flags_rejects_non_object(tmp_path): 

361 path = tmp_path / "flags.json" 

362 path.write_text(json.dumps(["not", "an", "object"])) 

363 with pytest.raises(ValueError, match="must contain a JSON object"): 

364 _load_local_flags(str(path)) 

365 

366 

367def test_setup_in_local_file_mode_loads_file_and_skips_growthbook(monkeypatch, tmp_path): 

368 path = tmp_path / "flags.json" 

369 path.write_text(json.dumps({"flag_x": "from_file"})) 

370 monkeypatch.setattr(experimentation, "_initialized", False) 

371 monkeypatch.setitem(config, "FEATURE_FLAGS_FILE_OVERRIDE_PATH", str(path)) 

372 # If GrowthBook were touched, this would blow up. 

373 monkeypatch.setattr(experimentation, "_fetch_features", lambda: pytest.fail("GrowthBook should not be touched")) 

374 

375 setup_experimentation() 

376 

377 assert _load_local_flags(str(path)) == {"flag_x": "from_file"} 

378 assert experimentation._refresh_thread is None