Coverage for app/backend/src/tests/test_experimentation.py: 100%

1import json 

2 

3import pytest 

4from growthbook.common_types import FeatureResult 

5from sqlalchemy import select 

6 

7from couchers import experimentation 

8from couchers.config import config 

9from couchers.context import make_background_user_context, make_logged_out_context 

10from couchers.db import session_scope 

11from couchers.experimentation import ( 

12 GrowthBookUnavailableError, 

13 _load_local_flags, 

14 _record_feature_usage, 

15 setup_experimentation, 

16) 

17from couchers.i18n import LocalizationContext 

18from couchers.metrics import feature_flag_evaluations_counter 

19from couchers.models.logging import ExperimentExposure, ExposureSource, FeatureUsage 

20from couchers.proto import bugs_pb2 

21from tests.fixtures.sessions import bugs_session 

22 

23 

24def _flag_eval_count(flag_key: str, source: str, value: str) -> float: 

25 return sum( 

26 s.value 

27 for m in feature_flag_evaluations_counter.collect() 

28 for s in m.samples 

29 if s.name == "couchers_feature_flag_evaluations_total" 

30 and s.labels.get("flag_key") == flag_key 

31 and s.labels.get("source") == source 

32 and s.labels.get("value") == value 

33 ) 

34 

35 

36# Raw GrowthBook feature definitions for exercising the framework's own bucketing/exposure mechanics. 

37# Most tests just need feature_flags.set(key, value); these go through feature_flags.set_definition(). 

38# A rollout with explicit coverage: bucketing needs a hash, so anonymous (logged-out) users are 

39# excluded even at 100% coverage and get the feature's default value instead. 

40_ROLLOUT_FLAG = {"defaultValue": "control", "rules": [{"force": "treatment", "coverage": 1.0}]} 

41# A global force with no coverage: applies to everyone, including anonymous users. 

42_GLOBAL_FORCE_FLAG = {"defaultValue": False, "rules": [{"force": True}]} 

43# An actual experiment: a logged-in user gets bucketed (coverage 1), which fires the exposure callback. 

44_EXPERIMENT_FLAG = { 

45 "defaultValue": "control", 

46 "rules": [{"key": "my_experiment", "variations": ["control", "treatment"], "coverage": 1.0}], 

47} 

48 

49 

50def test_logged_in_user_is_bucketed_into_rollout(db, feature_flags): 

51 feature_flags.set_definition("rollout_flag", _ROLLOUT_FLAG) 

52 context = make_background_user_context(123) 

53 assert context.get_string_value("rollout_flag", "fallback") == "treatment" 

54 

55 

56def test_anonymous_user_excluded_from_rollout_gets_feature_default(feature_flags): 

57 feature_flags.set_definition("rollout_flag", _ROLLOUT_FLAG) 

58 context = make_logged_out_context(LocalizationContext.en_utc()) 

59 # Previously this raised NotLoggedInContextException via context.user_id. 

60 assert context.get_string_value("rollout_flag", "fallback") == "control" 

61 

62 

63def test_anonymous_user_still_gets_global_force_on_flag(feature_flags): 

64 feature_flags.set_definition("global_flag", _GLOBAL_FORCE_FLAG) 

65 context = make_logged_out_context(LocalizationContext.en_utc()) 

66 assert context.get_boolean_value("global_flag", default=False) is True 

67 

68 

69def test_unknown_feature_returns_in_code_default(feature_flags): 

70 context = make_logged_out_context(LocalizationContext.en_utc()) 

71 assert context.get_string_value("does_not_exist", "my_default") == "my_default" 

72 

73 

74def test_evaluating_an_experiment_flag_records_exactly_one_exposure(db, feature_flags): 

75 # Evaluating an experiment-backed flag for a bucketed user records exactly one exposure - this is 

76 # the whole point of per-flag evaluation: exposure is logged only for flags the user actually hits. 

77 feature_flags.set_definition("experiment_flag", _EXPERIMENT_FLAG) 

78 context = make_background_user_context(123) 

79 assert context.get_object_value("experiment_flag", "control") in {"control", "treatment"} 

80 

81 with session_scope() as session: 

82 rows = session.execute(select(ExperimentExposure).where(ExperimentExposure.user_id == 123)).scalars().all() 

83 assert len(rows) == 1 

84 assert rows[0].experiment_key == "my_experiment" 

85 assert rows[0].source == ExposureSource.backend 

86 

87 

88def test_evaluate_feature_flag_servicer_returns_value(feature_flags, db): 

89 feature_flags.set("global_flag", True) 

90 with bugs_session() as bugs: 

91 res = bugs.EvaluateFeatureFlag(bugs_pb2.EvaluateFeatureFlagReq(flag_key="global_flag")) 

92 assert res.value.bool_value is True 

93 

94 

95def test_evaluate_feature_flag_servicer_unknown_leaves_value_unset(feature_flags, db): 

96 with bugs_session() as bugs: 

97 res = bugs.EvaluateFeatureFlag(bugs_pb2.EvaluateFeatureFlagReq(flag_key="does_not_exist")) 

98 assert not res.HasField("value") 

99 

100 

101def _get_usage(session, user_id): 

102 return ( 

103 session.execute(select(FeatureUsage).where(FeatureUsage.user_id == user_id).order_by(FeatureUsage.id)) 

104 .scalars() 

105 .all() 

106 ) 

107 

108 

109def test_record_feature_usage_appends_a_row(db): 

110 _record_feature_usage(1, "my_feature", FeatureResult(value=True, source="defaultValue")) 

111 

112 with session_scope() as session: 

113 rows = _get_usage(session, 1) 

114 assert len(rows) == 1 

115 assert rows[0].feature_key == "my_feature" 

116 assert rows[0].value is True 

117 assert rows[0].time is not None 

118 

119 

120def test_record_feature_usage_appends_a_row_per_check(db): 

121 # every check appends - the log is append-only, not deduplicated per (user, feature) 

122 _record_feature_usage(1, "my_feature", FeatureResult(value="first", source="force")) 

123 _record_feature_usage(1, "my_feature", FeatureResult(value="second", source="force")) 

124 

125 with session_scope() as session: 

126 rows = _get_usage(session, 1) 

127 assert len(rows) == 2 

128 assert [row.value for row in rows] == ["first", "second"] 

129 

130 

131def test_record_feature_usage_records_each_user_and_feature(db): 

132 _record_feature_usage(1, "feature_a", FeatureResult(value=1, source="force")) 

133 _record_feature_usage(1, "feature_b", FeatureResult(value=2, source="force")) 

134 _record_feature_usage(2, "feature_a", FeatureResult(value=3, source="force")) 

135 

136 with session_scope() as session: 

137 assert {row.feature_key for row in _get_usage(session, 1)} == {"feature_a", "feature_b"} 

138 assert len(_get_usage(session, 2)) == 1 

139 

140 

141def test_record_feature_usage_none_value(db): 

142 # unknown features evaluate to a None value - must persist without violating NOT NULL 

143 _record_feature_usage(1, "unknown_feature", FeatureResult(value=None, source="unknownFeature")) 

144 

145 with session_scope() as session: 

146 rows = _get_usage(session, 1) 

147 assert len(rows) == 1 

148 assert rows[0].value is None 

149 

150 

151def test_global_evaluation_excluded_from_rollout_gets_feature_default(feature_flags): 

152 # global (no-user) evaluation can't bucket into a rollout, so it gets the feature default 

153 feature_flags.set_definition("rollout_flag", _ROLLOUT_FLAG) 

154 assert experimentation.get_global_string_value("rollout_flag", "fallback") == "control" 

155 

156 

157def test_global_evaluation_gets_global_force_on_flag(feature_flags): 

158 feature_flags.set_definition("global_flag", _GLOBAL_FORCE_FLAG) 

159 assert experimentation.get_global_boolean_value("global_flag", default=False) is True 

160 

161 

162def test_global_evaluation_unknown_feature_returns_in_code_default(feature_flags): 

163 assert experimentation.get_global_string_value("does_not_exist", "my_default") == "my_default" 

164 

165 

166def test_evaluation_increments_metric_with_source_and_value(feature_flags): 

167 feature_flags.set("metric_flag", "yes") 

168 before = _flag_eval_count("metric_flag", "defaultValue", "yes") 

169 assert experimentation.get_global_string_value("metric_flag", "no") == "yes" 

170 assert _flag_eval_count("metric_flag", "defaultValue", "yes") == before + 1 

171 

172 

173def test_unknown_feature_increments_metric_with_unknown_source(feature_flags): 

174 before = _flag_eval_count("metric_unknown_flag", "unknownFeature", "fallback") 

175 assert experimentation.get_global_string_value("metric_unknown_flag", "fallback") == "fallback" 

176 assert _flag_eval_count("metric_unknown_flag", "unknownFeature", "fallback") == before + 1 

177 

178 

179@pytest.fixture 

180def setup_isolation(monkeypatch, tmp_path): 

181 """Run setup_experimentation() against a clean module state and a tmp cache path, and make sure the 

182 background refresh thread it starts is stopped afterwards.""" 

183 monkeypatch.setattr(experimentation, "_initialized", False) 

184 monkeypatch.setattr(experimentation, "_last_fetch_time", None) 

185 monkeypatch.setattr(experimentation, "_state", {"features": {}, "savedGroups": {}}) 

186 monkeypatch.setitem(config, "FEATURE_FLAGS_FILE_OVERRIDE_PATH", "") 

187 monkeypatch.setitem(config, "GROWTHBOOK_CACHE_PATH", str(tmp_path / "cache.json")) 

188 yield tmp_path / "cache.json" 

189 experimentation._refresh_stop.set() 

190 if experimentation._refresh_thread is not None: 

191 experimentation._refresh_thread.join(timeout=5) 

192 experimentation._refresh_stop.clear() 

193 experimentation._refresh_thread = None 

194 

195 

196def test_setup_writes_cache_and_records_fetch_time(setup_isolation, monkeypatch): 

197 cache = setup_isolation 

198 payload = {"features": {"f": {"defaultValue": True}}, "savedGroups": {}} 

199 monkeypatch.setattr(experimentation, "_fetch_features", lambda: payload) 

200 

201 setup_experimentation() 

202 

203 assert experimentation._state["features"] == {"f": {"defaultValue": True}} 

204 assert experimentation.seconds_since_last_fetch() is not None 

205 written = json.loads(cache.read_text()) 

206 assert written["response"] == payload 

207 assert "fetched_at" in written 

208 

209 

210def test_setup_falls_back_to_disk_cache_when_fetch_fails(setup_isolation, monkeypatch): 

211 cache = setup_isolation 

212 cached_payload = {"features": {"cached": {"defaultValue": "x"}}, "savedGroups": {}} 

213 cache.write_text(json.dumps({"fetched_at": 1000.0, "response": cached_payload})) 

214 monkeypatch.setattr(experimentation, "_fetch_features", lambda: None) 

215 

216 setup_experimentation() 

217 

218 assert experimentation._state["features"] == {"cached": {"defaultValue": "x"}} 

219 # fetch time reflects the cached pull time, so staleness is large immediately 

220 staleness = experimentation.seconds_since_last_fetch() 

221 assert staleness is not None and staleness > 0 

222 

223 

224def test_setup_raises_when_fetch_fails_and_no_cache(setup_isolation, monkeypatch): 

225 monkeypatch.setattr(experimentation, "_fetch_features", lambda: None) 

226 

227 with pytest.raises(GrowthBookUnavailableError): 

228 setup_experimentation() 

229 

230 

231def test_setup_raises_on_corrupt_cache(setup_isolation, monkeypatch): 

232 cache = setup_isolation 

233 cache.write_text("this is not json") 

234 monkeypatch.setattr(experimentation, "_fetch_features", lambda: None) 

235 

236 with pytest.raises(json.JSONDecodeError): 

237 setup_experimentation() 

238 

239 

240def test_seconds_since_last_fetch_none_when_never_fetched(setup_isolation): 

241 assert experimentation.seconds_since_last_fetch() is None 

242 

243 

244def test_flags_value_returned(flags): 

245 flags.set_string("my_flag", "from_file") 

246 context = make_logged_out_context(LocalizationContext.en_utc()) 

247 assert context.get_string_value("my_flag", "fallback") == "from_file" 

248 

249 

250def test_flags_missing_key_returns_in_code_default(flags): 

251 context = make_logged_out_context(LocalizationContext.en_utc()) 

252 assert context.get_string_value("missing_flag", "fallback") == "fallback" 

253 

254 

255def test_flags_boolean_value(flags): 

256 flags.set_boolean("bool_flag", False) 

257 context = make_logged_out_context(LocalizationContext.en_utc()) 

258 assert context.get_boolean_value("bool_flag", default=True) is False 

259 

260 

261def test_load_local_flags_from_file(tmp_path): 

262 path = tmp_path / "flags.json" 

263 path.write_text(json.dumps({"flag_a": True, "flag_b": "hello", "flag_c": 42})) 

264 assert _load_local_flags(str(path)) == {"flag_a": True, "flag_b": "hello", "flag_c": 42} 

265 

266 

267def test_load_local_flags_rejects_non_object(tmp_path): 

268 path = tmp_path / "flags.json" 

269 path.write_text(json.dumps(["not", "an", "object"])) 

270 with pytest.raises(ValueError, match="must contain a JSON object"): 

271 _load_local_flags(str(path)) 

272 

273 

274def test_setup_in_local_file_mode_loads_file_and_skips_growthbook(monkeypatch, tmp_path): 

275 path = tmp_path / "flags.json" 

276 path.write_text(json.dumps({"flag_x": "from_file"})) 

277 monkeypatch.setattr(experimentation, "_initialized", False) 

278 monkeypatch.setitem(config, "FEATURE_FLAGS_FILE_OVERRIDE_PATH", str(path)) 

279 # If GrowthBook were touched, this would blow up. 

280 monkeypatch.setattr(experimentation, "_fetch_features", lambda: pytest.fail("GrowthBook should not be touched")) 

281 

282 setup_experimentation() 

283 

284 assert _load_local_flags(str(path)) == {"flag_x": "from_file"} 

285 assert experimentation._refresh_thread is None