Coverage for app/backend/src/tests/test_experimentation.py: 100%

1import json 

2 

3import pytest 

4from growthbook.common_types import FeatureResult 

5from sqlalchemy import select 

6 

7from couchers import experimentation 

8from couchers.config import config 

9from couchers.context import make_background_user_context, make_logged_out_context 

10from couchers.db import session_scope 

11from couchers.experimentation import GrowthBookUnavailableError, _record_feature_usage, setup_experimentation 

12from couchers.i18n import LocalizationContext 

13from couchers.metrics import feature_flag_evaluations_counter 

14from couchers.models.logging import ExperimentExposure, ExposureSource, FeatureUsage 

15from couchers.proto import bugs_pb2 

16from tests.fixtures.sessions import bugs_session 

17 

18 

19def _flag_eval_count(flag_key: str, source: str, value: str) -> float: 

20 return sum( 

21 s.value 

22 for m in feature_flag_evaluations_counter.collect() 

23 for s in m.samples 

24 if s.name == "couchers_feature_flag_evaluations_total" 

25 and s.labels.get("flag_key") == flag_key 

26 and s.labels.get("source") == source 

27 and s.labels.get("value") == value 

28 ) 

29 

30 

31# Raw GrowthBook feature definitions for exercising the framework's own bucketing/exposure mechanics. 

32# Most tests just need feature_flags.set(key, value); these go through feature_flags.set_definition(). 

33# A rollout with explicit coverage: bucketing needs a hash, so anonymous (logged-out) users are 

34# excluded even at 100% coverage and get the feature's default value instead. 

35_ROLLOUT_FLAG = {"defaultValue": "control", "rules": [{"force": "treatment", "coverage": 1.0}]} 

36# A global force with no coverage: applies to everyone, including anonymous users. 

37_GLOBAL_FORCE_FLAG = {"defaultValue": False, "rules": [{"force": True}]} 

38# An actual experiment: a logged-in user gets bucketed (coverage 1), which fires the exposure callback. 

39_EXPERIMENT_FLAG = { 

40 "defaultValue": "control", 

41 "rules": [{"key": "my_experiment", "variations": ["control", "treatment"], "coverage": 1.0}], 

42} 

43 

44 

45def test_logged_in_user_is_bucketed_into_rollout(db, feature_flags): 

46 feature_flags.set_definition("rollout_flag", _ROLLOUT_FLAG) 

47 context = make_background_user_context(123) 

48 assert context.get_string_value("rollout_flag", "fallback") == "treatment" 

49 

50 

51def test_anonymous_user_excluded_from_rollout_gets_feature_default(feature_flags): 

52 feature_flags.set_definition("rollout_flag", _ROLLOUT_FLAG) 

53 context = make_logged_out_context(LocalizationContext.en_utc()) 

54 # Previously this raised NotLoggedInContextException via context.user_id. 

55 assert context.get_string_value("rollout_flag", "fallback") == "control" 

56 

57 

58def test_anonymous_user_still_gets_global_force_on_flag(feature_flags): 

59 feature_flags.set_definition("global_flag", _GLOBAL_FORCE_FLAG) 

60 context = make_logged_out_context(LocalizationContext.en_utc()) 

61 assert context.get_boolean_value("global_flag", default=False) is True 

62 

63 

64def test_unknown_feature_returns_in_code_default(feature_flags): 

65 context = make_logged_out_context(LocalizationContext.en_utc()) 

66 assert context.get_string_value("does_not_exist", "my_default") == "my_default" 

67 

68 

69def test_evaluating_an_experiment_flag_records_exactly_one_exposure(db, feature_flags): 

70 # Evaluating an experiment-backed flag for a bucketed user records exactly one exposure - this is 

71 # the whole point of per-flag evaluation: exposure is logged only for flags the user actually hits. 

72 feature_flags.set_definition("experiment_flag", _EXPERIMENT_FLAG) 

73 context = make_background_user_context(123) 

74 assert context.get_object_value("experiment_flag", "control") in {"control", "treatment"} 

75 

76 with session_scope() as session: 

77 rows = session.execute(select(ExperimentExposure).where(ExperimentExposure.user_id == 123)).scalars().all() 

78 assert len(rows) == 1 

79 assert rows[0].experiment_key == "my_experiment" 

80 assert rows[0].source == ExposureSource.backend 

81 

82 

83def test_evaluate_feature_flag_servicer_returns_value(feature_flags, db): 

84 feature_flags.set("global_flag", True) 

85 with bugs_session() as bugs: 

86 res = bugs.EvaluateFeatureFlag(bugs_pb2.EvaluateFeatureFlagReq(flag_key="global_flag")) 

87 assert res.value.bool_value is True 

88 

89 

90def test_evaluate_feature_flag_servicer_unknown_leaves_value_unset(feature_flags, db): 

91 with bugs_session() as bugs: 

92 res = bugs.EvaluateFeatureFlag(bugs_pb2.EvaluateFeatureFlagReq(flag_key="does_not_exist")) 

93 assert not res.HasField("value") 

94 

95 

96def _get_usage(session, user_id): 

97 return ( 

98 session.execute(select(FeatureUsage).where(FeatureUsage.user_id == user_id).order_by(FeatureUsage.id)) 

99 .scalars() 

100 .all() 

101 ) 

102 

103 

104def test_record_feature_usage_appends_a_row(db): 

105 _record_feature_usage(1, "my_feature", FeatureResult(value=True, source="defaultValue")) 

106 

107 with session_scope() as session: 

108 rows = _get_usage(session, 1) 

109 assert len(rows) == 1 

110 assert rows[0].feature_key == "my_feature" 

111 assert rows[0].value is True 

112 assert rows[0].time is not None 

113 

114 

115def test_record_feature_usage_appends_a_row_per_check(db): 

116 # every check appends - the log is append-only, not deduplicated per (user, feature) 

117 _record_feature_usage(1, "my_feature", FeatureResult(value="first", source="force")) 

118 _record_feature_usage(1, "my_feature", FeatureResult(value="second", source="force")) 

119 

120 with session_scope() as session: 

121 rows = _get_usage(session, 1) 

122 assert len(rows) == 2 

123 assert [row.value for row in rows] == ["first", "second"] 

124 

125 

126def test_record_feature_usage_records_each_user_and_feature(db): 

127 _record_feature_usage(1, "feature_a", FeatureResult(value=1, source="force")) 

128 _record_feature_usage(1, "feature_b", FeatureResult(value=2, source="force")) 

129 _record_feature_usage(2, "feature_a", FeatureResult(value=3, source="force")) 

130 

131 with session_scope() as session: 

132 assert {row.feature_key for row in _get_usage(session, 1)} == {"feature_a", "feature_b"} 

133 assert len(_get_usage(session, 2)) == 1 

134 

135 

136def test_record_feature_usage_none_value(db): 

137 # unknown features evaluate to a None value - must persist without violating NOT NULL 

138 _record_feature_usage(1, "unknown_feature", FeatureResult(value=None, source="unknownFeature")) 

139 

140 with session_scope() as session: 

141 rows = _get_usage(session, 1) 

142 assert len(rows) == 1 

143 assert rows[0].value is None 

144 

145 

146def test_global_evaluation_excluded_from_rollout_gets_feature_default(feature_flags): 

147 # global (no-user) evaluation can't bucket into a rollout, so it gets the feature default 

148 feature_flags.set_definition("rollout_flag", _ROLLOUT_FLAG) 

149 assert experimentation.get_global_string_value("rollout_flag", "fallback") == "control" 

150 

151 

152def test_global_evaluation_gets_global_force_on_flag(feature_flags): 

153 feature_flags.set_definition("global_flag", _GLOBAL_FORCE_FLAG) 

154 assert experimentation.get_global_boolean_value("global_flag", default=False) is True 

155 

156 

157def test_global_evaluation_unknown_feature_returns_in_code_default(feature_flags): 

158 assert experimentation.get_global_string_value("does_not_exist", "my_default") == "my_default" 

159 

160 

161def test_evaluation_increments_metric_with_source_and_value(feature_flags): 

162 feature_flags.set("metric_flag", "yes") 

163 before = _flag_eval_count("metric_flag", "defaultValue", "yes") 

164 assert experimentation.get_global_string_value("metric_flag", "no") == "yes" 

165 assert _flag_eval_count("metric_flag", "defaultValue", "yes") == before + 1 

166 

167 

168def test_unknown_feature_increments_metric_with_unknown_source(feature_flags): 

169 before = _flag_eval_count("metric_unknown_flag", "unknownFeature", "fallback") 

170 assert experimentation.get_global_string_value("metric_unknown_flag", "fallback") == "fallback" 

171 assert _flag_eval_count("metric_unknown_flag", "unknownFeature", "fallback") == before + 1 

172 

173 

174@pytest.fixture 

175def setup_isolation(monkeypatch, tmp_path): 

176 """Run setup_experimentation() against a clean module state and a tmp cache path, and make sure the 

177 background refresh thread it starts is stopped afterwards.""" 

178 monkeypatch.setattr(experimentation, "_initialized", False) 

179 monkeypatch.setattr(experimentation, "_last_fetch_time", None) 

180 monkeypatch.setattr(experimentation, "_state", {"features": {}, "savedGroups": {}}) 

181 monkeypatch.setitem(config, "FEATURE_FLAGS_FILE_OVERRIDE_PATH", "") 

182 monkeypatch.setitem(config, "GROWTHBOOK_CACHE_PATH", str(tmp_path / "cache.json")) 

183 yield tmp_path / "cache.json" 

184 experimentation._refresh_stop.set() 

185 if experimentation._refresh_thread is not None: 

186 experimentation._refresh_thread.join(timeout=5) 

187 experimentation._refresh_stop.clear() 

188 experimentation._refresh_thread = None 

189 

190 

191def test_setup_writes_cache_and_records_fetch_time(setup_isolation, monkeypatch): 

192 cache = setup_isolation 

193 payload = {"features": {"f": {"defaultValue": True}}, "savedGroups": {}} 

194 monkeypatch.setattr(experimentation, "_fetch_features", lambda: payload) 

195 

196 setup_experimentation() 

197 

198 assert experimentation._state["features"] == {"f": {"defaultValue": True}} 

199 assert experimentation.seconds_since_last_fetch() is not None 

200 written = json.loads(cache.read_text()) 

201 assert written["response"] == payload 

202 assert "fetched_at" in written 

203 

204 

205def test_setup_falls_back_to_disk_cache_when_fetch_fails(setup_isolation, monkeypatch): 

206 cache = setup_isolation 

207 cached_payload = {"features": {"cached": {"defaultValue": "x"}}, "savedGroups": {}} 

208 cache.write_text(json.dumps({"fetched_at": 1000.0, "response": cached_payload})) 

209 monkeypatch.setattr(experimentation, "_fetch_features", lambda: None) 

210 

211 setup_experimentation() 

212 

213 assert experimentation._state["features"] == {"cached": {"defaultValue": "x"}} 

214 # fetch time reflects the cached pull time, so staleness is large immediately 

215 staleness = experimentation.seconds_since_last_fetch() 

216 assert staleness is not None and staleness > 0 

217 

218 

219def test_setup_raises_when_fetch_fails_and_no_cache(setup_isolation, monkeypatch): 

220 monkeypatch.setattr(experimentation, "_fetch_features", lambda: None) 

221 

222 with pytest.raises(GrowthBookUnavailableError): 

223 setup_experimentation() 

224 

225 

226def test_setup_raises_on_corrupt_cache(setup_isolation, monkeypatch): 

227 cache = setup_isolation 

228 cache.write_text("this is not json") 

229 monkeypatch.setattr(experimentation, "_fetch_features", lambda: None) 

230 

231 with pytest.raises(json.JSONDecodeError): 

232 setup_experimentation() 

233 

234 

235def test_seconds_since_last_fetch_none_when_never_fetched(setup_isolation): 

236 assert experimentation.seconds_since_last_fetch() is None 

237 

238 

239def test_flags_value_returned(flags): 

240 flags.set_string("my_flag", "from_file") 

241 context = make_logged_out_context(LocalizationContext.en_utc()) 

242 assert context.get_string_value("my_flag", "fallback") == "from_file" 

243 

244 

245def test_flags_missing_key_returns_in_code_default(flags): 

246 context = make_logged_out_context(LocalizationContext.en_utc()) 

247 assert context.get_string_value("missing_flag", "fallback") == "fallback" 

248 

249 

250def test_flags_boolean_value(flags): 

251 flags.set_boolean("bool_flag", False) 

252 context = make_logged_out_context(LocalizationContext.en_utc()) 

253 assert context.get_boolean_value("bool_flag", default=True) is False 

254 

255 

256def test_load_local_flags_from_file(tmp_path): 

257 path = tmp_path / "flags.json" 

258 path.write_text(json.dumps({"flag_a": True, "flag_b": "hello", "flag_c": 42})) 

259 assert experimentation._load_local_flags(str(path)) == {"flag_a": True, "flag_b": "hello", "flag_c": 42} 

260 

261 

262def test_load_local_flags_rejects_non_object(tmp_path): 

263 path = tmp_path / "flags.json" 

264 path.write_text(json.dumps(["not", "an", "object"])) 

265 with pytest.raises(ValueError, match="must contain a JSON object"): 

266 experimentation._load_local_flags(str(path)) 

267 

268 

269def test_setup_in_local_file_mode_loads_file_and_skips_growthbook(monkeypatch, tmp_path): 

270 path = tmp_path / "flags.json" 

271 path.write_text(json.dumps({"flag_x": "from_file"})) 

272 monkeypatch.setattr(experimentation, "_initialized", False) 

273 monkeypatch.setitem(config, "FEATURE_FLAGS_FILE_OVERRIDE_PATH", str(path)) 

274 # If GrowthBook were touched, this would blow up. 

275 monkeypatch.setattr(experimentation, "_fetch_features", lambda: pytest.fail("GrowthBook should not be touched")) 

276 

277 setup_experimentation() 

278 

279 assert experimentation._load_local_flags(str(path)) == {"flag_x": "from_file"} 

280 assert experimentation._refresh_thread is None