Coverage for app/backend/src/tests/test_experimentation.py: 100%
220 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 14:45 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 14:45 +0000
1import json
2from datetime import date
4import pytest
5from google.protobuf import empty_pb2
6from growthbook.common_types import FeatureResult
7from sqlalchemy import select
9from couchers import experimentation
10from couchers.config import config
11from couchers.context import make_background_user_context, make_logged_out_context
12from couchers.db import session_scope
13from couchers.experimentation import (
14 GrowthBookUnavailableError,
15 _load_local_flags,
16 _record_feature_usage,
17 get_targeting_attributes,
18 setup_experimentation,
19)
20from couchers.i18n import LocalizationContext
21from couchers.metrics import feature_flag_evaluations_counter
22from couchers.models.logging import ExperimentExposure, ExposureSource, FeatureUsage
23from couchers.proto import account_pb2, bugs_pb2
24from tests.fixtures.db import generate_user, make_volunteer
25from tests.fixtures.feature_flags import FeatureFlags, Flags
26from tests.fixtures.sessions import account_session, bugs_session
29def _flag_eval_count(flag_key: str, source: str, value: str) -> float:
30 return sum(
31 s.value
32 for m in feature_flag_evaluations_counter.collect()
33 for s in m.samples
34 if s.name == "couchers_feature_flag_evaluations_total"
35 and s.labels.get("flag_key") == flag_key
36 and s.labels.get("source") == source
37 and s.labels.get("value") == value
38 )
41# Raw GrowthBook feature definitions for exercising the framework's own bucketing/exposure mechanics.
42# Most tests just need feature_flags.set(key, value); these go through feature_flags.set_definition().
43# A rollout with explicit coverage: bucketing needs a hash, so anonymous (logged-out) users are
44# excluded even at 100% coverage and get the feature's default value instead.
45_ROLLOUT_FLAG = {"defaultValue": "control", "rules": [{"force": "treatment", "coverage": 1.0}]}
46# A global force with no coverage: applies to everyone, including anonymous users.
47_GLOBAL_FORCE_FLAG = {"defaultValue": False, "rules": [{"force": True}]}
48# An actual experiment: a logged-in user gets bucketed (coverage 1), which fires the exposure callback.
49_EXPERIMENT_FLAG = {
50 "defaultValue": "control",
51 "rules": [{"key": "my_experiment", "variations": ["control", "treatment"], "coverage": 1.0}],
52}
54# Forced on only for current volunteers, via the is_volunteer targeting attribute.
55_VOLUNTEERS_FLAG = {"defaultValue": False, "rules": [{"condition": {"is_volunteer": True}, "force": True}]}
58def test_logged_in_user_is_bucketed_into_rollout(db, feature_flags: FeatureFlags):
59 feature_flags.set_definition("rollout_flag", _ROLLOUT_FLAG)
60 user, _ = generate_user()
61 context = make_background_user_context(user.id)
62 assert context.get_string_value("rollout_flag", "fallback") == "treatment"
65def test_anonymous_user_excluded_from_rollout_gets_feature_default(feature_flags: FeatureFlags):
66 feature_flags.set_definition("rollout_flag", _ROLLOUT_FLAG)
67 context = make_logged_out_context(LocalizationContext.en_utc())
68 # Previously this raised NotLoggedInContextException via context.user_id.
69 assert context.get_string_value("rollout_flag", "fallback") == "control"
72def test_anonymous_user_still_gets_global_force_on_flag(feature_flags: FeatureFlags):
73 feature_flags.set_definition("global_flag", _GLOBAL_FORCE_FLAG)
74 context = make_logged_out_context(LocalizationContext.en_utc())
75 assert context.get_boolean_value("global_flag", default=False) is True
78def test_unknown_feature_returns_in_code_default(feature_flags: FeatureFlags):
79 context = make_logged_out_context(LocalizationContext.en_utc())
80 assert context.get_string_value("does_not_exist", "my_default") == "my_default"
83def test_evaluating_an_experiment_flag_records_exactly_one_exposure(db, feature_flags: FeatureFlags):
84 # Evaluating an experiment-backed flag for a bucketed user records exactly one exposure - this is
85 # the whole point of per-flag evaluation: exposure is logged only for flags the user actually hits.
86 feature_flags.set_definition("experiment_flag", _EXPERIMENT_FLAG)
87 user, _ = generate_user()
88 context = make_background_user_context(user.id)
89 assert context.get_object_value("experiment_flag", "control") in {"control", "treatment"}
91 with session_scope() as session:
92 rows = session.execute(select(ExperimentExposure).where(ExperimentExposure.user_id == user.id)).scalars().all()
93 assert len(rows) == 1
94 assert rows[0].experiment_key == "my_experiment"
95 assert rows[0].source == ExposureSource.backend
98def test_targeting_attributes(db):
99 user, _ = generate_user()
100 volunteer, _ = generate_user()
101 past_volunteer, _ = generate_user()
102 with session_scope() as session:
103 session.add(make_volunteer(user_id=volunteer.id, role="Tester", started_volunteering=date(2020, 1, 1)))
104 session.add(
105 make_volunteer(
106 user_id=past_volunteer.id,
107 role="Tester",
108 started_volunteering=date(2020, 1, 1),
109 stopped_volunteering=date(2021, 1, 1),
110 )
111 )
113 assert get_targeting_attributes(None) == {"logged_in": False}
114 assert get_targeting_attributes(user.id) == {
115 "id": str(user.id),
116 "logged_in": True,
117 "is_volunteer": False,
118 "is_beta": False,
119 }
120 assert get_targeting_attributes(volunteer.id) == {
121 "id": str(volunteer.id),
122 "logged_in": True,
123 "is_volunteer": True,
124 "is_beta": False,
125 }
126 assert get_targeting_attributes(past_volunteer.id) == {
127 "id": str(past_volunteer.id),
128 "logged_in": True,
129 "is_volunteer": False,
130 "is_beta": False,
131 }
134def test_targeting_attributes_is_beta(db):
135 user, token = generate_user()
136 with account_session(token) as account:
137 account.SetBetaFeaturesOptIn(account_pb2.SetBetaFeaturesOptInReq(beta_features_opt_in=True))
138 assert get_targeting_attributes(user.id)["is_beta"] is True
141def test_flag_targeted_at_beta_users(db, feature_flags: FeatureFlags):
142 feature_flags.set_definition(
143 "beta_flag", {"defaultValue": False, "rules": [{"condition": {"is_beta": True}, "force": True}]}
144 )
145 user, token = generate_user()
146 assert make_background_user_context(user.id).get_boolean_value("beta_flag", default=False) is False
147 with account_session(token) as account:
148 account.SetBetaFeaturesOptIn(account_pb2.SetBetaFeaturesOptInReq(beta_features_opt_in=True))
149 assert make_background_user_context(user.id).get_boolean_value("beta_flag", default=False) is True
152def test_flag_targeted_at_volunteers(db, feature_flags: FeatureFlags):
153 feature_flags.set_definition("volunteers_flag", _VOLUNTEERS_FLAG)
154 user, _ = generate_user()
155 volunteer, _ = generate_user()
156 with session_scope() as session:
157 session.add(make_volunteer(user_id=volunteer.id, role="Tester", started_volunteering=date(2020, 1, 1)))
159 assert make_background_user_context(volunteer.id).get_boolean_value("volunteers_flag", default=False) is True
160 assert make_background_user_context(user.id).get_boolean_value("volunteers_flag", default=False) is False
161 logged_out = make_logged_out_context(LocalizationContext.en_utc())
162 assert logged_out.get_boolean_value("volunteers_flag", default=False) is False
165def test_get_feature_flag_attributes_servicer(db):
166 user, token = generate_user()
167 with bugs_session(token) as bugs:
168 res = bugs.GetFeatureFlagAttributes(empty_pb2.Empty())
169 assert json.loads(res.attributes_json) == {
170 "id": str(user.id),
171 "logged_in": True,
172 "is_volunteer": False,
173 "is_beta": False,
174 }
176 with bugs_session() as bugs:
177 res = bugs.GetFeatureFlagAttributes(empty_pb2.Empty())
178 assert json.loads(res.attributes_json) == {"logged_in": False}
181def test_evaluate_feature_flag_servicer_returns_value(feature_flags: FeatureFlags, db):
182 feature_flags.set("global_flag", True)
183 with bugs_session() as bugs:
184 res = bugs.EvaluateFeatureFlag(bugs_pb2.EvaluateFeatureFlagReq(flag_key="global_flag"))
185 assert res.value.bool_value is True
188def test_evaluate_feature_flag_servicer_unknown_leaves_value_unset(feature_flags: FeatureFlags, db):
189 with bugs_session() as bugs:
190 res = bugs.EvaluateFeatureFlag(bugs_pb2.EvaluateFeatureFlagReq(flag_key="does_not_exist"))
191 assert not res.HasField("value")
194def _get_usage(session, user_id):
195 return (
196 session.execute(select(FeatureUsage).where(FeatureUsage.user_id == user_id).order_by(FeatureUsage.id))
197 .scalars()
198 .all()
199 )
202def test_record_feature_usage_appends_a_row(db):
203 _record_feature_usage(1, "my_feature", FeatureResult(value=True, source="defaultValue"))
205 with session_scope() as session:
206 rows = _get_usage(session, 1)
207 assert len(rows) == 1
208 assert rows[0].feature_key == "my_feature"
209 assert rows[0].value is True
210 assert rows[0].time is not None
213def test_record_feature_usage_appends_a_row_per_check(db):
214 # every check appends - the log is append-only, not deduplicated per (user, feature)
215 _record_feature_usage(1, "my_feature", FeatureResult(value="first", source="force"))
216 _record_feature_usage(1, "my_feature", FeatureResult(value="second", source="force"))
218 with session_scope() as session:
219 rows = _get_usage(session, 1)
220 assert len(rows) == 2
221 assert [row.value for row in rows] == ["first", "second"]
224def test_record_feature_usage_records_each_user_and_feature(db):
225 _record_feature_usage(1, "feature_a", FeatureResult(value=1, source="force"))
226 _record_feature_usage(1, "feature_b", FeatureResult(value=2, source="force"))
227 _record_feature_usage(2, "feature_a", FeatureResult(value=3, source="force"))
229 with session_scope() as session:
230 assert {row.feature_key for row in _get_usage(session, 1)} == {"feature_a", "feature_b"}
231 assert len(_get_usage(session, 2)) == 1
234def test_record_feature_usage_none_value(db):
235 # unknown features evaluate to a None value - must persist without violating NOT NULL
236 _record_feature_usage(1, "unknown_feature", FeatureResult(value=None, source="unknownFeature"))
238 with session_scope() as session:
239 rows = _get_usage(session, 1)
240 assert len(rows) == 1
241 assert rows[0].value is None
244def test_global_evaluation_excluded_from_rollout_gets_feature_default(feature_flags: FeatureFlags):
245 # global (no-user) evaluation can't bucket into a rollout, so it gets the feature default
246 feature_flags.set_definition("rollout_flag", _ROLLOUT_FLAG)
247 assert experimentation.get_global_string_value("rollout_flag", "fallback") == "control"
250def test_global_evaluation_gets_global_force_on_flag(feature_flags: FeatureFlags):
251 feature_flags.set_definition("global_flag", _GLOBAL_FORCE_FLAG)
252 assert experimentation.get_global_boolean_value("global_flag", default=False) is True
255def test_global_evaluation_unknown_feature_returns_in_code_default(feature_flags: FeatureFlags):
256 assert experimentation.get_global_string_value("does_not_exist", "my_default") == "my_default"
259def test_evaluation_increments_metric_with_source_and_value(feature_flags: FeatureFlags):
260 feature_flags.set("metric_flag", "yes")
261 before = _flag_eval_count("metric_flag", "defaultValue", "yes")
262 assert experimentation.get_global_string_value("metric_flag", "no") == "yes"
263 assert _flag_eval_count("metric_flag", "defaultValue", "yes") == before + 1
266def test_unknown_feature_increments_metric_with_unknown_source(feature_flags: FeatureFlags):
267 before = _flag_eval_count("metric_unknown_flag", "unknownFeature", "fallback")
268 assert experimentation.get_global_string_value("metric_unknown_flag", "fallback") == "fallback"
269 assert _flag_eval_count("metric_unknown_flag", "unknownFeature", "fallback") == before + 1
272@pytest.fixture
273def setup_isolation(monkeypatch, tmp_path):
274 """Run setup_experimentation() against a clean module state and a tmp cache path, and make sure the
275 background refresh thread it starts is stopped afterwards."""
276 monkeypatch.setattr(experimentation, "_initialized", False)
277 monkeypatch.setattr(experimentation, "_last_fetch_time", None)
278 monkeypatch.setattr(experimentation, "_state", {"features": {}, "savedGroups": {}})
279 monkeypatch.setitem(config, "FEATURE_FLAGS_FILE_OVERRIDE_PATH", "")
280 monkeypatch.setitem(config, "GROWTHBOOK_CACHE_PATH", str(tmp_path / "cache.json"))
281 yield tmp_path / "cache.json"
282 experimentation._refresh_stop.set()
283 if experimentation._refresh_thread is not None:
284 experimentation._refresh_thread.join(timeout=5)
285 experimentation._refresh_stop.clear()
286 experimentation._refresh_thread = None
289def test_setup_writes_cache_and_records_fetch_time(setup_isolation, monkeypatch):
290 cache = setup_isolation
291 payload = {"features": {"f": {"defaultValue": True}}, "savedGroups": {}}
292 monkeypatch.setattr(experimentation, "_fetch_features", lambda: payload)
294 setup_experimentation()
296 assert experimentation._state["features"] == {"f": {"defaultValue": True}}
297 assert experimentation.seconds_since_last_fetch() is not None
298 written = json.loads(cache.read_text())
299 assert written["response"] == payload
300 assert "fetched_at" in written
303def test_setup_falls_back_to_disk_cache_when_fetch_fails(setup_isolation, monkeypatch):
304 cache = setup_isolation
305 cached_payload = {"features": {"cached": {"defaultValue": "x"}}, "savedGroups": {}}
306 cache.write_text(json.dumps({"fetched_at": 1000.0, "response": cached_payload}))
307 monkeypatch.setattr(experimentation, "_fetch_features", lambda: None)
309 setup_experimentation()
311 assert experimentation._state["features"] == {"cached": {"defaultValue": "x"}}
312 # fetch time reflects the cached pull time, so staleness is large immediately
313 staleness = experimentation.seconds_since_last_fetch()
314 assert staleness is not None and staleness > 0
317def test_setup_raises_when_fetch_fails_and_no_cache(setup_isolation, monkeypatch):
318 monkeypatch.setattr(experimentation, "_fetch_features", lambda: None)
320 with pytest.raises(GrowthBookUnavailableError):
321 setup_experimentation()
324def test_setup_raises_on_corrupt_cache(setup_isolation, monkeypatch):
325 cache = setup_isolation
326 cache.write_text("this is not json")
327 monkeypatch.setattr(experimentation, "_fetch_features", lambda: None)
329 with pytest.raises(json.JSONDecodeError):
330 setup_experimentation()
333def test_seconds_since_last_fetch_none_when_never_fetched(setup_isolation):
334 assert experimentation.seconds_since_last_fetch() is None
337def test_flags_value_returned(flags: Flags):
338 flags.set_string("my_flag", "from_file")
339 context = make_logged_out_context(LocalizationContext.en_utc())
340 assert context.get_string_value("my_flag", "fallback") == "from_file"
343def test_flags_missing_key_returns_in_code_default(flags: Flags):
344 context = make_logged_out_context(LocalizationContext.en_utc())
345 assert context.get_string_value("missing_flag", "fallback") == "fallback"
348def test_flags_boolean_value(flags: Flags):
349 flags.set_boolean("bool_flag", False)
350 context = make_logged_out_context(LocalizationContext.en_utc())
351 assert context.get_boolean_value("bool_flag", default=True) is False
354def test_load_local_flags_from_file(tmp_path):
355 path = tmp_path / "flags.json"
356 path.write_text(json.dumps({"flag_a": True, "flag_b": "hello", "flag_c": 42}))
357 assert _load_local_flags(str(path)) == {"flag_a": True, "flag_b": "hello", "flag_c": 42}
360def test_load_local_flags_rejects_non_object(tmp_path):
361 path = tmp_path / "flags.json"
362 path.write_text(json.dumps(["not", "an", "object"]))
363 with pytest.raises(ValueError, match="must contain a JSON object"):
364 _load_local_flags(str(path))
367def test_setup_in_local_file_mode_loads_file_and_skips_growthbook(monkeypatch, tmp_path):
368 path = tmp_path / "flags.json"
369 path.write_text(json.dumps({"flag_x": "from_file"}))
370 monkeypatch.setattr(experimentation, "_initialized", False)
371 monkeypatch.setitem(config, "FEATURE_FLAGS_FILE_OVERRIDE_PATH", str(path))
372 # If GrowthBook were touched, this would blow up.
373 monkeypatch.setattr(experimentation, "_fetch_features", lambda: pytest.fail("GrowthBook should not be touched"))
375 setup_experimentation()
377 assert _load_local_flags(str(path)) == {"flag_x": "from_file"}
378 assert experimentation._refresh_thread is None