Coverage for app / backend / src / tests / test_experimentation.py: 100%

1import pytest 

2from growthbook.common_types import FeatureResult 

3from sqlalchemy import select 

4 

5from couchers import experimentation 

6from couchers.config import config 

7from couchers.context import make_background_user_context, make_logged_out_context 

8from couchers.db import session_scope 

9from couchers.experimentation import _record_feature_usage 

10from couchers.i18n import LocalizationContext 

11from couchers.models.logging import ExperimentExposure, FeatureUsage 

12from couchers.proto import bugs_pb2 

13from tests.fixtures.sessions import bugs_session 

14 

15 

16@pytest.fixture 

17def experimentation_snapshot(monkeypatch): 

18 """Enable experimentation with an in-memory feature snapshot for evaluation.""" 

19 monkeypatch.setattr(experimentation, "_initialized", True) 

20 features = { 

21 # A rollout with explicit coverage: bucketing needs a hash, so anonymous (logged-out) users 

22 # are excluded even at 100% coverage and get the feature's default value instead. 

23 "rollout_flag": {"defaultValue": "control", "rules": [{"force": "treatment", "coverage": 1.0}]}, 

24 # A global force with no coverage: applies to everyone, including anonymous users. 

25 "global_flag": {"defaultValue": False, "rules": [{"force": True}]}, 

26 # An actual experiment: a logged-in user gets bucketed (coverage 1), which fires the exposure 

27 # tracking callback unless it's explicitly suppressed. 

28 "experiment_flag": { 

29 "defaultValue": "control", 

30 "rules": [{"key": "my_experiment", "variations": ["control", "treatment"], "coverage": 1.0}], 

31 }, 

32 } 

33 monkeypatch.setattr(experimentation, "_state", {"features": features, "savedGroups": {}}) 

34 monkeypatch.setitem(config, "EXPERIMENTATION_ENABLED", True) 

35 monkeypatch.setitem(config, "EXPERIMENTATION_PASS_ALL_GATES", False) 

36 

37 

38def test_logged_in_user_is_bucketed_into_rollout(db, experimentation_snapshot): 

39 context = make_background_user_context(123) 

40 assert context.get_string_value("rollout_flag", "fallback") == "treatment" 

41 

42 

43def test_anonymous_user_excluded_from_rollout_gets_feature_default(experimentation_snapshot): 

44 context = make_logged_out_context(LocalizationContext.en_utc()) 

45 # Previously this raised NotLoggedInContextException via context.user_id. 

46 assert context.get_string_value("rollout_flag", "fallback") == "control" 

47 

48 

49def test_anonymous_user_still_gets_global_force_on_flag(experimentation_snapshot): 

50 context = make_logged_out_context(LocalizationContext.en_utc()) 

51 assert context.get_boolean_value("global_flag", default=False) is True 

52 

53 

54def test_unknown_feature_returns_in_code_default(experimentation_snapshot): 

55 context = make_logged_out_context(LocalizationContext.en_utc()) 

56 assert context.get_string_value("does_not_exist", "my_default") == "my_default" 

57 

58 

59def test_value_method_returns_in_code_default_when_disabled(monkeypatch, experimentation_snapshot): 

60 monkeypatch.setitem(config, "EXPERIMENTATION_ENABLED", False) 

61 context = make_background_user_context(123) 

62 assert context.get_string_value("global_flag", "off") == "off" 

63 

64 

65def test_evaluating_an_experiment_flag_records_exactly_one_exposure(db, experimentation_snapshot): 

66 # Evaluating an experiment-backed flag for a bucketed user records exactly one exposure - this is 

67 # the whole point of per-flag evaluation: exposure is logged only for flags the user actually hits. 

68 context = make_background_user_context(123) 

69 assert context.get_object_value("experiment_flag", "control") in {"control", "treatment"} 

70 

71 with session_scope() as session: 

72 rows = session.execute(select(ExperimentExposure).where(ExperimentExposure.user_id == 123)).scalars().all() 

73 assert len(rows) == 1 

74 assert rows[0].experiment_key == "my_experiment" 

75 

76 

77def test_evaluate_feature_flag_servicer_returns_value(experimentation_snapshot, db): 

78 with bugs_session() as bugs: 

79 res = bugs.EvaluateFeatureFlag(bugs_pb2.EvaluateFeatureFlagReq(flag_key="global_flag")) 

80 assert res.value.bool_value is True 

81 

82 

83def test_evaluate_feature_flag_servicer_unknown_leaves_value_unset(experimentation_snapshot, db): 

84 with bugs_session() as bugs: 

85 res = bugs.EvaluateFeatureFlag(bugs_pb2.EvaluateFeatureFlagReq(flag_key="does_not_exist")) 

86 assert not res.HasField("value") 

87 

88 

89def _get_usage(session, user_id): 

90 return ( 

91 session.execute(select(FeatureUsage).where(FeatureUsage.user_id == user_id).order_by(FeatureUsage.id)) 

92 .scalars() 

93 .all() 

94 ) 

95 

96 

97def test_record_feature_usage_appends_a_row(db): 

98 _record_feature_usage(1, "my_feature", FeatureResult(value=True, source="defaultValue")) 

99 

100 with session_scope() as session: 

101 rows = _get_usage(session, 1) 

102 assert len(rows) == 1 

103 assert rows[0].feature_key == "my_feature" 

104 assert rows[0].value is True 

105 assert rows[0].time is not None 

106 

107 

108def test_record_feature_usage_appends_a_row_per_check(db): 

109 # every check appends - the log is append-only, not deduplicated per (user, feature) 

110 _record_feature_usage(1, "my_feature", FeatureResult(value="first", source="force")) 

111 _record_feature_usage(1, "my_feature", FeatureResult(value="second", source="force")) 

112 

113 with session_scope() as session: 

114 rows = _get_usage(session, 1) 

115 assert len(rows) == 2 

116 assert [row.value for row in rows] == ["first", "second"] 

117 

118 

119def test_record_feature_usage_records_each_user_and_feature(db): 

120 _record_feature_usage(1, "feature_a", FeatureResult(value=1, source="force")) 

121 _record_feature_usage(1, "feature_b", FeatureResult(value=2, source="force")) 

122 _record_feature_usage(2, "feature_a", FeatureResult(value=3, source="force")) 

123 

124 with session_scope() as session: 

125 assert {row.feature_key for row in _get_usage(session, 1)} == {"feature_a", "feature_b"} 

126 assert len(_get_usage(session, 2)) == 1 

127 

128 

129def test_record_feature_usage_none_value(db): 

130 # unknown features evaluate to a None value - must persist without violating NOT NULL 

131 _record_feature_usage(1, "unknown_feature", FeatureResult(value=None, source="unknownFeature")) 

132 

133 with session_scope() as session: 

134 rows = _get_usage(session, 1) 

135 assert len(rows) == 1 

136 assert rows[0].value is None 

137 

138 

139def test_global_evaluation_excluded_from_rollout_gets_feature_default(experimentation_snapshot): 

140 # global (no-user) evaluation can't bucket into a rollout, so it gets the feature default 

141 assert experimentation.get_global_string_value("rollout_flag", "fallback") == "control" 

142 

143 

144def test_global_evaluation_gets_global_force_on_flag(experimentation_snapshot): 

145 assert experimentation.get_global_boolean_value("global_flag", default=False) is True 

146 

147 

148def test_global_evaluation_unknown_feature_returns_in_code_default(experimentation_snapshot): 

149 assert experimentation.get_global_string_value("does_not_exist", "my_default") == "my_default"