Coverage for app/backend/src/tests/test_hybrid_properties.py: 100%
260 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-18 18:02 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-18 18:02 +0000
1"""
2Hybrid properties are two implementations of one truth: a python body that runs on a loaded instance,
3and a SQL expression that runs in the database. Nothing in the ORM checks that they agree, so they can
4drift apart silently, which is how the lite_users strong verification bug survived for 21 months.
6This module discovers every hybrid on every model, builds a deliberately diverse population for it,
7and asserts that the python value equals the value postgres computes, for every row. A hybrid that
8cannot run in python at all (its body is written in terms of `func.now()` and friends, so evaluating
9it on an instance yields a SQL expression rather than a value) is listed in SQL_ONLY, and we assert
10that it really is inert in python rather than quietly returning something wrong.
12Adding a hybrid to a model without adding it here fails test_every_hybrid_is_covered.
13"""
15from collections.abc import Callable
16from datetime import date, timedelta
17from typing import Any
19import pytest
20from psycopg.types.range import TimestamptzRange
21from sqlalchemy import inspect, select, update
22from sqlalchemy.ext.hybrid import HybridExtensionType
23from sqlalchemy.orm import Session
24from sqlalchemy.sql.elements import ClauseElement
26from couchers.constants import GUIDELINES_VERSION, PHONE_VERIFICATION_LIFETIME, TOS_VERSION
27from couchers.crypto import random_hex
28from couchers.db import session_scope
29from couchers.models import (
30 AccountDeletionToken,
31 ActivenessProbe,
32 ActivenessProbeStatus,
33 BackgroundJob,
34 BackgroundJobState,
35 Base,
36 ContributorForm,
37 Conversation,
38 Event,
39 EventOccurrence,
40 GroupChat,
41 GroupChatRole,
42 GroupChatSubscription,
43 HostingStatus,
44 HostRequest,
45 HostRequestStatus,
46 InitiatedUpload,
47 LoginToken,
48 ModerationObjectType,
49 ModNote,
50 Node,
51 NodeType,
52 PassportSex,
53 PasswordResetToken,
54 PostalVerificationAttempt,
55 PostalVerificationStatus,
56 SignupFlow,
57 SleepingArrangement,
58 StayProposalKind,
59 StrongVerificationAttempt,
60 StrongVerificationAttemptStatus,
61 Thread,
62 User,
63 UserSession,
64)
65from couchers.moderation.utils import create_moderation
66from couchers.utils import create_coordinate, create_polygon_lat_lng, now, to_multi
67from tests.fixtures.db import generate_user, make_user_invisible
69# Hybrids whose body only makes sense in SQL: evaluating them on an instance yields a SQLAlchemy
70# expression, which blows up the moment anything treats it as a value. They have no python
71# implementation to disagree with, so there is nothing to compare -- but see
72# test_sql_only_hybrids_are_inert_in_python, which holds them to being loudly, not quietly, unusable.
73SQL_ONLY = {
74 "BackgroundJob.ready_for_retry": "compares next_attempt_after against func.now()",
75 "GroupChatSubscription.is_muted": "compares muted_until against func.now()",
76 "InitiatedUpload.is_valid": "compares created/expiry against func.now()",
77 "UserSession.is_valid": "compares created/expiry/last_seen against func.now() and a SQL interval",
78}
81def _label(model: type[Base], name: str) -> str:
82 return f"{model.__name__}.{name}"
85def _hybrids(extension_type: HybridExtensionType) -> list[tuple[type[Base], str]]:
86 # `@x.inplace.expression` binds one hybrid to two names, the public one and the private one holding
87 # the SQL expression, so dedupe on the descriptor itself and keep the name people write in queries
88 found: dict[tuple[type[Base], int], str] = {}
89 for mapper in Base.registry.mappers:
90 for name, descriptor in mapper.all_orm_descriptors.items():
91 if descriptor.extension_type != extension_type:
92 continue
93 key = (mapper.class_, id(descriptor))
94 if key not in found or found[key].startswith("_"):
95 found[key] = name
96 return sorted(((model, name) for (model, _), name in found.items()), key=lambda pair: _label(*pair))
99HYBRID_PROPERTIES = _hybrids(HybridExtensionType.HYBRID_PROPERTY)
100HYBRID_METHODS = _hybrids(HybridExtensionType.HYBRID_METHOD)
102Population = Callable[[], None]
103POPULATIONS: dict[type[Base], Population] = {}
106def _populates(model: type[Base]) -> Callable[[Population], Population]:
107 def decorator(population: Population) -> Population:
108 POPULATIONS[model] = population
109 return population
111 return decorator
114## Populations: one per model, each diverse enough that every hybrid on the model takes at least two
115## different values across the rows (test_hybrid_agrees_with_sql asserts that).
118@_populates(User)
119def _populate_users() -> None:
120 generate_user()
121 generate_user(accepted_tos=TOS_VERSION - 1)
122 generate_user(accepted_community_guidelines=GUIDELINES_VERSION - 1)
123 generate_user(max_guests=3, sleeping_arrangement=SleepingArrangement.private)
124 generate_user(max_guests=3, sleeping_arrangement=None)
125 banned, _ = generate_user()
126 make_user_invisible(banned.id)
127 generate_user(delete_user=True)
128 shadowed, _ = generate_user()
129 relocating, _ = generate_user()
130 noted, _ = generate_user()
131 acknowledged, _ = generate_user()
132 probed, _ = generate_user()
133 responded, _ = generate_user()
134 phone_verified, _ = generate_user()
135 phone_stale, _ = generate_user()
136 code_sent, _ = generate_user()
137 moderator, _ = generate_user(is_superuser=True)
139 with session_scope() as session:
140 session.execute(update(User).where(User.id == shadowed.id).values(shadowed_at=now() - timedelta(days=1)))
141 session.execute(update(User).where(User.id == relocating.id).values(needs_to_update_location=True))
142 session.add(
143 ModNote(user_id=noted.id, creator_user_id=moderator.id, internal_id="pending", note_content="Be nice")
144 )
145 session.add(
146 ModNote(
147 user_id=acknowledged.id,
148 creator_user_id=moderator.id,
149 internal_id="acknowledged",
150 note_content="Be nice",
151 acknowledged=now() - timedelta(days=1),
152 )
153 )
154 session.add(ActivenessProbe(user_id=probed.id))
155 session.add(
156 ActivenessProbe(
157 user_id=responded.id,
158 responded=now() - timedelta(days=1),
159 response=ActivenessProbeStatus.still_active,
160 )
161 )
162 # a phone number is required whenever the verification is: see the phone_verified_conditions constraint
163 session.execute(
164 update(User)
165 .where(User.id == phone_verified.id)
166 .values(phone="+46701740601", phone_verification_verified=now() - timedelta(days=1))
167 )
168 session.execute(
169 update(User)
170 .where(User.id == phone_stale.id)
171 .values(
172 phone="+46701740602",
173 phone_verification_verified=now() - PHONE_VERIFICATION_LIFETIME - timedelta(days=1),
174 )
175 )
176 session.execute(
177 update(User).where(User.id == code_sent.id).values(phone_verification_sent=now() - timedelta(hours=1))
178 )
181@_populates(ModNote)
182@_populates(ActivenessProbe)
183def _populate_user_flags() -> None:
184 _populate_users()
187WOMAN_BIRTHDATE = date(1990, 3, 4)
188MAN_BIRTHDATE = date(1985, 11, 22)
191@_populates(StrongVerificationAttempt)
192def _populate_strong_verification_attempts() -> None:
193 woman, _ = generate_user(gender="Woman", birthdate=WOMAN_BIRTHDATE)
194 man, _ = generate_user(gender="Man", birthdate=MAN_BIRTHDATE)
196 with session_scope() as session:
197 # succeeded and unexpired: the only shape that verifies anyone
198 session.add(_attempt(woman.id, 1, StrongVerificationAttemptStatus.succeeded, expiry_days=365))
199 # succeeded but the passport has expired
200 session.add(_attempt(man.id, 2, StrongVerificationAttemptStatus.succeeded, expiry_days=-1))
201 # same passport data as the first attempt, but the data has since been deleted
202 session.add(
203 _attempt(woman.id, 3, StrongVerificationAttemptStatus.deleted, expiry_days=365, has_full_data=False)
204 )
205 # never got any data at all
206 session.add(_attempt(man.id, 4, StrongVerificationAttemptStatus.failed, expiry_days=None))
209def _attempt(
210 user_id: int,
211 n: int,
212 status: StrongVerificationAttemptStatus,
213 *,
214 expiry_days: int | None,
215 has_full_data: bool = True,
216) -> StrongVerificationAttempt:
217 """A strong verification attempt in one of the shapes the check constraints allow."""
218 has_minimal_data = expiry_days is not None
219 # full data implies minimal data
220 has_full_data = has_full_data and has_minimal_data
221 return StrongVerificationAttempt(
222 verification_attempt_token=f"verification_attempt_token_{n}",
223 user_id=user_id,
224 status=status,
225 has_full_data=has_full_data,
226 passport_encrypted_data=b"not real" if has_full_data else None,
227 # the passport always describes the first user, so pairing it with the second is a real mismatch
228 passport_date_of_birth=WOMAN_BIRTHDATE if has_full_data else None,
229 passport_sex=PassportSex.female if has_full_data else None,
230 has_minimal_data=has_minimal_data,
231 passport_expiry_date=date.today() + timedelta(days=expiry_days) if expiry_days is not None else None,
232 passport_nationality="UTO" if has_minimal_data else None,
233 passport_last_three_document_chars=f"{n:03}" if has_minimal_data else None,
234 iris_token=f"iris_token_{n}",
235 iris_session_id=n,
236 )
239@_populates(PostalVerificationAttempt)
240def _populate_postal_verification_attempts() -> None:
241 verified, _ = generate_user()
242 cancelled, _ = generate_user()
243 pending, _ = generate_user()
245 with session_scope() as session:
246 session.add(
247 PostalVerificationAttempt(
248 user_id=verified.id,
249 status=PostalVerificationStatus.succeeded,
250 address_line_1="1 Test Street",
251 city="Testing city",
252 country_code="US",
253 verification_code="ABC123",
254 postcard_sent_at=now() - timedelta(days=10),
255 verified_at=now() - timedelta(days=1),
256 )
257 )
258 session.add(
259 PostalVerificationAttempt(
260 user_id=cancelled.id,
261 status=PostalVerificationStatus.cancelled,
262 address_line_1="2 Test Street",
263 city="Testing city",
264 country_code="US",
265 )
266 )
267 session.add(
268 PostalVerificationAttempt(
269 user_id=pending.id,
270 status=PostalVerificationStatus.pending_address_confirmation,
271 address_line_1="3 Test Street",
272 city="Testing city",
273 country_code="US",
274 )
275 )
278@_populates(HostRequest)
279def _populate_host_requests() -> None:
280 surfer, _ = generate_user()
281 host, _ = generate_user()
282 today = date.today()
284 with session_scope() as session:
285 # the stay just ended, so the reference window is open
286 _host_request(session, surfer.id, host.id, HostRequestStatus.accepted, today - timedelta(days=1))
287 # the reference window closed 14 days after the stay
288 _host_request(session, surfer.id, host.id, HostRequestStatus.confirmed, today - timedelta(days=30))
289 # the stay hasn't happened yet
290 _host_request(session, surfer.id, host.id, HostRequestStatus.confirmed, today + timedelta(days=30))
291 # never went ahead
292 _host_request(session, surfer.id, host.id, HostRequestStatus.rejected, today - timedelta(days=1))
295def _host_request(
296 session: Session, surfer_id: int, host_id: int, status: HostRequestStatus, to_date: date
297) -> HostRequest:
298 conversation = Conversation()
299 session.add(conversation)
300 session.flush()
301 moderation_state = create_moderation(
302 session=session,
303 object_type=ModerationObjectType.host_request,
304 object_id=conversation.id,
305 creator_user_id=surfer_id,
306 )
307 host_request = HostRequest(
308 conversation_id=conversation.id,
309 initiator_user_id=surfer_id,
310 recipient_user_id=host_id,
311 proposal_kind=StayProposalKind.request,
312 moderation_state_id=moderation_state.id,
313 from_date=to_date - timedelta(days=2),
314 to_date=to_date,
315 status=status,
316 hosting_city="Testing city",
317 hosting_location=create_coordinate(40.7108, -73.9740),
318 hosting_radius=100,
319 )
320 session.add(host_request)
321 session.flush()
322 return host_request
325@_populates(EventOccurrence)
326def _populate_event_occurrences() -> None:
327 creator, _ = generate_user()
329 with session_scope() as session:
330 node = Node(
331 geom=to_multi(create_polygon_lat_lng([[0, 0], [0, 2], [2, 2], [2, 0], [0, 0]])),
332 node_type=NodeType.world,
333 )
334 session.add(node)
335 session.flush()
336 event = Event(
337 parent_node_id=node.id,
338 title="Testing event",
339 creator_user_id=creator.id,
340 owner_user_id=creator.id,
341 )
342 session.add(event)
343 session.flush()
345 # occurrences may not overlap within an event
346 for days, hours in [(1, 2), (10, 3)]:
347 start = now() + timedelta(days=days)
349 thread = Thread()
350 session.add(thread)
351 session.flush()
353 def create_occurrence(moderation_state_id: int, start=start, hours=hours, thread=thread) -> int:
354 occurrence = EventOccurrence(
355 event_id=event.id,
356 moderation_state_id=moderation_state_id,
357 creator_user_id=creator.id,
358 content="Testing event occurrence",
359 geom=create_coordinate(1, 1),
360 address="Somewhere",
361 timezone="Etc/UTC",
362 during=TimestamptzRange(start, start + timedelta(hours=hours)),
363 thread_id=thread.id,
364 )
365 session.add(occurrence)
366 session.flush()
367 return occurrence.id
369 create_moderation(
370 session=session,
371 object_type=ModerationObjectType.event_occurrence,
372 object_id=create_occurrence,
373 creator_user_id=creator.id,
374 )
377@_populates(GroupChatSubscription)
378def _populate_group_chat_subscriptions() -> None:
379 creator, _ = generate_user()
380 other, _ = generate_user()
382 with session_scope() as session:
383 conversation = Conversation()
384 session.add(conversation)
385 session.flush()
386 moderation_state = create_moderation(
387 session=session,
388 object_type=ModerationObjectType.group_chat,
389 object_id=conversation.id,
390 creator_user_id=creator.id,
391 )
392 session.add(
393 GroupChat(
394 conversation_id=conversation.id,
395 creator_id=creator.id,
396 is_dm=True,
397 moderation_state_id=moderation_state.id,
398 )
399 )
400 muted = GroupChatSubscription(user_id=creator.id, group_chat_id=conversation.id, role=GroupChatRole.admin)
401 session.add(muted)
402 session.add(
403 GroupChatSubscription(user_id=other.id, group_chat_id=conversation.id, role=GroupChatRole.participant)
404 )
405 session.flush()
406 session.execute(
407 update(GroupChatSubscription)
408 .where(GroupChatSubscription.id == muted.id)
409 .values(muted_until=now() + timedelta(days=7))
410 )
413@_populates(UserSession)
414def _populate_user_sessions() -> None:
415 user, _ = generate_user()
416 with session_scope() as session:
417 session.add(UserSession(token=random_hex(32), user_id=user.id, long_lived=True, is_api_key=True))
418 session.add(
419 UserSession(token=random_hex(32), user_id=user.id, long_lived=False, is_api_key=False, deleted=now())
420 )
423@_populates(LoginToken)
424def _populate_login_tokens() -> None:
425 user, _ = generate_user()
426 with session_scope() as session:
427 session.add(LoginToken(token=random_hex(32), user_id=user.id, expiry=now() + timedelta(hours=1)))
428 session.add(LoginToken(token=random_hex(32), user_id=user.id, expiry=now() - timedelta(hours=1)))
431@_populates(PasswordResetToken)
432def _populate_password_reset_tokens() -> None:
433 user, _ = generate_user()
434 with session_scope() as session:
435 session.add(PasswordResetToken(token=random_hex(32), user_id=user.id, expiry=now() + timedelta(hours=1)))
436 session.add(PasswordResetToken(token=random_hex(32), user_id=user.id, expiry=now() - timedelta(hours=1)))
439@_populates(AccountDeletionToken)
440def _populate_account_deletion_tokens() -> None:
441 user, _ = generate_user()
442 with session_scope() as session:
443 session.add(AccountDeletionToken(token=random_hex(32), user_id=user.id, expiry=now() + timedelta(hours=1)))
444 session.add(AccountDeletionToken(token=random_hex(32), user_id=user.id, expiry=now() - timedelta(hours=1)))
447@_populates(InitiatedUpload)
448def _populate_initiated_uploads() -> None:
449 user, _ = generate_user()
450 with session_scope() as session:
451 session.add(
452 InitiatedUpload(
453 key=random_hex(32),
454 created=now() - timedelta(hours=1),
455 expiry=now() + timedelta(hours=1),
456 initiator_user_id=user.id,
457 )
458 )
459 session.add(
460 InitiatedUpload(
461 key=random_hex(32),
462 created=now() - timedelta(hours=2),
463 expiry=now() - timedelta(hours=1),
464 initiator_user_id=user.id,
465 )
466 )
469@_populates(ContributorForm)
470def _populate_contributor_forms() -> None:
471 user, _ = generate_user()
472 with session_scope() as session:
473 session.add(ContributorForm(user_id=user.id, contribute_ways=[]))
474 session.add(ContributorForm(user_id=user.id, contribute_ways=["community"]))
475 session.add(ContributorForm(user_id=user.id, contribute_ways=[], ideas="I have one"))
478@_populates(SignupFlow)
479def _populate_signup_flows() -> None:
480 with session_scope() as session:
481 # a flow that has been completed all the way through
482 session.add(
483 SignupFlow(
484 name="Completed",
485 email="completed@couchers.org.invalid",
486 flow_token=random_hex(32),
487 email_verified=True,
488 email_token=random_hex(32),
489 email_token_expiry=now() + timedelta(hours=1),
490 username="completed",
491 birthdate=date(1990, 1, 1),
492 gender="Woman",
493 hosting_status=HostingStatus.cant_host,
494 city="Testing city",
495 geom=create_coordinate(40.7108, -73.9740),
496 geom_radius=100,
497 accepted_tos=TOS_VERSION,
498 opt_out_of_newsletter=False,
499 filled_motivations=True,
500 )
501 )
502 # the email token has expired, and the account details were never filled in
503 session.add(
504 SignupFlow(
505 name="Expired",
506 email="expired@couchers.org.invalid",
507 flow_token=random_hex(32),
508 email_token=random_hex(32),
509 email_token_expiry=now() - timedelta(hours=1),
510 )
511 )
512 # never got as far as being sent an email
513 session.add(
514 SignupFlow(
515 name="Fresh",
516 email="fresh@couchers.org.invalid",
517 flow_token=random_hex(32),
518 )
519 )
521 for flow in session.execute(select(SignupFlow)).scalars().all():
522 if flow.name == "Completed":
523 flow.accepted_community_guidelines = GUIDELINES_VERSION
526@_populates(BackgroundJob)
527def _populate_background_jobs() -> None:
528 with session_scope() as session:
529 session.add(BackgroundJob(job_type="dummy_job", payload=b""))
530 session.add(BackgroundJob(job_type="dummy_job", payload=b"", state=BackgroundJobState.completed))
531 session.add(BackgroundJob(job_type="dummy_job", payload=b"", state=BackgroundJobState.error, try_count=5))
534## The tests
537def test_every_hybrid_is_covered() -> None:
538 """A new hybrid on a model has to bring a population with it, or it goes untested."""
539 models = {model for model, _ in HYBRID_PROPERTIES + HYBRID_METHODS}
540 assert models - POPULATIONS.keys() == set(), "these models have hybrids but no population"
541 assert POPULATIONS.keys() - models == set(), "these populations are for models without hybrids"
542 assert SQL_ONLY.keys() <= {_label(model, name) for model, name in HYBRID_PROPERTIES}, "stale SQL_ONLY entries"
543 assert {model for model, _ in HYBRID_METHODS} == {StrongVerificationAttempt}, (
544 "test_hybrid_method_agrees_with_sql only knows how to bind a User as the subject"
545 )
548COMPARABLE = [pair for pair in HYBRID_PROPERTIES if _label(*pair) not in SQL_ONLY]
549SQL_ONLY_PROPERTIES = [pair for pair in HYBRID_PROPERTIES if _label(*pair) in SQL_ONLY]
552@pytest.mark.parametrize(("model", "name"), COMPARABLE, ids=[_label(*pair) for pair in COMPARABLE])
553def test_hybrid_agrees_with_sql(db, model: type[Base], name: str) -> None:
554 POPULATIONS[model]()
556 with session_scope() as session:
557 mapper = inspect(model)
558 sql_values = _sql_values(session, model, name)
560 for instance in session.execute(select(model)).scalars():
561 key = tuple(mapper.primary_key_from_instance(instance))
562 python_value = getattr(instance, name)
563 assert _agree(python_value, sql_values[key]), (
564 f"{_label(model, name)} disagrees on {key}: python says {python_value!r}, "
565 f"postgres says {sql_values[key]!r}"
566 )
569@pytest.mark.parametrize(("model", "name"), SQL_ONLY_PROPERTIES, ids=[_label(*pair) for pair in SQL_ONLY_PROPERTIES])
570def test_sql_only_hybrids_are_inert_in_python(db, model: type[Base], name: str) -> None:
571 """
572 A hybrid with no python implementation must fail loudly rather than answer wrongly: reading it off
573 an instance either raises, or hands back a SQL expression that raises the moment anything reads it
574 as a boolean. If one of these ever starts returning a value, it needs comparing, not listing here.
575 """
576 POPULATIONS[model]()
578 with session_scope() as session:
579 _sql_values(session, model, name)
580 instances = session.execute(select(model)).scalars().all()
581 assert instances
582 for instance in instances:
583 with pytest.raises(TypeError):
584 value = getattr(instance, name)
585 assert isinstance(value, ClauseElement), f"{_label(model, name)} returns a python value now"
586 bool(value)
589@pytest.mark.parametrize(("model", "name"), HYBRID_METHODS, ids=[_label(model, name) for model, name in HYBRID_METHODS])
590def test_hybrid_method_agrees_with_sql(db, model: type[Base], name: str) -> None:
591 """
592 These take a subject, so they have two forms that have to agree: evaluated in python on a pair of
593 instances, and evaluated in SQL over the subject's table. Every pair is checked, not just the
594 matching ones: the lite_users bug was a query that reported the right answer for the pairs it was
595 meant to cover and a wrong one for everybody else.
596 """
597 POPULATIONS[model]()
599 with session_scope() as session:
600 mapper = inspect(model)
601 instances = session.execute(select(model)).scalars().all()
602 users = session.execute(select(User).order_by(User.id)).scalars().all()
603 assert len(instances) >= 2 and len(users) >= 2
605 seen = set()
606 for instance in instances:
607 key = tuple(mapper.primary_key_from_instance(instance))
608 for user in users:
609 python_value = getattr(instance, name)(user)
610 # the subject comes from the users table, exactly as it does in a real query; the join
611 # pins it to this one user so the row is the pair under test
612 sql_value: Any = session.execute(
613 select(getattr(model, name)(User))
614 .select_from(model)
615 .join(User, User.id == user.id)
616 .where(*(c == v for c, v in zip(mapper.primary_key, key)))
617 ).scalar_one()
618 assert _agree(python_value, sql_value), (
619 f"{_label(model, name)} on {key} against user {user.id}: python says "
620 f"{python_value!r}, postgres says {sql_value!r}"
621 )
622 seen.add(bool(sql_value))
624 assert seen == {True, False}, f"{_label(model, name)} takes the same value on every pair"
627def _sql_values(session: Session, model: type[Base], name: str) -> dict[tuple[Any, ...], Any]:
628 """The hybrid as postgres computes it, per row, and a check that the population actually varies it."""
629 mapper = inspect(model)
630 values = {
631 tuple(row[:-1]): row[-1] for row in session.execute(select(*mapper.primary_key, getattr(model, name))).all()
632 }
633 assert len(values) >= 2, "the population needs at least two rows to be worth comparing"
634 assert len(set(values.values())) >= 2, (
635 f"{_label(model, name)} takes the same value on every row: the population doesn't exercise it"
636 )
637 return values
640def _agree(python_value: Any, sql_value: Any) -> bool:
641 """
642 Postgres computes a predicate over a NULL column as NULL, which is falsy everywhere these hybrids
643 are used (WHERE, AND, OR), so python's False agrees with it. A python True against a NULL does not.
644 """
645 if sql_value is None:
646 return python_value is False
647 return bool(python_value == sql_value)