From 634b2e39c0f4e693c5c09a5b12f10aca83b1b770 Mon Sep 17 00:00:00 2001 From: claude-checkpoint-tooling Date: Wed, 19 Aug 2026 15:18:49 +0000 Subject: [PATCH] Add test inventory + checkpoint test runner docs/testing/test_inventory.md: honest, current list of what automated coverage actually exists (38 backend pytest tests across 2 files, 0 Godot tests merged yet) and what doesn't (no integration/endpoint tests at all, no coverage yet for the in-review RS256/moderation work) -- verified against a genuinely fresh checkout + fresh Postgres, not just read off the file tree. scripts/run_checkpoint_tests.py: runs the whole inventory in one shot -- throwaway Postgres, migrations, pytest, then starts the backend server and runs every scripts/smoke_test_*.gd against it headlessly. Verified working both with and without a smoke test present (temporarily tested against PC1's not-yet-merged scripts/api_client.gd + smoke_test_api.gd, removed before this commit -- that PR carries its own files). docs/testing/README.md: what running this at a Package Checkpoint review does and doesn't prove, and how to grow the inventory as real coverage gets added. Co-Authored-By: Claude Sonnet 5 --- docs/testing/README.md | 50 ++++++++++ docs/testing/test_inventory.md | 78 +++++++++++++++ scripts/run_checkpoint_tests.py | 170 ++++++++++++++++++++++++++++++++ 3 files changed, 298 insertions(+) create mode 100644 docs/testing/README.md create mode 100644 docs/testing/test_inventory.md create mode 100755 scripts/run_checkpoint_tests.py diff --git a/docs/testing/README.md b/docs/testing/README.md new file mode 100644 index 0000000..fdc0489 --- /dev/null +++ b/docs/testing/README.md @@ -0,0 +1,50 @@ +# Testing at Package Checkpoints + +A Package Checkpoint (PC1, PC2, ...) is a periodic review point — every few +real milestones, the client and server builds get packaged together and +independently reviewed before anyone calls anything actually done. See +`project_agp` in Claude's memory, or ask, for the full definition; this +doc is just the mechanical "how do I run the tests" part of that review. + +## Running the suite + +``` +python scripts/run_checkpoint_tests.py +``` + +Brings up a throwaway Postgres, runs Alembic migrations against it, runs the +backend pytest suite, starts the backend server, and runs every Godot +headless smoke test (`scripts/smoke_test_*.gd`) against it. Tears the +Postgres container down afterward (`--keep-postgres` to skip that if you +want to poke at the DB after a failure). + +If port 5432 is already taken on the machine, pass `--postgres-port` with +something else. + +Exit code is 0 only if everything that actually ran passed — suites with +nothing to run yet (no Godot binary found, no smoke test scripts yet) are +reported SKIPPED, not treated as a failure. + +## What this does and doesn't prove + +Passing this script means: migrations apply cleanly to a genuinely empty +database, the backend's unit-level logic is correct, and whatever Godot +smoke tests exist actually round-trip against a live backend. It does +**not** mean full endpoint/integration coverage exists — see +`test_inventory.md` for the current, honest list of what's tested and what +isn't. A checkpoint review should still involve reading the actual diffs +for anything this script's current coverage doesn't reach, not just running +green and calling it done. + +## Growing this over time + +As real integration tests get added (endpoint tests hitting the FastAPI app, +more Godot smoke coverage beyond `health_check`), they should land under +`backend/tests/` / `scripts/smoke_test_*.gd` respectively — this script +picks up new files automatically, nothing to register by hand. Update +`test_inventory.md`'s tables when you do, so the inventory stays a source +of truth instead of quietly going stale. + +Eventual goal (not implemented yet): gate merges to `main` on this script +passing, the same way a real CI pipeline would. For now it's a manual step +run at checkpoint review time. diff --git a/docs/testing/test_inventory.md b/docs/testing/test_inventory.md new file mode 100644 index 0000000..9bfd3eb --- /dev/null +++ b/docs/testing/test_inventory.md @@ -0,0 +1,78 @@ +# Test Inventory + +What automated coverage actually exists, what it actually verifies, and what +doesn't have coverage yet. Keep this current as tests are added — it's the +thing a Package Checkpoint review (see `docs/testing/README.md`) reads first +to know what "run the tests" actually checked. + +Run everything below in one shot with `scripts/run_checkpoint_tests.py`. + +Last verified against `main`: 2026-08-19 (commit `1c75367`). + +## Backend — `backend/tests/` (pytest) + +Run: `cd backend && python -m pytest -v` (needs `DATABASE_URL` pointed at a +reachable Postgres — see the runner script; most of these tests don't +actually touch the DB, but the app's import chain requires a valid URL to +construct). + +**Current total: 38 tests, all passing on a clean `main` checkout.** + +| File | Tests | What it covers | +|---|---|---| +| `test_auth_age_gate.py` | 10 | `calculate_age()` date-math correctness (exact boundary, leap years, birthday-not-yet-this-year) + `RegisterRequest` date-of-birth validation (parametrized valid/invalid date rejection). Backs the 18+ registration gate (PR #23). | +| `test_gltf_metadata.py` | 21 | glTF/GLB binary + JSON parsing: magic-byte detection, chunk-padding handling, mesh/animation/texture-reference metadata extraction, malformed-input rejection. Backs the asset-import pipeline (PR #27). | + +**What these are, honestly:** pure unit tests of standalone functions +(`calculate_age`, the glTF parser, Pydantic validators). None of them spin +up the FastAPI app or hit a real endpoint — they run in ~1s and don't +actually need Postgres reachable at all (the import chain just needs a +syntactically valid `DATABASE_URL`, nothing connects during collection). + +**Known coverage gaps** (real gaps, not placeholders — nothing enforces filling +these yet): +- No integration/endpoint tests at all — `/register`, `/login`, `/refresh`, + `/worlds`, `/assets/*` have zero tests that actually call them through + the FastAPI app (e.g. via `httpx.AsyncClient` + `ASGITransport`) against a + real (or test) database. Everything currently shipped works because it + was manually verified at merge time (checkpoint review), not because a + test asserts it. +- No tests for the RS256 auth upgrade (key generation, refresh-token + issuance/persistence) — PR #30, pending review as of this writing, + manually verified (correct key usage, all 38 existing tests still pass) + but not covered by an automated test of its own yet. +- No tests for the moderation schema/migration (in progress as of this + writing) beyond "the migration runs." + +## Godot client — `scripts/smoke_test_*.gd` + +**Nothing merged into `main` yet as of this writing** — `sandbox_probe/test_script.gd` +is the only Godot script on `main` right now, and it's **not a real +test** (a leftover manual probe, returns a hardcoded string, asserts +nothing). Don't count it toward coverage; delete or repurpose it if anyone +actually needs a sandbox probe again. + +`scripts/smoke_test_api.gd` (PC1, PR #29, pending review as of this writing) +will be the first real one once merged — `ApiClient` autoload's +`health_check()` against a **live backend** (set `AGP_API_BASE_URL` to point +at one), verified by hand end-to-end before opening the PR: exercises the +real HTTP request path (`HTTPRequest` node lifecycle, JSON parse, the +result-dictionary contract), not just that the script compiles. Exits 1 on +failure, so it's CI/script-friendly. + +Run (once merged): `godot --headless --path . --import` once (builds the +class cache — skip this and every `class_name`-declared class falsely +errors as undeclared), then for each `scripts/smoke_test_*.gd`: +`godot --headless --path . --script res://scripts/.gd` + +**Known gap once it lands:** only `health_check()` is exercised end-to-end. +`login`, `register`, `get_worlds`, `get_me` are implemented (see +`scripts/api_client.gd`) but have no smoke coverage yet — each would need a +real user/world fixture in the target backend to test against meaningfully. + +## What "full coverage" should eventually mean here + +Not defined precisely yet — the honest target as of 2026-08-19 is just +"more than pure-function unit tests." As real integration/endpoint tests get +added, update the tables above rather than letting this doc drift stale; +a wrong inventory is worse than no inventory. diff --git a/scripts/run_checkpoint_tests.py b/scripts/run_checkpoint_tests.py new file mode 100755 index 0000000..e951de2 --- /dev/null +++ b/scripts/run_checkpoint_tests.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Run the full automated test inventory for a Package Checkpoint review. + +See docs/testing/test_inventory.md for what each suite actually covers (and +doesn't) -- keep that doc in sync as tests are added, this script just runs +whatever's there. + +Usage: + python scripts/run_checkpoint_tests.py [--keep-postgres] [--postgres-port PORT] + +Exits 0 if everything that ran passed, 1 otherwise. Suites with nothing to +run (e.g. no scripts/smoke_test_*.gd yet, or no Godot binary found) are +reported as SKIPPED, not counted as a failure. +""" +import argparse +import glob +import os +import shutil +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +BACKEND = ROOT / "backend" + + +def run(cmd, cwd=None, env=None, capture=False): + print(f"$ {' '.join(cmd)}" + (f" (cwd={cwd})" if cwd else "")) + return subprocess.run(cmd, cwd=cwd, env=env, capture_output=capture, text=True) + + +def find_godot(): + for candidate in ("godot", str(Path.home() / "godot" / "godot")): + if shutil.which(candidate) or Path(candidate).is_file(): + return candidate + return None + + +def wait_for_http(url: str, timeout_s: int = 20) -> bool: + import urllib.request + import urllib.error + + deadline = time.time() + timeout_s + while time.time() < deadline: + try: + with urllib.request.urlopen(url, timeout=2) as resp: + if resp.status == 200: + return True + except (urllib.error.URLError, ConnectionError, OSError): + pass + time.sleep(1) + return False + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--keep-postgres", action="store_true", + help="Don't tear down the checkpoint Postgres container afterward") + parser.add_argument("--postgres-port", default=os.environ.get("CHECKPOINT_POSTGRES_PORT", "5432"), + help="Host port for the checkpoint's own Postgres container (default 5432 -- " + "override if that's already in use on this machine)") + parser.add_argument("--backend-port", default=os.environ.get("CHECKPOINT_BACKEND_PORT", "18080"), + help="Port to run the backend server on for the Godot smoke tests") + args = parser.parse_args() + + results = {} + + # --- Backend: bring up Postgres --- + compose_env = os.environ.copy() + compose_env["COMPOSE_PROJECT_NAME"] = "agp-checkpoint" + compose_env["POSTGRES_HOST_PORT"] = args.postgres_port + compose_env["POSTGRES_PORT"] = args.postgres_port # some branches use this name instead, see tools/definitions.py + print("=== Backend: bringing up Postgres ===") + run(["docker", "compose", "up", "-d", "postgres"], cwd=BACKEND, env=compose_env) + + print("Waiting for Postgres to be ready...") + pg_ready = False + for _ in range(20): + r = run(["docker", "compose", "exec", "-T", "postgres", "pg_isready", "-U", "agp_user", "-d", "agp_db"], + cwd=BACKEND, env=compose_env, capture=True) + if r.returncode == 0: + pg_ready = True + break + time.sleep(1) + if not pg_ready: + print("Postgres never became ready -- aborting.") + sys.exit(1) + + db_url = f"postgresql+asyncpg://agp_user:agp_password@localhost:{args.postgres_port}/agp_db" + backend_env = os.environ.copy() + backend_env["DATABASE_URL"] = db_url + + print("\n=== Backend: alembic upgrade head ===") + r = run([sys.executable, "-m", "alembic", "upgrade", "head"], cwd=BACKEND, env=backend_env) + results["migrations (fresh DB)"] = r.returncode == 0 + + print("\n=== Backend: pytest ===") + r = run([sys.executable, "-m", "pytest", "-v"], cwd=BACKEND, env=backend_env) + results["backend pytest"] = r.returncode == 0 + + # --- Godot: headless smoke tests (need a live backend server) --- + godot_bin = find_godot() + smoke_tests = sorted(glob.glob(str(ROOT / "scripts" / "smoke_test_*.gd"))) + + if not godot_bin: + print("\n=== Godot: binary not found (checked PATH and ~/godot/godot), skipping smoke tests ===") + results["godot smoke tests"] = None + elif not smoke_tests: + print("\n=== Godot: no scripts/smoke_test_*.gd found yet, nothing to run ===") + results["godot smoke tests"] = None + else: + print(f"\n=== Backend: starting server on :{args.backend_port} for Godot smoke tests ===") + server_proc = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "app.main:app", "--host", "127.0.0.1", "--port", args.backend_port], + cwd=BACKEND, env=backend_env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + try: + base_url = f"http://127.0.0.1:{args.backend_port}" + if not wait_for_http(f"{base_url}/api/health"): + print("Backend server never became healthy -- skipping Godot smoke tests.") + results["godot smoke tests"] = False + else: + godot_env = os.environ.copy() + godot_env["AGP_API_BASE_URL"] = base_url + + print("=== Godot: import pass (builds the class cache) ===") + run([godot_bin, "--headless", "--path", str(ROOT), "--import"], env=godot_env) + run([godot_bin, "--headless", "--path", str(ROOT), "--quit"], env=godot_env) + + godot_ok = True + for test in smoke_tests: + rel = os.path.relpath(test, ROOT) + print(f"\n--- {rel} ---") + r = run([godot_bin, "--headless", "--path", str(ROOT), "--script", f"res://{rel}"], env=godot_env) + if r.returncode != 0: + godot_ok = False + results["godot smoke tests"] = godot_ok + finally: + server_proc.terminate() + try: + server_proc.wait(timeout=10) + except subprocess.TimeoutExpired: + server_proc.kill() + + if not args.keep_postgres: + print("\n=== Backend: tearing down checkpoint Postgres ===") + run(["docker", "compose", "down", "-v", "postgres"], cwd=BACKEND, env=compose_env) + + # --- Summary --- + print("\n" + "=" * 50) + print("SUMMARY") + print("=" * 50) + overall_ok = True + for name, ok in results.items(): + if ok is None: + print(f" {name}: SKIPPED") + elif ok: + print(f" {name}: PASS") + else: + print(f" {name}: FAIL") + overall_ok = False + + print("\nSee docs/testing/test_inventory.md for what each suite actually covers (and doesn't).") + sys.exit(0 if overall_ok else 1) + + +if __name__ == "__main__": + main() -- 2.52.0