Add test inventory + checkpoint test runner
docs/testing/test_inventory.md: honest, current list of what automated coverage actually exists (38 backend pytest tests across 2 files, 0 Godot tests merged yet) and what doesn't (no integration/endpoint tests at all, no coverage yet for the in-review RS256/moderation work) -- verified against a genuinely fresh checkout + fresh Postgres, not just read off the file tree. scripts/run_checkpoint_tests.py: runs the whole inventory in one shot -- throwaway Postgres, migrations, pytest, then starts the backend server and runs every scripts/smoke_test_*.gd against it headlessly. Verified working both with and without a smoke test present (temporarily tested against PC1's not-yet-merged scripts/api_client.gd + smoke_test_api.gd, removed before this commit -- that PR carries its own files). docs/testing/README.md: what running this at a Package Checkpoint review does and doesn't prove, and how to grow the inventory as real coverage gets added. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
claude-checkpoint-tooling
Executable
+170
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the full automated test inventory for a Package Checkpoint review.
|
||||
|
||||
See docs/testing/test_inventory.md for what each suite actually covers (and
|
||||
doesn't) -- keep that doc in sync as tests are added, this script just runs
|
||||
whatever's there.
|
||||
|
||||
Usage:
|
||||
python scripts/run_checkpoint_tests.py [--keep-postgres] [--postgres-port PORT]
|
||||
|
||||
Exits 0 if everything that ran passed, 1 otherwise. Suites with nothing to
|
||||
run (e.g. no scripts/smoke_test_*.gd yet, or no Godot binary found) are
|
||||
reported as SKIPPED, not counted as a failure.
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
BACKEND = ROOT / "backend"
|
||||
|
||||
|
||||
def run(cmd, cwd=None, env=None, capture=False):
|
||||
print(f"$ {' '.join(cmd)}" + (f" (cwd={cwd})" if cwd else ""))
|
||||
return subprocess.run(cmd, cwd=cwd, env=env, capture_output=capture, text=True)
|
||||
|
||||
|
||||
def find_godot():
|
||||
for candidate in ("godot", str(Path.home() / "godot" / "godot")):
|
||||
if shutil.which(candidate) or Path(candidate).is_file():
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def wait_for_http(url: str, timeout_s: int = 20) -> bool:
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
deadline = time.time() + timeout_s
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=2) as resp:
|
||||
if resp.status == 200:
|
||||
return True
|
||||
except (urllib.error.URLError, ConnectionError, OSError):
|
||||
pass
|
||||
time.sleep(1)
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--keep-postgres", action="store_true",
|
||||
help="Don't tear down the checkpoint Postgres container afterward")
|
||||
parser.add_argument("--postgres-port", default=os.environ.get("CHECKPOINT_POSTGRES_PORT", "5432"),
|
||||
help="Host port for the checkpoint's own Postgres container (default 5432 -- "
|
||||
"override if that's already in use on this machine)")
|
||||
parser.add_argument("--backend-port", default=os.environ.get("CHECKPOINT_BACKEND_PORT", "18080"),
|
||||
help="Port to run the backend server on for the Godot smoke tests")
|
||||
args = parser.parse_args()
|
||||
|
||||
results = {}
|
||||
|
||||
# --- Backend: bring up Postgres ---
|
||||
compose_env = os.environ.copy()
|
||||
compose_env["COMPOSE_PROJECT_NAME"] = "agp-checkpoint"
|
||||
compose_env["POSTGRES_HOST_PORT"] = args.postgres_port
|
||||
compose_env["POSTGRES_PORT"] = args.postgres_port # some branches use this name instead, see tools/definitions.py
|
||||
print("=== Backend: bringing up Postgres ===")
|
||||
run(["docker", "compose", "up", "-d", "postgres"], cwd=BACKEND, env=compose_env)
|
||||
|
||||
print("Waiting for Postgres to be ready...")
|
||||
pg_ready = False
|
||||
for _ in range(20):
|
||||
r = run(["docker", "compose", "exec", "-T", "postgres", "pg_isready", "-U", "agp_user", "-d", "agp_db"],
|
||||
cwd=BACKEND, env=compose_env, capture=True)
|
||||
if r.returncode == 0:
|
||||
pg_ready = True
|
||||
break
|
||||
time.sleep(1)
|
||||
if not pg_ready:
|
||||
print("Postgres never became ready -- aborting.")
|
||||
sys.exit(1)
|
||||
|
||||
db_url = f"postgresql+asyncpg://agp_user:agp_password@localhost:{args.postgres_port}/agp_db"
|
||||
backend_env = os.environ.copy()
|
||||
backend_env["DATABASE_URL"] = db_url
|
||||
|
||||
print("\n=== Backend: alembic upgrade head ===")
|
||||
r = run([sys.executable, "-m", "alembic", "upgrade", "head"], cwd=BACKEND, env=backend_env)
|
||||
results["migrations (fresh DB)"] = r.returncode == 0
|
||||
|
||||
print("\n=== Backend: pytest ===")
|
||||
r = run([sys.executable, "-m", "pytest", "-v"], cwd=BACKEND, env=backend_env)
|
||||
results["backend pytest"] = r.returncode == 0
|
||||
|
||||
# --- Godot: headless smoke tests (need a live backend server) ---
|
||||
godot_bin = find_godot()
|
||||
smoke_tests = sorted(glob.glob(str(ROOT / "scripts" / "smoke_test_*.gd")))
|
||||
|
||||
if not godot_bin:
|
||||
print("\n=== Godot: binary not found (checked PATH and ~/godot/godot), skipping smoke tests ===")
|
||||
results["godot smoke tests"] = None
|
||||
elif not smoke_tests:
|
||||
print("\n=== Godot: no scripts/smoke_test_*.gd found yet, nothing to run ===")
|
||||
results["godot smoke tests"] = None
|
||||
else:
|
||||
print(f"\n=== Backend: starting server on :{args.backend_port} for Godot smoke tests ===")
|
||||
server_proc = subprocess.Popen(
|
||||
[sys.executable, "-m", "uvicorn", "app.main:app", "--host", "127.0.0.1", "--port", args.backend_port],
|
||||
cwd=BACKEND, env=backend_env,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
)
|
||||
try:
|
||||
base_url = f"http://127.0.0.1:{args.backend_port}"
|
||||
if not wait_for_http(f"{base_url}/api/health"):
|
||||
print("Backend server never became healthy -- skipping Godot smoke tests.")
|
||||
results["godot smoke tests"] = False
|
||||
else:
|
||||
godot_env = os.environ.copy()
|
||||
godot_env["AGP_API_BASE_URL"] = base_url
|
||||
|
||||
print("=== Godot: import pass (builds the class cache) ===")
|
||||
run([godot_bin, "--headless", "--path", str(ROOT), "--import"], env=godot_env)
|
||||
run([godot_bin, "--headless", "--path", str(ROOT), "--quit"], env=godot_env)
|
||||
|
||||
godot_ok = True
|
||||
for test in smoke_tests:
|
||||
rel = os.path.relpath(test, ROOT)
|
||||
print(f"\n--- {rel} ---")
|
||||
r = run([godot_bin, "--headless", "--path", str(ROOT), "--script", f"res://{rel}"], env=godot_env)
|
||||
if r.returncode != 0:
|
||||
godot_ok = False
|
||||
results["godot smoke tests"] = godot_ok
|
||||
finally:
|
||||
server_proc.terminate()
|
||||
try:
|
||||
server_proc.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
server_proc.kill()
|
||||
|
||||
if not args.keep_postgres:
|
||||
print("\n=== Backend: tearing down checkpoint Postgres ===")
|
||||
run(["docker", "compose", "down", "-v", "postgres"], cwd=BACKEND, env=compose_env)
|
||||
|
||||
# --- Summary ---
|
||||
print("\n" + "=" * 50)
|
||||
print("SUMMARY")
|
||||
print("=" * 50)
|
||||
overall_ok = True
|
||||
for name, ok in results.items():
|
||||
if ok is None:
|
||||
print(f" {name}: SKIPPED")
|
||||
elif ok:
|
||||
print(f" {name}: PASS")
|
||||
else:
|
||||
print(f" {name}: FAIL")
|
||||
overall_ok = False
|
||||
|
||||
print("\nSee docs/testing/test_inventory.md for what each suite actually covers (and doesn't).")
|
||||
sys.exit(0 if overall_ok else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user