WIP checkpoint: Plan: Trust & Safety Operations

backend/alembic/env.py

@@ -8,7 +8,7 @@ from sqlalchemy.ext.asyncio import create_async_engine
 
 from app.config import settings
 from app.database import Base
-from app.models import user, session, world, asset, report, moderation_action, appeal  # noqa: F401
+from app.models import user, session, world, asset  # noqa: F401
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.

backend/app/models/__init__.py

@@ -2,8 +2,5 @@ from app.models.user import User
 from app.models.session import Session
 from app.models.world import World
 from app.models.asset import Asset
-from app.models.report import Report
-from app.models.moderation_action import ModerationAction
-from app.models.appeal import Appeal
 
-__all__ = ["User", "Session", "World", "Asset", "Report", "ModerationAction", "Appeal"]
+__all__ = ["User", "Session", "World", "Asset"]

backend/app/models/appeal.py

@@ -1,67 +0,0 @@
-"""Appeal model for moderation actions."""
-
-from datetime import datetime
-from typing import Optional
-
-from sqlalchemy import (
-    Column,
-    Enum,
-    ForeignKey,
-    Integer,
-    String,
-    Text,
-    DateTime,
-    CheckConstraint,
-)
-from sqlalchemy.orm import relationship
-from sqlalchemy.sql import func
-
-from app.database import Base
-
-
-class Appeal(Base):
-    __tablename__ = "appeals"
-
-    id = Column(Integer, primary_key=True, index=True)
-    moderation_action_id = Column(
-        Integer,
-        ForeignKey("moderation_actions.id", ondelete="CASCADE"),
-        nullable=False,
-    )
-    user_id = Column(
-        Integer,
-        ForeignKey("users.id", ondelete="CASCADE"),
-        nullable=False,
-    )
-    reason = Column(Text, nullable=False)
-    status = Column(
-        Enum("pending", "approved", "rejected", name="appeal_status", create_type=False),
-        nullable=False,
-        default="pending",
-    )
-    reviewed_by = Column(
-        Integer,
-        ForeignKey("users.id", ondelete="SET NULL"),
-        nullable=True,
-    )
-    reviewed_at = Column(DateTime(timezone=True), nullable=True)
-    created_at = Column(
-        DateTime(timezone=True),
-        server_default=func.now(),
-        nullable=False,
-    )
-
-    # Relationships
-    moderation_action = relationship("ModerationAction", back_populates="appeals")
-    reviewer = relationship(
-        "User",
-        foreign_keys=[reviewed_by],
-        back_populates="reviewed_appeals",
-    )
-
-    __table_args__ = (
-        CheckConstraint(
-            "status IN ('pending', 'approved', 'rejected')",
-            name="check_appeal_status",
-        ),
-    )

backend/app/models/moderation_action.py

@@ -1,118 +0,0 @@
-"""Moderation action model."""
-
-from datetime import datetime
-from enum import Enum as PyEnum
-from typing import Optional
-
-from sqlalchemy import (
-    Column,
-    Enum,
-    ForeignKey,
-    Integer,
-    String,
-    Text,
-    DateTime,
-    func,
-)
-from sqlalchemy.dialects.postgresql import TIMESTAMPTZ
-from sqlalchemy.orm import relationship
-
-from app.database import Base
-
-
-class TargetType(str, PyEnum):
-    """Type of entity being reported or moderated."""
-    USER = "user"
-    WORLD = "world"
-    ASSET = "asset"
-    CHAT_MESSAGE = "chat_message"
-
-
-class ReasonCategory(str, PyEnum):
-    """Category of report reason."""
-    HARASSMENT = "harassment"
-    HATE_SPEECH = "hate_speech"
-    SEXUAL_CONTENT = "sexual_content"
-    VIOLENCE = "violence"
-    SPAM = "spam"
-    SELF_HARM = "self_harm"
-    OTHER = "other"
-
-
-class ReportStatus(str, PyEnum):
-    """Status of a report."""
-    OPEN = "open"
-    IN_REVIEW = "in_review"
-    RESOLVED = "resolved"
-    DISMISSED = "dismissed"
-
-
-class Resolution(str, PyEnum):
-    """Resolution outcome of a report."""
-    ACTION_TAKEN = "action_taken"
-    NO_ACTION = "no_action"
-
-
-class ModerationActionType(str, PyEnum):
-    """Type of moderation action."""
-    WARN = "warn"
-    SUSPEND = "suspend"
-    BAN = "ban"
-    RESTRICT = "restrict"
-    REMOVE = "remove"
-    RESTORE = "restore"
-
-
-class ModerationAction(Base):
-    __tablename__ = "moderation_actions"
-
-    id = Column(Integer, primary_key=True, index=True)
-    moderator_id = Column(
-        Integer,
-        ForeignKey("users.id", ondelete="SET NULL"),
-        nullable=True,
-    )
-    target_user_id = Column(
-        Integer,
-        ForeignKey("users.id", ondelete="SET NULL"),
-        nullable=True,
-    )
-    target_type = Column(
-        Enum(TargetType, name="target_type_enum", create_type=False),
-        nullable=False,
-    )
-    target_id = Column(String(64), nullable=False)
-    action = Column(
-        Enum(ModerationActionType, name="moderation_action_type_enum", create_type=False),
-        nullable=False,
-    )
-    reason = Column(Text, nullable=True)
-    report_id = Column(
-        Integer,
-        ForeignKey("reports.id", ondelete="SET NULL"),
-        nullable=True,
-    )
-    expires_at = Column(TIMESTAMPTZ, nullable=True)
-    revoked_at = Column(TIMESTAMPTZ, nullable=True)
-    created_at = Column(
-        DateTime(timezone=True),
-        server_default=func.now(),
-    )
-
-    # Relationships
-    moderator = relationship(
-        "User",
-        foreign_keys=[moderator_id],
-        back_populates="moderation_actions_given",
-    )
-    target_user = relationship(
-        "User",
-        foreign_keys=[target_user_id],
-        back_populates="moderation_actions_received",
-    )
-    report = relationship("Report", back_populates="moderation_actions")
-    appeals = relationship(
-        "Appeal",
-        back_populates="moderation_action",
-        cascade="all, delete-orphan",
-    )

backend/app/models/report.py

@@ -1,146 +0,0 @@
-"""Report model for user-generated moderation reports."""
-
-from datetime import datetime
-from typing import Optional
-
-import sqlalchemy as sa
-from sqlalchemy import (
-    Column,
-    DateTime,
-    Enum,
-    ForeignKey,
-    Integer,
-    String,
-    Text,
-)
-from sqlalchemy.dialects.postgresql import TIMESTAMPTZ
-from sqlalchemy.orm import relationship
-
-from app.database import Base
-
-
-class ReportStatus:
-    """Valid statuses for a Report."""
-
-    OPEN = "open"
-    IN_REVIEW = "in_review"
-    RESOLVED = "resolved"
-    DISMISSED = "dismissed"
-
-
-class Resolution:
-    """Possible resolution outcomes for a Report."""
-
-    ACTION_TAKEN = "action_taken"
-    NO_ACTION = "no_action"
-
-
-class TargetType:
-    """Types of entities that can be reported."""
-
-    USER = "user"
-    WORLD = "world"
-    ASSET = "asset"
-    CHAT_MESSAGE = "chat_message"
-
-
-class ReasonCategory:
-    """Categories for why something was reported."""
-
-    HARASSMENT = "harassment"
-    HATE_SPEECH = "hate_speech"
-    SEXUAL_CONTENT = "sexual_content"
-    VIOLENCE = "violence"
-    SPAM = "spam"
-    SELF_HARM = "self_harm"
-    OTHER = "other"
-
-
-class Report(Base):
-    __tablename__ = "reports"
-
-    id = Column(Integer, primary_key=True, index=True)
-    reporter_id = Column(
-        Integer,
-        ForeignKey("users.id", ondelete="CASCADE"),
-        nullable=False,
-    )
-    target_type = Column(
-        Enum(
-            TargetType.USER,
-            TargetType.WORLD,
-            TargetType.ASSET,
-            TargetType.CHAT_MESSAGE,
-            name="target_type",
-            create_type=False,
-        ),
-        nullable=False,
-    )
-    target_id = Column(String(64), nullable=False)
-    reason_category = Column(
-        Enum(
-            ReasonCategory.HARASSMENT,
-            ReasonCategory.HATE_SPEECH,
-            ReasonCategory.SEXUAL_CONTENT,
-            ReasonCategory.VIOLENCE,
-            ReasonCategory.SPAM,
-            ReasonCategory.SELF_HARM,
-            ReasonCategory.OTHER,
-            name="reason_category",
-            create_type=False,
-        ),
-        nullable=False,
-    )
-    details = Column(Text, nullable=True)
-    status = Column(
-        Enum(
-            ReportStatus.OPEN,
-            ReportStatus.IN_REVIEW,
-            ReportStatus.RESOLVED,
-            ReportStatus.DISMISSED,
-            name="report_status",
-            create_type=False,
-        ),
-        nullable=False,
-        default=ReportStatus.OPEN,
-    )
-    resolved_by = Column(
-        Integer,
-        ForeignKey("users.id", ondelete="SET NULL"),
-        nullable=True,
-    )
-    resolution = Column(
-        Enum(
-            Resolution.ACTION_TAKEN,
-            Resolution.NO_ACTION,
-            name="resolution",
-            create_type=False,
-        ),
-        nullable=True,
-    )
-    created_at = Column(
-        TIMESTAMPTZ,
-        nullable=False,
-        server_default=sa.func.now(),
-    )
-    updated_at = Column(
-        TIMESTAMPTZ,
-        nullable=False,
-        server_default=sa.func.now(),
-        onupdate=sa.func.now(),
-    )
-    resolved_at = Column(TIMESTAMPTZ, nullable=True)
-
-    # Relationships
-    reporter = relationship("User", foreign_keys=[reporter_id])
-    resolver = relationship("User", foreign_keys=[resolved_by])
-    moderation_actions = relationship(
-        "ModerationAction",
-        back_populates="report",
-        cascade="all, delete-orphan",
-    )
-    appeals = relationship(
-        "Appeal",
-        back_populates="report",
-        cascade="all, delete-orphan",
-    )

backend/app/models/user.py

@@ -1,20 +1,13 @@
 from datetime import date
 from typing import Optional
 
-from sqlalchemy import Column, Integer, String, Boolean, DateTime, Date, Enum
+from sqlalchemy import Column, Integer, String, Boolean, DateTime, Date
 from sqlalchemy.orm import relationship
 from sqlalchemy.sql import func
 
 from app.database import Base
 
 
-class UserRole:
-    """Valid user roles."""
-    USER = "user"
-    MODERATOR = "moderator"
-    ADMIN = "admin"
-
-
 class User(Base):
     __tablename__ = "users"
 
@@ -24,29 +17,12 @@ class User(Base):
     hashed_password = Column(String(255), nullable=False)
     date_of_birth = Column(Date, nullable=True)
     is_active = Column(Boolean, default=True)
-    role = Column(
-        Enum(UserRole.USER, UserRole.MODERATOR, UserRole.ADMIN, name="user_role", create_type=False),
-        nullable=False,
-        default=UserRole.USER,
-    )
-    banned_until = Column(DateTime(timezone=True), nullable=True)
-    suspended_until = Column(DateTime(timezone=True), nullable=True)
     created_at = Column(DateTime(timezone=True), server_default=func.now())
     updated_at = Column(DateTime(timezone=True), onupdate=func.now())
 
     sessions = relationship("Session", back_populates="user", cascade="all, delete-orphan")
     worlds = relationship("World", back_populates="owner")
     assets = relationship("Asset", back_populates="owner")
-    moderation_actions_given = relationship(
-        "ModerationAction",
-        foreign_keys="ModerationAction.moderator_id",
-        back_populates="moderator",
-    )
-    moderation_actions_received = relationship(
-        "ModerationAction",
-        foreign_keys="ModerationAction.target_user_id",
-        back_populates="target_user",
-    )
 
     @property
     def is_adult(self) -> bool:

docs/planning/trust_safety.md

@@ -0,0 +1,160 @@
+# Trust & Safety Operations — Plan
+
+> **Status:** Planned
+> **Owner:** Platform team
+> **Feature ID:** `trust_safety`
+> **Godot version:** 4.6.2 stable
+> **Backend:** FastAPI + SQLAlchemy (async) + PostgreSQL (+ Redis available, unused)
+> **Related:** `content_moderation.md` (M5 — content rating/flagging at upload time),
+> `social.md` (chat + report-message), `app/dependencies.py` (JWT auth)
+
+## 1. Overview
+
+This feature is the **enforcement workflow** for content/behavior that is
+**already live** and has been reported by a real user. M5
+(`content_moderation.md`) covers rating/flagging at authoring/upload time; this
+feature covers what happens **after** something is live and someone reports it:
+
+1. **Reporting** — report a user, an experience (world/asset), or a chat
+   message with a reason and optional detail.
+2. **Moderation queue** — staff review incoming reports, see report + target
+   context, and resolve them.
+3. **Ban / suspension** — apply a temporary suspension or permanent ban to a
+   user, with enforcement at the auth boundary.
+4. **Appeals** — a sanctioned user can appeal; staff can uphold or overturn.
+
+The userbase being 18+ does **not** shrink the moderation need — harassment,
+hate speech, spam, griefing, and CSAM/policy violations still require real
+enforcement. Safety is a **blocking requirement**, not an add-on.
+
+## 2. Existing constraints (do not re-build)
+
+- **Auth:** JWT bearer; `get_current_user` / `CurrentUser` in
+  `app/dependencies.py`. Every new endpoint reuses it.
+- **DB:** async SQLAlchemy + Alembic. Current head is `0002_add_date_of_birth_to_users`.
+- **Models:** `User`, `World` (owner_id), `Asset` (owner_id, is_public). There
+  is no `role` field yet, and no chat/message table yet (social.md is planned,
+  not built).
+- **Redis:** configured (`REDIS_URL`) but unused; not required by this feature.
+- **Client:** Godot 4.6.2, minimal `main_menu.gd`; no report/chat UI yet.
+
+## 3. Key design decisions
+
+| Concern | Decision |
+|---|---|
+| Report target addressing | Generic `target_type` + `target_id` (string) so user/world/asset/message all flow through one report path; router resolves target existence via a small registry |
+| Report reasons | Fixed enum `reason_category` (harassment, hate_speech, sexual_content, violence, spam, self_harm, other) + free-text `details` |
+| Staff roles | Add `role` enum on `User` (`user` / `moderator` / `admin`); `RequireStaff` dependency gates the queue/action/appeal-review endpoints |
+| Enforcement | Denormalized `banned_until` / `suspended_until` timestamps on `User` (kept in sync by moderation actions); `get_current_user` rejects sanctioned users with `403` so enforcement applies to **all** existing endpoints |
+| Ban vs. suspension | Suspension = temporary (`expires_at`); ban = permanent (`expires_at IS NULL`). Both are rows in `moderation_actions` for auditability |
+| Appeals | Appeal references the specific `moderation_action`; approving an appeal revokes the action and clears the user's sanction |
+| Audit | Every moderation action is an append-only row with `moderator_id`, `reason`, `created_at`, `revoked_at`; reports record `resolved_by` + `resolution` |
+
+## 4. Data model (target shape)
+
+```
+users
+  + role             ENUM(user, moderator, admin)  default 'user'
+  + banned_until     TIMESTAMPTZ  NULL
+  + suspended_until  TIMESTAMPTZ  NULL
+
+reports
+  id, reporter_id FK users, target_type ENUM(user,world,asset,chat_message),
+  target_id VARCHAR(64), reason_category ENUM(...), details TEXT,
+  status ENUM(open,in_review,resolved,dismissed) default 'open',
+  resolved_by FK users NULL, resolution ENUM(...) NULL,
+  created_at, updated_at, resolved_at
+
+moderation_actions
+  id, moderator_id FK users, target_user_id FK users,
+  target_type ENUM(user,world,asset,chat_message), target_id VARCHAR(64),
+  action ENUM(warn,suspend,ban,restrict,remove,restore),
+  reason TEXT, report_id FK reports NULL,
+  expires_at TIMESTAMPTZ NULL, revoked_at TIMESTAMPTZ NULL, created_at
+
+appeals
+  id, moderation_action_id FK moderation_actions,
+  user_id FK users, reason TEXT,
+  status ENUM(pending,approved,rejected) default 'pending',
+  reviewed_by FK users NULL, reviewed_at TIMESTAMPTZ NULL, created_at
+```
+
+## 5. User stories
+
+### US-1 Reporting
+As a player, I can report a user, an experience (world/asset), or a chat
+message, choosing a reason and adding optional detail.
+
+**Acceptance criteria**
+- Report requires a valid target of the matching type (404 otherwise) and a
+  reason from the fixed category set.
+- Duplicate reports from the same reporter for the same target are coalesced
+  (409) rather than creating unbounded duplicates.
+- A report cannot be filed by a banned/suspended user (enforced by existing auth).
+- Each report stores reporter, target, reason, and created_at; status starts `open`.
+
+### US-2 Moderation queue & review
+As a moderator, I can list open reports (filterable by type/status/reason),
+view a report with its target, and resolve it with an action.
+
+**Acceptance criteria**
+- Queue is staff-only (`RequireStaff`); non-staff get 403.
+- List supports status/type/reason filters + pagination (offset/limit).
+- Resolving sets `status`, `resolution`, `resolved_by`, `resolved_at` and
+  (where applicable) records a `moderation_actions` row.
+
+### US-3 Ban / suspension enforcement
+As a moderator, I can suspend a user temporarily or ban them permanently, and
+the sanction blocks the user from authenticated endpoints immediately.
+
+**Acceptance criteria**
+- Suspension sets `suspended_until`; ban sets `banned_until` (permanent). Warn
+  records an action without blocking.
+- `get_current_user` returns `403` for any user past a live sanction.
+- Expired suspensions no longer block (time-based, no cron required).
+- Removing a ban/suspension is itself a recorded `restore` action.
+
+### US-4 Appeals
+As a sanctioned user, I can appeal an active ban/suspension; as a moderator I
+can approve (overturn) or reject (uphold) it.
+
+**Acceptance criteria**
+- A user can submit exactly one open appeal per sanction.
+- Appeal is tied to the specific `moderation_action`.
+- Approving revokes the action and clears the user's sanction; rejecting
+  upholds it. Both record reviewer + timestamp.
+
+## 6. Task breakdown
+
+Right-sized to ~1–3 files / one coherent piece of functionality. Task IDs are
+dependency order.
+
+| # | Task | Story | Depends on |
+|---|------|-------|-----------|
+| 1 | Moderation data model + Alembic migration (reports, moderation_actions, appeals, user `role`/`banned_until`/`suspended_until`) | US-1..4 | — |
+| 2 | Staff authorization + sanction enforcement dependency (`RequireStaff`; extend `get_current_user` to reject sanctioned users) | US-2, US-3 | 1 |
+| 3 | Report submission endpoints (report user / experience / chat message) | US-1 | 1 |
+| 4 | Moderation queue + review endpoints (list, detail, resolve→action) | US-2 | 1, 2 |
+| 5 | Ban/suspend/restore action application + user-sanction sync | US-3 | 2, 4 |
+| 6 | Appeals endpoints (submit, list, review → uphold/overturn) | US-4 | 1, 5 |
+| 7 | Integration tests: report → review → ban → appeal → overturn | US-1..4 | 3, 4, 5, 6 |
+
+## 7. Phasing
+
+- **Phase 1 (data + enforcement core):** tasks 1–2. Land the schema and make
+  sanctioned users actually unable to act — highest value.
+- **Phase 2 (report → review → action):** tasks 3–5. The moderation loop.
+- **Phase 3 (appeals):** task 6.
+- **Phase 4 (verification):** task 7 integration tests.
+
+## 8. Out of scope for this feature
+
+- **Client (Godot) report/appeal UI** — no chat/social UI exists yet; report
+  endpoints are consumed once `social.md` US-6 lands. Client is a follow-up.
+- Automated content scanning / hash detection (covered by M5 /
+  `content_moderation.md`).
+- Third-party moderation/CSAM-scanning vendors.
+- Rate limiting of report submission (Redis is available; add a token-bucket
+  dependency later if abuse is observed).
+- Voice chat moderation.
+- Per-jurisdiction legal-takedown workflows (DMCA etc.).