WIP checkpoint: Text filtering service + profanity config

backend/app/services/__init__.py

@@ -0,0 +1 @@
+"""Application services."""

backend/app/services/text_filter.py

@@ -0,0 +1,340 @@
+"""Pluggable server-side text filtering service.
+
+Provides a ``TextFilter`` protocol with a deterministic default
+implementation (profanity/term list with normalization) and a hook
+for third-party providers.
+
+Usage::
+
+    from app.services.text_filter import TextFilter, FilterResult, \
+        ProfanityTextFilter, get_text_filter
+
+    # Default factory reads config from the path below
+    filt = get_text_filter()
+    result = filt.check("Hello wor1d!")
+    # result.action == FilterAction.ALLOW
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import unicodedata
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from enum import Enum
+from pathlib import Path
+from typing import Any
+
+# ---------------------------------------------------------------------------
+# Enums / Data classes
+# ---------------------------------------------------------------------------
+
+
+class FilterAction(str, Enum):
+    """Action the filter wants the caller to take."""
+
+    ALLOW = "allow"
+    REDACT = "redact"
+    REJECT = "reject"
+
+
+@dataclass(frozen=True)
+class FilterResult:
+    """Immutable result of a single ``TextFilter.check()`` call.
+
+    Attributes:
+        action: What to do with the message.
+        filtered_text: The text after redaction (may equal *text* if
+            action is ``ALLOW``).
+        matched_terms: Sorted list of terms that triggered the filter.
+    """
+
+    action: FilterAction
+    filtered_text: str
+    matched_terms: list[str] = field(default_factory=list)
+
+    @property
+    def is_allowed(self) -> bool:
+        return self.action == FilterAction.ALLOW
+
+    @property
+    def is_rejected(self) -> bool:
+        return self.action == FilterAction.REJECT
+
+    @property
+    def is_redacted(self) -> bool:
+        return self.action == FilterAction.REDACT
+
+
+# ---------------------------------------------------------------------------
+# Protocol / abstract base
+# ---------------------------------------------------------------------------
+
+
+class TextFilter(ABC):
+    """Protocol for server-side text filters.
+
+    Every concrete filter must implement ``check()``.  The default
+    implementation (``ProfanityTextFilter``) is deterministic and
+    requires no external service.  A third-party provider can be
+    plugged in by subclassing ``TextFilter`` and overriding ``check``.
+    """
+
+    @abstractmethod
+    def check(self, text: str) -> FilterResult:
+        """Analyse *text* and return a ``FilterResult``.
+
+        Args:
+            text: Raw message text to filter.
+
+        Returns:
+            A ``FilterResult`` with the decision and any redacted text.
+        """
+
+
+# ---------------------------------------------------------------------------
+# Normalisation helpers
+# ---------------------------------------------------------------------------
+
+# Common leetspeak substitutions (char -> set of possible originals)
+_LEETSPEAK: dict[str, str] = {
+    "0": "o",
+    "1": "i",
+    "3": "e",
+    "4": "a",
+    "5": "s",
+    "7": "t",
+    "8": "b",
+    "9": "g",
+    "@": "a",
+}
+
+# Normalise separators that are often used to obfuscate words
+_SEPARATORS = re.compile(r"[\s_\-\.]+")
+
+
+def _normalize(text: str) -> str:
+    """Case-fold, strip diacritics, and replace leetspeak / separators.
+
+    This produces a canonical form for matching against the term list.
+    """
+    # Case-fold
+    text = text.casefold()
+    # Strip diacritics (e.g. é -> e)
+    text = unicodedata.normalize("NFKD", text)
+    text = "".join(c for c in text if not unicodedata.combining(c))
+    # Replace leetspeak
+    normalized = ""
+    for ch in text:
+        normalized += _LEETSPEAK.get(ch, ch)
+    # Collapse separators
+    normalized = _SEPARATORS.sub("", normalized)
+    return normalized
+
+
+# ---------------------------------------------------------------------------
+# Default implementation
+# ---------------------------------------------------------------------------
+
+# Default config path (relative to project root)
+_DEFAULT_CONFIG_PATH = Path(__file__).resolve().parent.parent.parent / "data" / "text_filter_config.json"
+
+
+@dataclass
+class _FilterPolicy:
+    """Filtering policy loaded from config."""
+
+    mode: str  # "redact" or "reject"
+    redaction_char: str = "*"
+    # Minimum term length to consider (avoids matching single chars)
+    min_term_length: int = 3
+    # If True, match terms case-insensitively after normalisation
+    case_insensitive: bool = True
+    # If True, match whole words only (after normalisation)
+    whole_word: bool = True
+
+
+class ProfanityTextFilter(TextFilter):
+    """Deterministic text filter backed by a JSON term list.
+
+    The config file is a JSON object with:
+
+    .. code-block:: json
+
+        {
+            "mode": "redact",
+            "redaction_char": "*",
+            "min_term_length": 3,
+            "case_insensitive": true,
+            "whole_word": true,
+            "terms": ["badword", "another", ...]
+        }
+
+    Terms are matched after normalisation (case-fold, leetspeak
+    substitution, separator removal, diacritic stripping).
+    """
+
+    def __init__(
+        self,
+        config_path: str | Path | None = None,
+        terms: list[str] | None = None,
+        policy: _FilterPolicy | None = None,
+    ) -> None:
+        """
+        Args:
+            config_path: Path to a JSON config file.  Ignored if *terms*
+                is provided.
+            terms: Inline list of terms (overrides config file).
+            policy: Filter policy.  Defaults to a redact policy.
+        """
+        if terms is not None:
+            self._terms: list[str] = [t.lower() for t in terms]
+        elif config_path is not None:
+            self._terms = self._load_terms(Path(config_path))
+        else:
+            # Try default path
+            if _DEFAULT_CONFIG_PATH.exists():
+                self._terms = self._load_terms(_DEFAULT_CONFIG_PATH)
+            else:
+                self._terms = []
+
+        self._policy = policy or _FilterPolicy(mode="redact")
+        # Pre-compile patterns for each term
+        self._patterns: list[tuple[re.Pattern[str], str]] = []
+        for term in self._terms:
+            if len(term) < self._policy.min_term_length:
+                continue
+            if self._policy.whole_word:
+                pattern = re.compile(
+                    r"(?<!\w)" + re.escape(term) + r"(?!\w)",
+                    re.IGNORECASE if self._policy.case_insensitive else 0,
+                )
+            else:
+                pattern = re.compile(
+                    re.escape(term),
+                    re.IGNORECASE if self._policy.case_insensitive else 0,
+                )
+            self._patterns.append((pattern, term))
+
+    # -- config loading --------------------------------------------------
+
+    @staticmethod
+    def _load_terms(config_path: Path) -> list[str]:
+        """Load terms from a JSON config file."""
+        with open(config_path, "r", encoding="utf-8") as f:
+            data: dict[str, Any] = json.load(f)
+
+        terms: list[str] = []
+        for raw in data.get("terms", []):
+            term = str(raw).strip()
+            if term:
+                terms.append(term.casefold())
+        return terms
+
+    # -- public API ------------------------------------------------------
+
+    def check(self, text: str) -> FilterResult:
+        """Check *text* against the loaded term list.
+
+        Returns a ``FilterResult`` with the appropriate action,
+        filtered text, and matched terms.
+        """
+        if not text:
+            return FilterResult(
+                action=FilterAction.ALLOW,
+                filtered_text=text,
+                matched_terms=[],
+            )
+
+        normalized = _normalize(text)
+        matched_terms: list[str] = []
+
+        # Collect all matches
+        for pattern, term in self._patterns:
+            if pattern.search(normalized):
+                if term not in matched_terms:
+                    matched_terms.append(term)
+
+        if not matched_terms:
+            return FilterResult(
+                action=FilterAction.ALLOW,
+                filtered_text=text,
+                matched_terms=[],
+            )
+
+        matched_terms.sort()
+
+        # Build filtered text by replacing matched terms
+        filtered = text
+        for term in matched_terms:
+            norm_term = _normalize(term)
+            # Find all occurrences in the normalised text and replace in original
+            # We need to map positions from normalised text back to original
+            filtered = self._replace_terms(filtered, term, norm_term)
+
+        action = FilterAction.REJECT if self._policy.mode == "reject" else FilterAction.REDACT
+
+        return FilterResult(
+            action=action,
+            filtered_text=filtered,
+            matched_terms=matched_terms,
+        )
+
+    def _replace_terms(
+        self, text: str, term: str, norm_term: str
+    ) -> str:
+        """Replace occurrences of *term* in *text* with redaction chars.
+
+        Uses normalised text for matching but preserves original casing
+        in the output.
+        """
+        norm_text = _normalize(text)
+        norm_term_len = len(norm_term)
+        result_chars = list(text)
+
+        start = 0
+        while start < len(norm_text):
+            idx = norm_text.find(norm_term, start)
+            if idx == -1:
+                break
+            # Replace characters in original text
+            for i in range(idx, min(idx + norm_term_len, len(result_chars))):
+                result_chars[i] = self._policy.redaction_char
+            start = idx + norm_term_len
+
+        return "".join(result_chars)
+
+    @property
+    def terms(self) -> list[str]:
+        """Read-only access to the loaded term list."""
+        return list(self._terms)
+
+    @property
+    def policy(self) -> _FilterPolicy:
+        """Read-only access to the filter policy."""
+        return self._policy
+
+
+# ---------------------------------------------------------------------------
+# Factory
+# ---------------------------------------------------------------------------
+
+
+def get_text_filter(
+    config_path: str | Path | None = None,
+    terms: list[str] | None = None,
+    policy: _FilterPolicy | None = None,
+) -> TextFilter:
+    """Return a ``TextFilter`` instance.
+
+    By default this creates a ``ProfanityTextFilter``.  To use a
+    third-party provider, subclass ``TextFilter`` and pass an
+    instance here instead.
+    """
+    return ProfanityTextFilter(
+        config_path=config_path,
+        terms=terms,
+        policy=policy,
+    )