diff --git a/backend/app/services/__init__.py b/backend/app/services/__init__.py new file mode 100644 index 0000000..a69ee20 --- /dev/null +++ b/backend/app/services/__init__.py @@ -0,0 +1 @@ +"""Application services.""" diff --git a/backend/app/services/text_filter.py b/backend/app/services/text_filter.py new file mode 100644 index 0000000..87d99d8 --- /dev/null +++ b/backend/app/services/text_filter.py @@ -0,0 +1,340 @@ +"""Pluggable server-side text filtering service. + +Provides a ``TextFilter`` protocol with a deterministic default +implementation (profanity/term list with normalization) and a hook +for third-party providers. + +Usage:: + + from app.services.text_filter import TextFilter, FilterResult, \ + ProfanityTextFilter, get_text_filter + + # Default factory reads config from the path below + filt = get_text_filter() + result = filt.check("Hello wor1d!") + # result.action == FilterAction.ALLOW +""" + +from __future__ import annotations + +import json +import os +import re +import unicodedata +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import Any + +# --------------------------------------------------------------------------- +# Enums / Data classes +# --------------------------------------------------------------------------- + + +class FilterAction(str, Enum): + """Action the filter wants the caller to take.""" + + ALLOW = "allow" + REDACT = "redact" + REJECT = "reject" + + +@dataclass(frozen=True) +class FilterResult: + """Immutable result of a single ``TextFilter.check()`` call. + + Attributes: + action: What to do with the message. + filtered_text: The text after redaction (may equal *text* if + action is ``ALLOW``). + matched_terms: Sorted list of terms that triggered the filter. + """ + + action: FilterAction + filtered_text: str + matched_terms: list[str] = field(default_factory=list) + + @property + def is_allowed(self) -> bool: + return self.action == FilterAction.ALLOW + + @property + def is_rejected(self) -> bool: + return self.action == FilterAction.REJECT + + @property + def is_redacted(self) -> bool: + return self.action == FilterAction.REDACT + + +# --------------------------------------------------------------------------- +# Protocol / abstract base +# --------------------------------------------------------------------------- + + +class TextFilter(ABC): + """Protocol for server-side text filters. + + Every concrete filter must implement ``check()``. The default + implementation (``ProfanityTextFilter``) is deterministic and + requires no external service. A third-party provider can be + plugged in by subclassing ``TextFilter`` and overriding ``check``. + """ + + @abstractmethod + def check(self, text: str) -> FilterResult: + """Analyse *text* and return a ``FilterResult``. + + Args: + text: Raw message text to filter. + + Returns: + A ``FilterResult`` with the decision and any redacted text. + """ + + +# --------------------------------------------------------------------------- +# Normalisation helpers +# --------------------------------------------------------------------------- + +# Common leetspeak substitutions (char -> set of possible originals) +_LEETSPEAK: dict[str, str] = { + "0": "o", + "1": "i", + "3": "e", + "4": "a", + "5": "s", + "7": "t", + "8": "b", + "9": "g", + "@": "a", +} + +# Normalise separators that are often used to obfuscate words +_SEPARATORS = re.compile(r"[\s_\-\.]+") + + +def _normalize(text: str) -> str: + """Case-fold, strip diacritics, and replace leetspeak / separators. + + This produces a canonical form for matching against the term list. + """ + # Case-fold + text = text.casefold() + # Strip diacritics (e.g. é -> e) + text = unicodedata.normalize("NFKD", text) + text = "".join(c for c in text if not unicodedata.combining(c)) + # Replace leetspeak + normalized = "" + for ch in text: + normalized += _LEETSPEAK.get(ch, ch) + # Collapse separators + normalized = _SEPARATORS.sub("", normalized) + return normalized + + +# --------------------------------------------------------------------------- +# Default implementation +# --------------------------------------------------------------------------- + +# Default config path (relative to project root) +_DEFAULT_CONFIG_PATH = Path(__file__).resolve().parent.parent.parent / "data" / "text_filter_config.json" + + +@dataclass +class _FilterPolicy: + """Filtering policy loaded from config.""" + + mode: str # "redact" or "reject" + redaction_char: str = "*" + # Minimum term length to consider (avoids matching single chars) + min_term_length: int = 3 + # If True, match terms case-insensitively after normalisation + case_insensitive: bool = True + # If True, match whole words only (after normalisation) + whole_word: bool = True + + +class ProfanityTextFilter(TextFilter): + """Deterministic text filter backed by a JSON term list. + + The config file is a JSON object with: + + .. code-block:: json + + { + "mode": "redact", + "redaction_char": "*", + "min_term_length": 3, + "case_insensitive": true, + "whole_word": true, + "terms": ["badword", "another", ...] + } + + Terms are matched after normalisation (case-fold, leetspeak + substitution, separator removal, diacritic stripping). + """ + + def __init__( + self, + config_path: str | Path | None = None, + terms: list[str] | None = None, + policy: _FilterPolicy | None = None, + ) -> None: + """ + Args: + config_path: Path to a JSON config file. Ignored if *terms* + is provided. + terms: Inline list of terms (overrides config file). + policy: Filter policy. Defaults to a redact policy. + """ + if terms is not None: + self._terms: list[str] = [t.lower() for t in terms] + elif config_path is not None: + self._terms = self._load_terms(Path(config_path)) + else: + # Try default path + if _DEFAULT_CONFIG_PATH.exists(): + self._terms = self._load_terms(_DEFAULT_CONFIG_PATH) + else: + self._terms = [] + + self._policy = policy or _FilterPolicy(mode="redact") + # Pre-compile patterns for each term + self._patterns: list[tuple[re.Pattern[str], str]] = [] + for term in self._terms: + if len(term) < self._policy.min_term_length: + continue + if self._policy.whole_word: + pattern = re.compile( + r"(? list[str]: + """Load terms from a JSON config file.""" + with open(config_path, "r", encoding="utf-8") as f: + data: dict[str, Any] = json.load(f) + + terms: list[str] = [] + for raw in data.get("terms", []): + term = str(raw).strip() + if term: + terms.append(term.casefold()) + return terms + + # -- public API ------------------------------------------------------ + + def check(self, text: str) -> FilterResult: + """Check *text* against the loaded term list. + + Returns a ``FilterResult`` with the appropriate action, + filtered text, and matched terms. + """ + if not text: + return FilterResult( + action=FilterAction.ALLOW, + filtered_text=text, + matched_terms=[], + ) + + normalized = _normalize(text) + matched_terms: list[str] = [] + + # Collect all matches + for pattern, term in self._patterns: + if pattern.search(normalized): + if term not in matched_terms: + matched_terms.append(term) + + if not matched_terms: + return FilterResult( + action=FilterAction.ALLOW, + filtered_text=text, + matched_terms=[], + ) + + matched_terms.sort() + + # Build filtered text by replacing matched terms + filtered = text + for term in matched_terms: + norm_term = _normalize(term) + # Find all occurrences in the normalised text and replace in original + # We need to map positions from normalised text back to original + filtered = self._replace_terms(filtered, term, norm_term) + + action = FilterAction.REJECT if self._policy.mode == "reject" else FilterAction.REDACT + + return FilterResult( + action=action, + filtered_text=filtered, + matched_terms=matched_terms, + ) + + def _replace_terms( + self, text: str, term: str, norm_term: str + ) -> str: + """Replace occurrences of *term* in *text* with redaction chars. + + Uses normalised text for matching but preserves original casing + in the output. + """ + norm_text = _normalize(text) + norm_term_len = len(norm_term) + result_chars = list(text) + + start = 0 + while start < len(norm_text): + idx = norm_text.find(norm_term, start) + if idx == -1: + break + # Replace characters in original text + for i in range(idx, min(idx + norm_term_len, len(result_chars))): + result_chars[i] = self._policy.redaction_char + start = idx + norm_term_len + + return "".join(result_chars) + + @property + def terms(self) -> list[str]: + """Read-only access to the loaded term list.""" + return list(self._terms) + + @property + def policy(self) -> _FilterPolicy: + """Read-only access to the filter policy.""" + return self._policy + + +# --------------------------------------------------------------------------- +# Factory +# --------------------------------------------------------------------------- + + +def get_text_filter( + config_path: str | Path | None = None, + terms: list[str] | None = None, + policy: _FilterPolicy | None = None, +) -> TextFilter: + """Return a ``TextFilter`` instance. + + By default this creates a ``ProfanityTextFilter``. To use a + third-party provider, subclass ``TextFilter`` and pass an + instance here instead. + """ + return ProfanityTextFilter( + config_path=config_path, + terms=terms, + policy=policy, + )