Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 coder-bot
|
0ace1d8400 |
@@ -8,7 +8,7 @@ from sqlalchemy.ext.asyncio import create_async_engine
|
||||
|
||||
from app.config import settings
|
||||
from app.database import Base
|
||||
from app.models import user, session, world, asset, conversation, message # noqa: F401
|
||||
from app.models import user, session, world, asset # noqa: F401
|
||||
|
||||
# this is the Alembic Config object, which provides
|
||||
# access to the values within the .ini file in use.
|
||||
|
||||
@@ -1,228 +0,0 @@
|
||||
"""chat data models
|
||||
|
||||
Revision ID: 1a1983be633a
|
||||
Revises: 0001_initial
|
||||
Create Date: 2026-08-18 15:06:05.948446
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "1a1983be633a"
|
||||
down_revision: Union[str, None] = "0001_initial"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Create chat tables: conversations, conversation_participants,
|
||||
messages, message_reports."""
|
||||
|
||||
# ── ENUM types ──────────────────────────────────────────────────
|
||||
|
||||
msg_report_reason = postgresql.ENUM(
|
||||
"spam", "inappropriate", "off_topic", "other",
|
||||
name="message_report_reason",
|
||||
)
|
||||
msg_report_reason.create(op.get_bind(), checkfirst=True)
|
||||
|
||||
msg_report_status = postgresql.ENUM(
|
||||
"pending", "resolved", "dismissed",
|
||||
name="message_report_status",
|
||||
)
|
||||
msg_report_status.create(op.get_bind(), checkfirst=True)
|
||||
|
||||
# ── conversations ───────────────────────────────────────────────
|
||||
|
||||
op.create_table(
|
||||
"conversations",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"latest_message_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("messages.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.func.now(),
|
||||
onupdate=sa.func.now(),
|
||||
),
|
||||
)
|
||||
op.create_index(op.f("ix_conversations_id"), "conversations", ["id"], unique=False)
|
||||
|
||||
# Unique pair constraint — populated later via participants
|
||||
op.create_unique_constraint(
|
||||
"uq_conversation_participants",
|
||||
"conversations",
|
||||
["participant_1_id", "participant_2_id"],
|
||||
)
|
||||
|
||||
# ── conversation_participants ───────────────────────────────────
|
||||
|
||||
op.create_table(
|
||||
"conversation_participants",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"conversation_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("conversations.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"user_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("left_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_read_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_conversation_participants_id"),
|
||||
"conversation_participants",
|
||||
["id"],
|
||||
unique=False,
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_conversation_participants_user_id"),
|
||||
"conversation_participants",
|
||||
["user_id"],
|
||||
unique=False,
|
||||
)
|
||||
op.create_unique_constraint(
|
||||
"uq_conversation_user",
|
||||
"conversation_participants",
|
||||
["conversation_id", "user_id"],
|
||||
)
|
||||
|
||||
# ── messages ────────────────────────────────────────────────────
|
||||
|
||||
op.create_table(
|
||||
"messages",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"sender_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"conversation_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("conversations.id", ondelete="CASCADE"),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column(
|
||||
"world_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("worlds.id", ondelete="CASCADE"),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column("room_id", sa.String(length=36), nullable=True),
|
||||
sa.Column("body", sa.Text(), nullable=False),
|
||||
sa.Column("is_filtered", sa.Boolean(), nullable=False, server_default="false"),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
)
|
||||
op.create_index(op.f("ix_messages_id"), "messages", ["id"], unique=False)
|
||||
op.create_index(
|
||||
"ix_messages_conversation_created",
|
||||
"messages",
|
||||
["conversation_id", "created_at"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_messages_world_created",
|
||||
"messages",
|
||||
["world_id", "created_at"],
|
||||
)
|
||||
|
||||
# ── message_reports ─────────────────────────────────────────────
|
||||
|
||||
op.create_table(
|
||||
"message_reports",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"message_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("messages.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"reporter_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("users.id", ondelete="SET NULL"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"reason",
|
||||
postgresql.ENUM(
|
||||
"spam", "inappropriate", "off_topic", "other",
|
||||
name="message_report_reason",
|
||||
create_type=False,
|
||||
),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"status",
|
||||
postgresql.ENUM(
|
||||
"pending", "resolved", "dismissed",
|
||||
name="message_report_status",
|
||||
create_type=False,
|
||||
),
|
||||
nullable=False,
|
||||
server_default="pending",
|
||||
),
|
||||
sa.Column(
|
||||
"moderator_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("users.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
)
|
||||
op.create_index(op.f("ix_message_reports_id"), "message_reports", ["id"], unique=False)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Drop chat tables in reverse dependency order."""
|
||||
op.drop_index(op.f("ix_message_reports_id"), table_name="message_reports")
|
||||
op.drop_table("message_reports")
|
||||
|
||||
op.drop_index(op.f("ix_messages_world_created"), table_name="messages")
|
||||
op.drop_index(op.f("ix_messages_conversation_created"), table_name="messages")
|
||||
op.drop_index(op.f("ix_messages_id"), table_name="messages")
|
||||
op.drop_table("messages")
|
||||
|
||||
op.drop_index(op.f("ix_conversation_participants_user_id"), table_name="conversation_participants")
|
||||
op.drop_index(op.f("ix_conversation_participants_id"), table_name="conversation_participants")
|
||||
op.drop_table("conversation_participants")
|
||||
|
||||
op.drop_index(op.f("ix_conversations_id"), table_name="conversations")
|
||||
op.drop_table("conversations")
|
||||
|
||||
# Drop ENUM types
|
||||
postgresql.ENUM(name="message_report_status").drop(op.get_bind(), checkfirst=True)
|
||||
postgresql.ENUM(name="message_report_reason").drop(op.get_bind(), checkfirst=True)
|
||||
@@ -2,16 +2,5 @@ from app.models.user import User
|
||||
from app.models.session import Session
|
||||
from app.models.world import World
|
||||
from app.models.asset import Asset
|
||||
from app.models.conversation import Conversation, ConversationParticipant
|
||||
from app.models.message import Message, MessageReport
|
||||
|
||||
__all__ = [
|
||||
"User",
|
||||
"Session",
|
||||
"World",
|
||||
"Asset",
|
||||
"Conversation",
|
||||
"ConversationParticipant",
|
||||
"Message",
|
||||
"MessageReport",
|
||||
]
|
||||
__all__ = ["User", "Session", "World", "Asset"]
|
||||
|
||||
@@ -1,94 +0,0 @@
|
||||
"""Conversation and participant models for DM chat."""
|
||||
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
Integer,
|
||||
String,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
UniqueConstraint,
|
||||
Index,
|
||||
)
|
||||
from sqlalchemy.orm import relationship
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class Conversation(Base):
|
||||
"""A DM conversation between two users.
|
||||
|
||||
Each conversation is uniquely identified by the pair of participant
|
||||
user IDs (regardless of order). A conversation is soft-deletable
|
||||
via ``left_at`` on the participant row.
|
||||
"""
|
||||
|
||||
__tablename__ = "conversations"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
# The most recent message's ID in this conversation (denormalised
|
||||
# for fast listing; updated by triggers or app logic).
|
||||
latest_message_id = Column(
|
||||
Integer, ForeignKey("messages.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
# Relationships
|
||||
latest_message = relationship(
|
||||
"Message",
|
||||
foreign_keys=[latest_message_id],
|
||||
lazy="select",
|
||||
)
|
||||
participants = relationship(
|
||||
"ConversationParticipant",
|
||||
back_populates="conversation",
|
||||
cascade="all, delete-orphan",
|
||||
order_by="ConversationParticipant.id",
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
# Ensure exactly one conversation per unique pair of users
|
||||
UniqueConstraint("participant_1_id", "participant_2_id", name="uq_conversation_participants"),
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<Conversation id={self.id}>"
|
||||
|
||||
|
||||
class ConversationParticipant(Base):
|
||||
"""Link table between conversations and users.
|
||||
|
||||
Stores per-participant metadata such as when they left the
|
||||
conversation (for soft-delete / archival).
|
||||
"""
|
||||
|
||||
__tablename__ = "conversation_participants"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
conversation_id = Column(
|
||||
Integer,
|
||||
ForeignKey("conversations.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
user_id = Column(
|
||||
Integer,
|
||||
ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
# When the participant left (NULL = still active).
|
||||
left_at = Column(DateTime(timezone=True), nullable=True)
|
||||
# Timestamp of the last message read by this participant.
|
||||
last_read_at = Column(DateTime(timezone=True), nullable=True)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
conversation = relationship("Conversation", back_populates="participants")
|
||||
user = relationship("User")
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("conversation_id", "user_id", name="uq_conversation_user"),
|
||||
Index("ix_conversation_participants_user_id", "user_id"),
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<ConversationParticipant conversation_id={self.conversation_id} user_id={self.user_id}>"
|
||||
@@ -1,96 +0,0 @@
|
||||
"""Message models — messages, message reports."""
|
||||
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Boolean,
|
||||
Index,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import ENUM
|
||||
from sqlalchemy.orm import relationship
|
||||
from sqlalchemy.sql import func
|
||||
import enum
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class MessageReportStatus(str, enum.Enum):
|
||||
PENDING = "pending"
|
||||
RESOLVED = "resolved"
|
||||
DISMISSED = "dismissed"
|
||||
|
||||
|
||||
class MessageReportReason(str, enum.Enum):
|
||||
SPAM = "spam"
|
||||
INAPPROPRIATE = "inappropriate"
|
||||
OFF_TOPIC = "off_topic"
|
||||
OTHER = "other"
|
||||
|
||||
|
||||
class Message(Base):
|
||||
__tablename__ = "messages"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
sender_id = Column(
|
||||
Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
conversation_id = Column(
|
||||
Integer, ForeignKey("conversations.id", ondelete="CASCADE"), nullable=True
|
||||
)
|
||||
world_id = Column(
|
||||
Integer, ForeignKey("worlds.id", ondelete="CASCADE"), nullable=True
|
||||
)
|
||||
room_id = Column(String(36), nullable=True)
|
||||
body = Column(Text, nullable=False)
|
||||
is_filtered = Column(Boolean, default=False, nullable=False)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
# Relationships
|
||||
sender = relationship("User", back_populates="sent_messages")
|
||||
conversation = relationship("Conversation", back_populates="messages")
|
||||
world = relationship("World")
|
||||
reports = relationship(
|
||||
"MessageReport", back_populates="message", cascade="all, delete-orphan"
|
||||
)
|
||||
|
||||
# Indexes for history queries
|
||||
__table_args__ = (
|
||||
Index("ix_messages_conversation_created", "conversation_id", "created_at"),
|
||||
Index("ix_messages_world_created", "world_id", "created_at"),
|
||||
)
|
||||
|
||||
|
||||
class MessageReport(Base):
|
||||
__tablename__ = "message_reports"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
message_id = Column(
|
||||
Integer, ForeignKey("messages.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
reporter_id = Column(
|
||||
Integer, ForeignKey("users.id", ondelete="SET NULL"), nullable=False
|
||||
)
|
||||
reason = Column(
|
||||
ENUM(MessageReportReason, name="message_report_reason", create_type=False),
|
||||
nullable=False,
|
||||
)
|
||||
details = Column(Text, nullable=True)
|
||||
status = Column(
|
||||
ENUM(MessageReportStatus, name="message_report_status", create_type=False),
|
||||
default=MessageReportStatus.PENDING,
|
||||
nullable=False,
|
||||
)
|
||||
moderator_id = Column(
|
||||
Integer, ForeignKey("users.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
resolved_at = Column(DateTime(timezone=True), nullable=True)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
# Relationships
|
||||
message = relationship("Message", back_populates="reports")
|
||||
reporter = relationship("User", foreign_keys=[reporter_id])
|
||||
moderator = relationship("User", foreign_keys=[moderator_id])
|
||||
@@ -19,14 +19,3 @@ class User(Base):
|
||||
sessions = relationship("Session", back_populates="user", cascade="all, delete-orphan")
|
||||
worlds = relationship("World", back_populates="owner")
|
||||
assets = relationship("Asset", back_populates="owner")
|
||||
sent_messages = relationship("Message", back_populates="sender")
|
||||
reported_messages = relationship(
|
||||
"MessageReport",
|
||||
foreign_keys="MessageReport.reporter_id",
|
||||
back_populates="reporter",
|
||||
)
|
||||
moderated_reports = relationship(
|
||||
"MessageReport",
|
||||
foreign_keys="MessageReport.moderator_id",
|
||||
back_populates="moderator",
|
||||
)
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Application services."""
|
||||
@@ -0,0 +1,340 @@
|
||||
"""Pluggable server-side text filtering service.
|
||||
|
||||
Provides a ``TextFilter`` protocol with a deterministic default
|
||||
implementation (profanity/term list with normalization) and a hook
|
||||
for third-party providers.
|
||||
|
||||
Usage::
|
||||
|
||||
from app.services.text_filter import TextFilter, FilterResult, \
|
||||
ProfanityTextFilter, get_text_filter
|
||||
|
||||
# Default factory reads config from the path below
|
||||
filt = get_text_filter()
|
||||
result = filt.check("Hello wor1d!")
|
||||
# result.action == FilterAction.ALLOW
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import unicodedata
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Enums / Data classes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class FilterAction(str, Enum):
|
||||
"""Action the filter wants the caller to take."""
|
||||
|
||||
ALLOW = "allow"
|
||||
REDACT = "redact"
|
||||
REJECT = "reject"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FilterResult:
|
||||
"""Immutable result of a single ``TextFilter.check()`` call.
|
||||
|
||||
Attributes:
|
||||
action: What to do with the message.
|
||||
filtered_text: The text after redaction (may equal *text* if
|
||||
action is ``ALLOW``).
|
||||
matched_terms: Sorted list of terms that triggered the filter.
|
||||
"""
|
||||
|
||||
action: FilterAction
|
||||
filtered_text: str
|
||||
matched_terms: list[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def is_allowed(self) -> bool:
|
||||
return self.action == FilterAction.ALLOW
|
||||
|
||||
@property
|
||||
def is_rejected(self) -> bool:
|
||||
return self.action == FilterAction.REJECT
|
||||
|
||||
@property
|
||||
def is_redacted(self) -> bool:
|
||||
return self.action == FilterAction.REDACT
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Protocol / abstract base
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TextFilter(ABC):
|
||||
"""Protocol for server-side text filters.
|
||||
|
||||
Every concrete filter must implement ``check()``. The default
|
||||
implementation (``ProfanityTextFilter``) is deterministic and
|
||||
requires no external service. A third-party provider can be
|
||||
plugged in by subclassing ``TextFilter`` and overriding ``check``.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def check(self, text: str) -> FilterResult:
|
||||
"""Analyse *text* and return a ``FilterResult``.
|
||||
|
||||
Args:
|
||||
text: Raw message text to filter.
|
||||
|
||||
Returns:
|
||||
A ``FilterResult`` with the decision and any redacted text.
|
||||
"""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Normalisation helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Common leetspeak substitutions (char -> set of possible originals)
|
||||
_LEETSPEAK: dict[str, str] = {
|
||||
"0": "o",
|
||||
"1": "i",
|
||||
"3": "e",
|
||||
"4": "a",
|
||||
"5": "s",
|
||||
"7": "t",
|
||||
"8": "b",
|
||||
"9": "g",
|
||||
"@": "a",
|
||||
}
|
||||
|
||||
# Normalise separators that are often used to obfuscate words
|
||||
_SEPARATORS = re.compile(r"[\s_\-\.]+")
|
||||
|
||||
|
||||
def _normalize(text: str) -> str:
|
||||
"""Case-fold, strip diacritics, and replace leetspeak / separators.
|
||||
|
||||
This produces a canonical form for matching against the term list.
|
||||
"""
|
||||
# Case-fold
|
||||
text = text.casefold()
|
||||
# Strip diacritics (e.g. é -> e)
|
||||
text = unicodedata.normalize("NFKD", text)
|
||||
text = "".join(c for c in text if not unicodedata.combining(c))
|
||||
# Replace leetspeak
|
||||
normalized = ""
|
||||
for ch in text:
|
||||
normalized += _LEETSPEAK.get(ch, ch)
|
||||
# Collapse separators
|
||||
normalized = _SEPARATORS.sub("", normalized)
|
||||
return normalized
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Default implementation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Default config path (relative to project root)
|
||||
_DEFAULT_CONFIG_PATH = Path(__file__).resolve().parent.parent.parent / "data" / "text_filter_config.json"
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FilterPolicy:
|
||||
"""Filtering policy loaded from config."""
|
||||
|
||||
mode: str # "redact" or "reject"
|
||||
redaction_char: str = "*"
|
||||
# Minimum term length to consider (avoids matching single chars)
|
||||
min_term_length: int = 3
|
||||
# If True, match terms case-insensitively after normalisation
|
||||
case_insensitive: bool = True
|
||||
# If True, match whole words only (after normalisation)
|
||||
whole_word: bool = True
|
||||
|
||||
|
||||
class ProfanityTextFilter(TextFilter):
|
||||
"""Deterministic text filter backed by a JSON term list.
|
||||
|
||||
The config file is a JSON object with:
|
||||
|
||||
.. code-block:: json
|
||||
|
||||
{
|
||||
"mode": "redact",
|
||||
"redaction_char": "*",
|
||||
"min_term_length": 3,
|
||||
"case_insensitive": true,
|
||||
"whole_word": true,
|
||||
"terms": ["badword", "another", ...]
|
||||
}
|
||||
|
||||
Terms are matched after normalisation (case-fold, leetspeak
|
||||
substitution, separator removal, diacritic stripping).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config_path: str | Path | None = None,
|
||||
terms: list[str] | None = None,
|
||||
policy: _FilterPolicy | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
config_path: Path to a JSON config file. Ignored if *terms*
|
||||
is provided.
|
||||
terms: Inline list of terms (overrides config file).
|
||||
policy: Filter policy. Defaults to a redact policy.
|
||||
"""
|
||||
if terms is not None:
|
||||
self._terms: list[str] = [t.lower() for t in terms]
|
||||
elif config_path is not None:
|
||||
self._terms = self._load_terms(Path(config_path))
|
||||
else:
|
||||
# Try default path
|
||||
if _DEFAULT_CONFIG_PATH.exists():
|
||||
self._terms = self._load_terms(_DEFAULT_CONFIG_PATH)
|
||||
else:
|
||||
self._terms = []
|
||||
|
||||
self._policy = policy or _FilterPolicy(mode="redact")
|
||||
# Pre-compile patterns for each term
|
||||
self._patterns: list[tuple[re.Pattern[str], str]] = []
|
||||
for term in self._terms:
|
||||
if len(term) < self._policy.min_term_length:
|
||||
continue
|
||||
if self._policy.whole_word:
|
||||
pattern = re.compile(
|
||||
r"(?<!\w)" + re.escape(term) + r"(?!\w)",
|
||||
re.IGNORECASE if self._policy.case_insensitive else 0,
|
||||
)
|
||||
else:
|
||||
pattern = re.compile(
|
||||
re.escape(term),
|
||||
re.IGNORECASE if self._policy.case_insensitive else 0,
|
||||
)
|
||||
self._patterns.append((pattern, term))
|
||||
|
||||
# -- config loading --------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _load_terms(config_path: Path) -> list[str]:
|
||||
"""Load terms from a JSON config file."""
|
||||
with open(config_path, "r", encoding="utf-8") as f:
|
||||
data: dict[str, Any] = json.load(f)
|
||||
|
||||
terms: list[str] = []
|
||||
for raw in data.get("terms", []):
|
||||
term = str(raw).strip()
|
||||
if term:
|
||||
terms.append(term.casefold())
|
||||
return terms
|
||||
|
||||
# -- public API ------------------------------------------------------
|
||||
|
||||
def check(self, text: str) -> FilterResult:
|
||||
"""Check *text* against the loaded term list.
|
||||
|
||||
Returns a ``FilterResult`` with the appropriate action,
|
||||
filtered text, and matched terms.
|
||||
"""
|
||||
if not text:
|
||||
return FilterResult(
|
||||
action=FilterAction.ALLOW,
|
||||
filtered_text=text,
|
||||
matched_terms=[],
|
||||
)
|
||||
|
||||
normalized = _normalize(text)
|
||||
matched_terms: list[str] = []
|
||||
|
||||
# Collect all matches
|
||||
for pattern, term in self._patterns:
|
||||
if pattern.search(normalized):
|
||||
if term not in matched_terms:
|
||||
matched_terms.append(term)
|
||||
|
||||
if not matched_terms:
|
||||
return FilterResult(
|
||||
action=FilterAction.ALLOW,
|
||||
filtered_text=text,
|
||||
matched_terms=[],
|
||||
)
|
||||
|
||||
matched_terms.sort()
|
||||
|
||||
# Build filtered text by replacing matched terms
|
||||
filtered = text
|
||||
for term in matched_terms:
|
||||
norm_term = _normalize(term)
|
||||
# Find all occurrences in the normalised text and replace in original
|
||||
# We need to map positions from normalised text back to original
|
||||
filtered = self._replace_terms(filtered, term, norm_term)
|
||||
|
||||
action = FilterAction.REJECT if self._policy.mode == "reject" else FilterAction.REDACT
|
||||
|
||||
return FilterResult(
|
||||
action=action,
|
||||
filtered_text=filtered,
|
||||
matched_terms=matched_terms,
|
||||
)
|
||||
|
||||
def _replace_terms(
|
||||
self, text: str, term: str, norm_term: str
|
||||
) -> str:
|
||||
"""Replace occurrences of *term* in *text* with redaction chars.
|
||||
|
||||
Uses normalised text for matching but preserves original casing
|
||||
in the output.
|
||||
"""
|
||||
norm_text = _normalize(text)
|
||||
norm_term_len = len(norm_term)
|
||||
result_chars = list(text)
|
||||
|
||||
start = 0
|
||||
while start < len(norm_text):
|
||||
idx = norm_text.find(norm_term, start)
|
||||
if idx == -1:
|
||||
break
|
||||
# Replace characters in original text
|
||||
for i in range(idx, min(idx + norm_term_len, len(result_chars))):
|
||||
result_chars[i] = self._policy.redaction_char
|
||||
start = idx + norm_term_len
|
||||
|
||||
return "".join(result_chars)
|
||||
|
||||
@property
|
||||
def terms(self) -> list[str]:
|
||||
"""Read-only access to the loaded term list."""
|
||||
return list(self._terms)
|
||||
|
||||
@property
|
||||
def policy(self) -> _FilterPolicy:
|
||||
"""Read-only access to the filter policy."""
|
||||
return self._policy
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Factory
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_text_filter(
|
||||
config_path: str | Path | None = None,
|
||||
terms: list[str] | None = None,
|
||||
policy: _FilterPolicy | None = None,
|
||||
) -> TextFilter:
|
||||
"""Return a ``TextFilter`` instance.
|
||||
|
||||
By default this creates a ``ProfanityTextFilter``. To use a
|
||||
third-party provider, subclass ``TextFilter`` and pass an
|
||||
instance here instead.
|
||||
"""
|
||||
return ProfanityTextFilter(
|
||||
config_path=config_path,
|
||||
terms=terms,
|
||||
policy=policy,
|
||||
)
|
||||
Reference in New Issue
Block a user