Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| efc39eb50a | |||
| c792629f2e | |||
| cff7fd96d2 |
@@ -8,7 +8,7 @@ from sqlalchemy.ext.asyncio import create_async_engine
|
|||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.database import Base
|
from app.database import Base
|
||||||
from app.models import user, session, world, asset # noqa: F401
|
from app.models import user, session, world, asset, conversation, message # noqa: F401
|
||||||
|
|
||||||
# this is the Alembic Config object, which provides
|
# this is the Alembic Config object, which provides
|
||||||
# access to the values within the .ini file in use.
|
# access to the values within the .ini file in use.
|
||||||
|
|||||||
@@ -0,0 +1,228 @@
|
|||||||
|
"""chat data models
|
||||||
|
|
||||||
|
Revision ID: 1a1983be633a
|
||||||
|
Revises: 0001_initial
|
||||||
|
Create Date: 2026-08-18 15:06:05.948446
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision: str = "1a1983be633a"
|
||||||
|
down_revision: Union[str, None] = "0001_initial"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""Create chat tables: conversations, conversation_participants,
|
||||||
|
messages, message_reports."""
|
||||||
|
|
||||||
|
# ── ENUM types ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
msg_report_reason = postgresql.ENUM(
|
||||||
|
"spam", "inappropriate", "off_topic", "other",
|
||||||
|
name="message_report_reason",
|
||||||
|
)
|
||||||
|
msg_report_reason.create(op.get_bind(), checkfirst=True)
|
||||||
|
|
||||||
|
msg_report_status = postgresql.ENUM(
|
||||||
|
"pending", "resolved", "dismissed",
|
||||||
|
name="message_report_status",
|
||||||
|
)
|
||||||
|
msg_report_status.create(op.get_bind(), checkfirst=True)
|
||||||
|
|
||||||
|
# ── conversations ───────────────────────────────────────────────
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"conversations",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column(
|
||||||
|
"latest_message_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("messages.id", ondelete="SET NULL"),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
onupdate=sa.func.now(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(op.f("ix_conversations_id"), "conversations", ["id"], unique=False)
|
||||||
|
|
||||||
|
# Unique pair constraint — populated later via participants
|
||||||
|
op.create_unique_constraint(
|
||||||
|
"uq_conversation_participants",
|
||||||
|
"conversations",
|
||||||
|
["participant_1_id", "participant_2_id"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# ── conversation_participants ───────────────────────────────────
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"conversation_participants",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column(
|
||||||
|
"conversation_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("conversations.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"user_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("left_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("last_read_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_conversation_participants_id"),
|
||||||
|
"conversation_participants",
|
||||||
|
["id"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_conversation_participants_user_id"),
|
||||||
|
"conversation_participants",
|
||||||
|
["user_id"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_unique_constraint(
|
||||||
|
"uq_conversation_user",
|
||||||
|
"conversation_participants",
|
||||||
|
["conversation_id", "user_id"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# ── messages ────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"messages",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column(
|
||||||
|
"sender_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"conversation_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("conversations.id", ondelete="CASCADE"),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"world_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("worlds.id", ondelete="CASCADE"),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
sa.Column("room_id", sa.String(length=36), nullable=True),
|
||||||
|
sa.Column("body", sa.Text(), nullable=False),
|
||||||
|
sa.Column("is_filtered", sa.Boolean(), nullable=False, server_default="false"),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(op.f("ix_messages_id"), "messages", ["id"], unique=False)
|
||||||
|
op.create_index(
|
||||||
|
"ix_messages_conversation_created",
|
||||||
|
"messages",
|
||||||
|
["conversation_id", "created_at"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_messages_world_created",
|
||||||
|
"messages",
|
||||||
|
["world_id", "created_at"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# ── message_reports ─────────────────────────────────────────────
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"message_reports",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column(
|
||||||
|
"message_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("messages.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"reporter_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("users.id", ondelete="SET NULL"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"reason",
|
||||||
|
postgresql.ENUM(
|
||||||
|
"spam", "inappropriate", "off_topic", "other",
|
||||||
|
name="message_report_reason",
|
||||||
|
create_type=False,
|
||||||
|
),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"status",
|
||||||
|
postgresql.ENUM(
|
||||||
|
"pending", "resolved", "dismissed",
|
||||||
|
name="message_report_status",
|
||||||
|
create_type=False,
|
||||||
|
),
|
||||||
|
nullable=False,
|
||||||
|
server_default="pending",
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"moderator_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("users.id", ondelete="SET NULL"),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(op.f("ix_message_reports_id"), "message_reports", ["id"], unique=False)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""Drop chat tables in reverse dependency order."""
|
||||||
|
op.drop_index(op.f("ix_message_reports_id"), table_name="message_reports")
|
||||||
|
op.drop_table("message_reports")
|
||||||
|
|
||||||
|
op.drop_index(op.f("ix_messages_world_created"), table_name="messages")
|
||||||
|
op.drop_index(op.f("ix_messages_conversation_created"), table_name="messages")
|
||||||
|
op.drop_index(op.f("ix_messages_id"), table_name="messages")
|
||||||
|
op.drop_table("messages")
|
||||||
|
|
||||||
|
op.drop_index(op.f("ix_conversation_participants_user_id"), table_name="conversation_participants")
|
||||||
|
op.drop_index(op.f("ix_conversation_participants_id"), table_name="conversation_participants")
|
||||||
|
op.drop_table("conversation_participants")
|
||||||
|
|
||||||
|
op.drop_index(op.f("ix_conversations_id"), table_name="conversations")
|
||||||
|
op.drop_table("conversations")
|
||||||
|
|
||||||
|
# Drop ENUM types
|
||||||
|
postgresql.ENUM(name="message_report_status").drop(op.get_bind(), checkfirst=True)
|
||||||
|
postgresql.ENUM(name="message_report_reason").drop(op.get_bind(), checkfirst=True)
|
||||||
@@ -2,5 +2,16 @@ from app.models.user import User
|
|||||||
from app.models.session import Session
|
from app.models.session import Session
|
||||||
from app.models.world import World
|
from app.models.world import World
|
||||||
from app.models.asset import Asset
|
from app.models.asset import Asset
|
||||||
|
from app.models.conversation import Conversation, ConversationParticipant
|
||||||
|
from app.models.message import Message, MessageReport
|
||||||
|
|
||||||
__all__ = ["User", "Session", "World", "Asset"]
|
__all__ = [
|
||||||
|
"User",
|
||||||
|
"Session",
|
||||||
|
"World",
|
||||||
|
"Asset",
|
||||||
|
"Conversation",
|
||||||
|
"ConversationParticipant",
|
||||||
|
"Message",
|
||||||
|
"MessageReport",
|
||||||
|
]
|
||||||
|
|||||||
@@ -0,0 +1,94 @@
|
|||||||
|
"""Conversation and participant models for DM chat."""
|
||||||
|
|
||||||
|
from sqlalchemy import (
|
||||||
|
Column,
|
||||||
|
Integer,
|
||||||
|
String,
|
||||||
|
DateTime,
|
||||||
|
ForeignKey,
|
||||||
|
UniqueConstraint,
|
||||||
|
Index,
|
||||||
|
)
|
||||||
|
from sqlalchemy.orm import relationship
|
||||||
|
from sqlalchemy.sql import func
|
||||||
|
|
||||||
|
from app.database import Base
|
||||||
|
|
||||||
|
|
||||||
|
class Conversation(Base):
|
||||||
|
"""A DM conversation between two users.
|
||||||
|
|
||||||
|
Each conversation is uniquely identified by the pair of participant
|
||||||
|
user IDs (regardless of order). A conversation is soft-deletable
|
||||||
|
via ``left_at`` on the participant row.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__tablename__ = "conversations"
|
||||||
|
|
||||||
|
id = Column(Integer, primary_key=True, index=True)
|
||||||
|
# The most recent message's ID in this conversation (denormalised
|
||||||
|
# for fast listing; updated by triggers or app logic).
|
||||||
|
latest_message_id = Column(
|
||||||
|
Integer, ForeignKey("messages.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||||
|
|
||||||
|
# Relationships
|
||||||
|
latest_message = relationship(
|
||||||
|
"Message",
|
||||||
|
foreign_keys=[latest_message_id],
|
||||||
|
lazy="select",
|
||||||
|
)
|
||||||
|
participants = relationship(
|
||||||
|
"ConversationParticipant",
|
||||||
|
back_populates="conversation",
|
||||||
|
cascade="all, delete-orphan",
|
||||||
|
order_by="ConversationParticipant.id",
|
||||||
|
)
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
# Ensure exactly one conversation per unique pair of users
|
||||||
|
UniqueConstraint("participant_1_id", "participant_2_id", name="uq_conversation_participants"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def __repr__(self) -> str:
|
||||||
|
return f"<Conversation id={self.id}>"
|
||||||
|
|
||||||
|
|
||||||
|
class ConversationParticipant(Base):
|
||||||
|
"""Link table between conversations and users.
|
||||||
|
|
||||||
|
Stores per-participant metadata such as when they left the
|
||||||
|
conversation (for soft-delete / archival).
|
||||||
|
"""
|
||||||
|
|
||||||
|
__tablename__ = "conversation_participants"
|
||||||
|
|
||||||
|
id = Column(Integer, primary_key=True, index=True)
|
||||||
|
conversation_id = Column(
|
||||||
|
Integer,
|
||||||
|
ForeignKey("conversations.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
user_id = Column(
|
||||||
|
Integer,
|
||||||
|
ForeignKey("users.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
# When the participant left (NULL = still active).
|
||||||
|
left_at = Column(DateTime(timezone=True), nullable=True)
|
||||||
|
# Timestamp of the last message read by this participant.
|
||||||
|
last_read_at = Column(DateTime(timezone=True), nullable=True)
|
||||||
|
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
|
||||||
|
conversation = relationship("Conversation", back_populates="participants")
|
||||||
|
user = relationship("User")
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("conversation_id", "user_id", name="uq_conversation_user"),
|
||||||
|
Index("ix_conversation_participants_user_id", "user_id"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def __repr__(self) -> str:
|
||||||
|
return f"<ConversationParticipant conversation_id={self.conversation_id} user_id={self.user_id}>"
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
"""Message models — messages, message reports."""
|
||||||
|
|
||||||
|
from sqlalchemy import (
|
||||||
|
Column,
|
||||||
|
Integer,
|
||||||
|
String,
|
||||||
|
Text,
|
||||||
|
DateTime,
|
||||||
|
ForeignKey,
|
||||||
|
Boolean,
|
||||||
|
Index,
|
||||||
|
)
|
||||||
|
from sqlalchemy.dialects.postgresql import ENUM
|
||||||
|
from sqlalchemy.orm import relationship
|
||||||
|
from sqlalchemy.sql import func
|
||||||
|
import enum
|
||||||
|
|
||||||
|
from app.database import Base
|
||||||
|
|
||||||
|
|
||||||
|
class MessageReportStatus(str, enum.Enum):
|
||||||
|
PENDING = "pending"
|
||||||
|
RESOLVED = "resolved"
|
||||||
|
DISMISSED = "dismissed"
|
||||||
|
|
||||||
|
|
||||||
|
class MessageReportReason(str, enum.Enum):
|
||||||
|
SPAM = "spam"
|
||||||
|
INAPPROPRIATE = "inappropriate"
|
||||||
|
OFF_TOPIC = "off_topic"
|
||||||
|
OTHER = "other"
|
||||||
|
|
||||||
|
|
||||||
|
class Message(Base):
|
||||||
|
__tablename__ = "messages"
|
||||||
|
|
||||||
|
id = Column(Integer, primary_key=True, index=True)
|
||||||
|
sender_id = Column(
|
||||||
|
Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False
|
||||||
|
)
|
||||||
|
conversation_id = Column(
|
||||||
|
Integer, ForeignKey("conversations.id", ondelete="CASCADE"), nullable=True
|
||||||
|
)
|
||||||
|
world_id = Column(
|
||||||
|
Integer, ForeignKey("worlds.id", ondelete="CASCADE"), nullable=True
|
||||||
|
)
|
||||||
|
room_id = Column(String(36), nullable=True)
|
||||||
|
body = Column(Text, nullable=False)
|
||||||
|
is_filtered = Column(Boolean, default=False, nullable=False)
|
||||||
|
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
|
||||||
|
# Relationships
|
||||||
|
sender = relationship("User", back_populates="sent_messages")
|
||||||
|
conversation = relationship("Conversation", back_populates="messages")
|
||||||
|
world = relationship("World")
|
||||||
|
reports = relationship(
|
||||||
|
"MessageReport", back_populates="message", cascade="all, delete-orphan"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Indexes for history queries
|
||||||
|
__table_args__ = (
|
||||||
|
Index("ix_messages_conversation_created", "conversation_id", "created_at"),
|
||||||
|
Index("ix_messages_world_created", "world_id", "created_at"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class MessageReport(Base):
|
||||||
|
__tablename__ = "message_reports"
|
||||||
|
|
||||||
|
id = Column(Integer, primary_key=True, index=True)
|
||||||
|
message_id = Column(
|
||||||
|
Integer, ForeignKey("messages.id", ondelete="CASCADE"), nullable=False
|
||||||
|
)
|
||||||
|
reporter_id = Column(
|
||||||
|
Integer, ForeignKey("users.id", ondelete="SET NULL"), nullable=False
|
||||||
|
)
|
||||||
|
reason = Column(
|
||||||
|
ENUM(MessageReportReason, name="message_report_reason", create_type=False),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
details = Column(Text, nullable=True)
|
||||||
|
status = Column(
|
||||||
|
ENUM(MessageReportStatus, name="message_report_status", create_type=False),
|
||||||
|
default=MessageReportStatus.PENDING,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
moderator_id = Column(
|
||||||
|
Integer, ForeignKey("users.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
resolved_at = Column(DateTime(timezone=True), nullable=True)
|
||||||
|
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
|
||||||
|
# Relationships
|
||||||
|
message = relationship("Message", back_populates="reports")
|
||||||
|
reporter = relationship("User", foreign_keys=[reporter_id])
|
||||||
|
moderator = relationship("User", foreign_keys=[moderator_id])
|
||||||
@@ -19,3 +19,14 @@ class User(Base):
|
|||||||
sessions = relationship("Session", back_populates="user", cascade="all, delete-orphan")
|
sessions = relationship("Session", back_populates="user", cascade="all, delete-orphan")
|
||||||
worlds = relationship("World", back_populates="owner")
|
worlds = relationship("World", back_populates="owner")
|
||||||
assets = relationship("Asset", back_populates="owner")
|
assets = relationship("Asset", back_populates="owner")
|
||||||
|
sent_messages = relationship("Message", back_populates="sender")
|
||||||
|
reported_messages = relationship(
|
||||||
|
"MessageReport",
|
||||||
|
foreign_keys="MessageReport.reporter_id",
|
||||||
|
back_populates="reporter",
|
||||||
|
)
|
||||||
|
moderated_reports = relationship(
|
||||||
|
"MessageReport",
|
||||||
|
foreign_keys="MessageReport.moderator_id",
|
||||||
|
back_populates="moderator",
|
||||||
|
)
|
||||||
|
|||||||
@@ -1 +0,0 @@
|
|||||||
"""Application services."""
|
|
||||||
@@ -1,340 +0,0 @@
|
|||||||
"""Pluggable server-side text filtering service.
|
|
||||||
|
|
||||||
Provides a ``TextFilter`` protocol with a deterministic default
|
|
||||||
implementation (profanity/term list with normalization) and a hook
|
|
||||||
for third-party providers.
|
|
||||||
|
|
||||||
Usage::
|
|
||||||
|
|
||||||
from app.services.text_filter import TextFilter, FilterResult, \
|
|
||||||
ProfanityTextFilter, get_text_filter
|
|
||||||
|
|
||||||
# Default factory reads config from the path below
|
|
||||||
filt = get_text_filter()
|
|
||||||
result = filt.check("Hello wor1d!")
|
|
||||||
# result.action == FilterAction.ALLOW
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import unicodedata
|
|
||||||
from abc import ABC, abstractmethod
|
|
||||||
from dataclasses import dataclass, field
|
|
||||||
from enum import Enum
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Enums / Data classes
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
|
|
||||||
class FilterAction(str, Enum):
|
|
||||||
"""Action the filter wants the caller to take."""
|
|
||||||
|
|
||||||
ALLOW = "allow"
|
|
||||||
REDACT = "redact"
|
|
||||||
REJECT = "reject"
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class FilterResult:
|
|
||||||
"""Immutable result of a single ``TextFilter.check()`` call.
|
|
||||||
|
|
||||||
Attributes:
|
|
||||||
action: What to do with the message.
|
|
||||||
filtered_text: The text after redaction (may equal *text* if
|
|
||||||
action is ``ALLOW``).
|
|
||||||
matched_terms: Sorted list of terms that triggered the filter.
|
|
||||||
"""
|
|
||||||
|
|
||||||
action: FilterAction
|
|
||||||
filtered_text: str
|
|
||||||
matched_terms: list[str] = field(default_factory=list)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def is_allowed(self) -> bool:
|
|
||||||
return self.action == FilterAction.ALLOW
|
|
||||||
|
|
||||||
@property
|
|
||||||
def is_rejected(self) -> bool:
|
|
||||||
return self.action == FilterAction.REJECT
|
|
||||||
|
|
||||||
@property
|
|
||||||
def is_redacted(self) -> bool:
|
|
||||||
return self.action == FilterAction.REDACT
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Protocol / abstract base
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
|
|
||||||
class TextFilter(ABC):
|
|
||||||
"""Protocol for server-side text filters.
|
|
||||||
|
|
||||||
Every concrete filter must implement ``check()``. The default
|
|
||||||
implementation (``ProfanityTextFilter``) is deterministic and
|
|
||||||
requires no external service. A third-party provider can be
|
|
||||||
plugged in by subclassing ``TextFilter`` and overriding ``check``.
|
|
||||||
"""
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def check(self, text: str) -> FilterResult:
|
|
||||||
"""Analyse *text* and return a ``FilterResult``.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
text: Raw message text to filter.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
A ``FilterResult`` with the decision and any redacted text.
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Normalisation helpers
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
# Common leetspeak substitutions (char -> set of possible originals)
|
|
||||||
_LEETSPEAK: dict[str, str] = {
|
|
||||||
"0": "o",
|
|
||||||
"1": "i",
|
|
||||||
"3": "e",
|
|
||||||
"4": "a",
|
|
||||||
"5": "s",
|
|
||||||
"7": "t",
|
|
||||||
"8": "b",
|
|
||||||
"9": "g",
|
|
||||||
"@": "a",
|
|
||||||
}
|
|
||||||
|
|
||||||
# Normalise separators that are often used to obfuscate words
|
|
||||||
_SEPARATORS = re.compile(r"[\s_\-\.]+")
|
|
||||||
|
|
||||||
|
|
||||||
def _normalize(text: str) -> str:
|
|
||||||
"""Case-fold, strip diacritics, and replace leetspeak / separators.
|
|
||||||
|
|
||||||
This produces a canonical form for matching against the term list.
|
|
||||||
"""
|
|
||||||
# Case-fold
|
|
||||||
text = text.casefold()
|
|
||||||
# Strip diacritics (e.g. é -> e)
|
|
||||||
text = unicodedata.normalize("NFKD", text)
|
|
||||||
text = "".join(c for c in text if not unicodedata.combining(c))
|
|
||||||
# Replace leetspeak
|
|
||||||
normalized = ""
|
|
||||||
for ch in text:
|
|
||||||
normalized += _LEETSPEAK.get(ch, ch)
|
|
||||||
# Collapse separators
|
|
||||||
normalized = _SEPARATORS.sub("", normalized)
|
|
||||||
return normalized
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Default implementation
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
# Default config path (relative to project root)
|
|
||||||
_DEFAULT_CONFIG_PATH = Path(__file__).resolve().parent.parent.parent / "data" / "text_filter_config.json"
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class _FilterPolicy:
|
|
||||||
"""Filtering policy loaded from config."""
|
|
||||||
|
|
||||||
mode: str # "redact" or "reject"
|
|
||||||
redaction_char: str = "*"
|
|
||||||
# Minimum term length to consider (avoids matching single chars)
|
|
||||||
min_term_length: int = 3
|
|
||||||
# If True, match terms case-insensitively after normalisation
|
|
||||||
case_insensitive: bool = True
|
|
||||||
# If True, match whole words only (after normalisation)
|
|
||||||
whole_word: bool = True
|
|
||||||
|
|
||||||
|
|
||||||
class ProfanityTextFilter(TextFilter):
|
|
||||||
"""Deterministic text filter backed by a JSON term list.
|
|
||||||
|
|
||||||
The config file is a JSON object with:
|
|
||||||
|
|
||||||
.. code-block:: json
|
|
||||||
|
|
||||||
{
|
|
||||||
"mode": "redact",
|
|
||||||
"redaction_char": "*",
|
|
||||||
"min_term_length": 3,
|
|
||||||
"case_insensitive": true,
|
|
||||||
"whole_word": true,
|
|
||||||
"terms": ["badword", "another", ...]
|
|
||||||
}
|
|
||||||
|
|
||||||
Terms are matched after normalisation (case-fold, leetspeak
|
|
||||||
substitution, separator removal, diacritic stripping).
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
config_path: str | Path | None = None,
|
|
||||||
terms: list[str] | None = None,
|
|
||||||
policy: _FilterPolicy | None = None,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Args:
|
|
||||||
config_path: Path to a JSON config file. Ignored if *terms*
|
|
||||||
is provided.
|
|
||||||
terms: Inline list of terms (overrides config file).
|
|
||||||
policy: Filter policy. Defaults to a redact policy.
|
|
||||||
"""
|
|
||||||
if terms is not None:
|
|
||||||
self._terms: list[str] = [t.lower() for t in terms]
|
|
||||||
elif config_path is not None:
|
|
||||||
self._terms = self._load_terms(Path(config_path))
|
|
||||||
else:
|
|
||||||
# Try default path
|
|
||||||
if _DEFAULT_CONFIG_PATH.exists():
|
|
||||||
self._terms = self._load_terms(_DEFAULT_CONFIG_PATH)
|
|
||||||
else:
|
|
||||||
self._terms = []
|
|
||||||
|
|
||||||
self._policy = policy or _FilterPolicy(mode="redact")
|
|
||||||
# Pre-compile patterns for each term
|
|
||||||
self._patterns: list[tuple[re.Pattern[str], str]] = []
|
|
||||||
for term in self._terms:
|
|
||||||
if len(term) < self._policy.min_term_length:
|
|
||||||
continue
|
|
||||||
if self._policy.whole_word:
|
|
||||||
pattern = re.compile(
|
|
||||||
r"(?<!\w)" + re.escape(term) + r"(?!\w)",
|
|
||||||
re.IGNORECASE if self._policy.case_insensitive else 0,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
pattern = re.compile(
|
|
||||||
re.escape(term),
|
|
||||||
re.IGNORECASE if self._policy.case_insensitive else 0,
|
|
||||||
)
|
|
||||||
self._patterns.append((pattern, term))
|
|
||||||
|
|
||||||
# -- config loading --------------------------------------------------
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _load_terms(config_path: Path) -> list[str]:
|
|
||||||
"""Load terms from a JSON config file."""
|
|
||||||
with open(config_path, "r", encoding="utf-8") as f:
|
|
||||||
data: dict[str, Any] = json.load(f)
|
|
||||||
|
|
||||||
terms: list[str] = []
|
|
||||||
for raw in data.get("terms", []):
|
|
||||||
term = str(raw).strip()
|
|
||||||
if term:
|
|
||||||
terms.append(term.casefold())
|
|
||||||
return terms
|
|
||||||
|
|
||||||
# -- public API ------------------------------------------------------
|
|
||||||
|
|
||||||
def check(self, text: str) -> FilterResult:
|
|
||||||
"""Check *text* against the loaded term list.
|
|
||||||
|
|
||||||
Returns a ``FilterResult`` with the appropriate action,
|
|
||||||
filtered text, and matched terms.
|
|
||||||
"""
|
|
||||||
if not text:
|
|
||||||
return FilterResult(
|
|
||||||
action=FilterAction.ALLOW,
|
|
||||||
filtered_text=text,
|
|
||||||
matched_terms=[],
|
|
||||||
)
|
|
||||||
|
|
||||||
normalized = _normalize(text)
|
|
||||||
matched_terms: list[str] = []
|
|
||||||
|
|
||||||
# Collect all matches
|
|
||||||
for pattern, term in self._patterns:
|
|
||||||
if pattern.search(normalized):
|
|
||||||
if term not in matched_terms:
|
|
||||||
matched_terms.append(term)
|
|
||||||
|
|
||||||
if not matched_terms:
|
|
||||||
return FilterResult(
|
|
||||||
action=FilterAction.ALLOW,
|
|
||||||
filtered_text=text,
|
|
||||||
matched_terms=[],
|
|
||||||
)
|
|
||||||
|
|
||||||
matched_terms.sort()
|
|
||||||
|
|
||||||
# Build filtered text by replacing matched terms
|
|
||||||
filtered = text
|
|
||||||
for term in matched_terms:
|
|
||||||
norm_term = _normalize(term)
|
|
||||||
# Find all occurrences in the normalised text and replace in original
|
|
||||||
# We need to map positions from normalised text back to original
|
|
||||||
filtered = self._replace_terms(filtered, term, norm_term)
|
|
||||||
|
|
||||||
action = FilterAction.REJECT if self._policy.mode == "reject" else FilterAction.REDACT
|
|
||||||
|
|
||||||
return FilterResult(
|
|
||||||
action=action,
|
|
||||||
filtered_text=filtered,
|
|
||||||
matched_terms=matched_terms,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _replace_terms(
|
|
||||||
self, text: str, term: str, norm_term: str
|
|
||||||
) -> str:
|
|
||||||
"""Replace occurrences of *term* in *text* with redaction chars.
|
|
||||||
|
|
||||||
Uses normalised text for matching but preserves original casing
|
|
||||||
in the output.
|
|
||||||
"""
|
|
||||||
norm_text = _normalize(text)
|
|
||||||
norm_term_len = len(norm_term)
|
|
||||||
result_chars = list(text)
|
|
||||||
|
|
||||||
start = 0
|
|
||||||
while start < len(norm_text):
|
|
||||||
idx = norm_text.find(norm_term, start)
|
|
||||||
if idx == -1:
|
|
||||||
break
|
|
||||||
# Replace characters in original text
|
|
||||||
for i in range(idx, min(idx + norm_term_len, len(result_chars))):
|
|
||||||
result_chars[i] = self._policy.redaction_char
|
|
||||||
start = idx + norm_term_len
|
|
||||||
|
|
||||||
return "".join(result_chars)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def terms(self) -> list[str]:
|
|
||||||
"""Read-only access to the loaded term list."""
|
|
||||||
return list(self._terms)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def policy(self) -> _FilterPolicy:
|
|
||||||
"""Read-only access to the filter policy."""
|
|
||||||
return self._policy
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Factory
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
|
|
||||||
def get_text_filter(
|
|
||||||
config_path: str | Path | None = None,
|
|
||||||
terms: list[str] | None = None,
|
|
||||||
policy: _FilterPolicy | None = None,
|
|
||||||
) -> TextFilter:
|
|
||||||
"""Return a ``TextFilter`` instance.
|
|
||||||
|
|
||||||
By default this creates a ``ProfanityTextFilter``. To use a
|
|
||||||
third-party provider, subclass ``TextFilter`` and pass an
|
|
||||||
instance here instead.
|
|
||||||
"""
|
|
||||||
return ProfanityTextFilter(
|
|
||||||
config_path=config_path,
|
|
||||||
terms=terms,
|
|
||||||
policy=policy,
|
|
||||||
)
|
|
||||||
Reference in New Issue
Block a user