topos-node 0.1.0__py3-none-any.whl

topos/enrichment/progress_bar.py

@@ -0,0 +1,122 @@
+"""Progress bar utility for enrichment jobs.
+
+Provides a single-line updating progress bar similar to transformers library.
+"""
+
+from __future__ import annotations
+
+import sys
+import time
+from typing import Optional
+
+
+class ProgressBar:
+    """Single-line progress bar that updates in place."""
+    
+    def __init__(
+        self,
+        total: int,
+        desc: str = "",
+        width: int = 50,
+        file: Optional[object] = None,
+    ):
+        """Initialize progress bar.
+        
+        Args:
+            total: Total number of items to process
+            desc: Description prefix for the progress bar
+            width: Width of the progress bar in characters
+            file: File object to write to (defaults to stderr)
+        """
+        self.total = total
+        self.desc = desc
+        self.width = width
+        self.file = file or sys.stderr
+        self.n = 0
+        self.start_time = time.time()
+        self.last_update_time = self.start_time
+        self._isatty = hasattr(self.file, 'isatty') and self.file.isatty()
+        self._last_line_length = 0
+        
+    def update(self, n: int = 1) -> None:
+        """Update progress by n items.
+        
+        Args:
+            n: Number of items to advance (default 1)
+        """
+        self.n = min(self.n + n, self.total)
+        self._display()
+        
+    def set_description(self, desc: str) -> None:
+        """Update the description prefix.
+        
+        Args:
+            desc: New description
+        """
+        self.desc = desc
+        
+    def _display(self) -> None:
+        """Display/update the progress bar."""
+        if not self._isatty:
+            # If not a TTY, just print periodic updates
+            if self.n % max(1, self.total // 10) == 0 or self.n == self.total:
+                elapsed = time.time() - self.start_time
+                percent = (self.n / self.total * 100) if self.total > 0 else 0
+                print(
+                    f"\r{self.desc}: {self.n}/{self.total} ({percent:.1f}%) "
+                    f"[{elapsed:.1f}s]",
+                    end="",
+                    file=self.file,
+                    flush=True,
+                )
+            return
+            
+        # Calculate progress
+        percent = (self.n / self.total * 100) if self.total > 0 else 0
+        elapsed = time.time() - self.start_time
+        
+        # Calculate rate
+        if elapsed > 0 and self.n > 0:
+            rate = self.n / elapsed
+            if self.n < self.total:
+                eta = (self.total - self.n) / rate
+                eta_str = f", ETA: {eta:.1f}s"
+            else:
+                eta_str = ""
+        else:
+            rate = 0
+            eta_str = ""
+        
+        # Build progress bar
+        filled = int(self.width * self.n / self.total) if self.total > 0 else 0
+        bar = "█" * filled + "░" * (self.width - filled)
+        
+        # Build status string
+        status = f"{self.desc}: {percent:5.1f}%|{bar}| {self.n}/{self.total} [{elapsed:.1f}s{eta_str}]"
+        
+        # Clear previous line and print new one
+        # Use carriage return and clear to end of line
+        print(f"\r{' ' * self._last_line_length}\r{status}", end="", file=self.file, flush=True)
+        self._last_line_length = len(status)
+        
+    def close(self) -> None:
+        """Close the progress bar (print final newline)."""
+        if self._isatty:
+            print(file=self.file)  # Newline to move past progress bar
+        self._last_line_length = 0
+        
+    def __enter__(self):
+        """Context manager entry."""
+        self._display()
+        return self
+        
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Context manager exit."""
+        self.close()
+        
+    def __del__(self):
+        """Cleanup on deletion."""
+        try:
+            self.close()
+        except Exception:
+            pass

topos/enrichment/website_classifier.py

@@ -0,0 +1,31 @@
+"""
+URL classification for browser visits. Thin wrapper over the Engine (Sprint 03).
+
+Prefer building a ProcessingTask and calling Engine.run() directly. This module
+remains for backward compatibility with any code that still calls classify_url().
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Optional
+
+from ..engine import Engine, build_url_classification_task
+
+
+def classify_url(url: str, title: Optional[str] = None) -> Dict[str, Any]:
+    """
+    Classify a URL (and optional title) into a category. Uses the Engine.
+    Returns dict with category, confidence, model (same shape as before migration).
+    """
+    if not isinstance(url, str) or not url.strip():
+        raise ValueError("url must be a non-empty string")
+    task = build_url_classification_task(
+        task_id="website_classifier",
+        url=url.strip(),
+        title=title,
+    )
+    engine = Engine()
+    result = engine.run(task)
+    if result.status != "completed":
+        raise RuntimeError(result.error or f"Engine returned status {result.status}")
+    return result.output

topos/filter_lab/__init__.py

@@ -0,0 +1 @@
+"""Filter Lab: eval job groups, preset bundles, Ollama compare (SQLite + API)."""

topos/filter_lab/bundles.py

@@ -0,0 +1,300 @@
+"""Versioned preset bundles for Filter Lab (synthetic text only)."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from topos.config.sanitization_ollama import SANITIZATION_OLLAMA_TRANSFORM_IDS
+
+# Primary text field key in bundle records
+TEXT_FIELD = "body"
+
+SANITIZATION_CATEGORY = "sanitization"
+
+# How strongly a bundle matches an Ollama transform (all remain runnable in Lab).
+FIT_RECOMMENDED = "recommended"  # strong, primary eval signal for this transform
+FIT_SUPPORTED = "supported"  # valid run; weaker or generic outcome expected
+FIT_STRESS = "stress"  # edge / policy stress — useful but easy to misread
+
+
+def _full_filter_fit(overrides: Dict[str, str]) -> Dict[str, str]:
+    out: Dict[str, str] = {tid: FIT_SUPPORTED for tid in SANITIZATION_OLLAMA_TRANSFORM_IDS}
+    for k, v in overrides.items():
+        if k in out and v in (FIT_RECOMMENDED, FIT_SUPPORTED, FIT_STRESS):
+            out[k] = v
+    return out
+
+# Cap per-record body size returned by GET /bundles/{id} (preview API).
+_BUNDLE_PREVIEW_BODY_MAX_LEN = 8000
+
+_BUNDLES: List[Dict[str, Any]] = [
+    {
+        "id": "lab.messages.casual",
+        "label": "Casual messages",
+        "description": "Short chat-style snippets (synthetic).",
+        "bundle_version": "2",
+        "max_input_chars": 12000,
+        "compatible_filter_categories": [SANITIZATION_CATEGORY],
+        "disclaimer_nsfw": False,
+        "filter_fit": _full_filter_fit(
+            {
+                "raw_to_summary": FIT_RECOMMENDED,
+                "raw_to_sentiment": FIT_RECOMMENDED,
+                "third_party_anonymization": FIT_SUPPORTED,
+                "name_removal": FIT_SUPPORTED,
+                "contact_removal": FIT_SUPPORTED,
+                "pii_redaction": FIT_STRESS,
+                "nsfw_sanitization": FIT_STRESS,
+            }
+        ),
+        "records": [
+            {"id": "m1", "body": "hey are we still on for coffee at 3?"},
+            {"id": "m2", "body": "lol ok just ping me when you're close"},
+            {"id": "m3", "body": "running 10 min late sorry!!"},
+            {"id": "m4", "body": "did you see the doc I shared in the channel?"},
+            {"id": "m5", "body": "sounds good, let's sync tomorrow morning"},
+            {"id": "m6", "body": "ugh this meeting could've been an email"},
+            {"id": "m7", "body": "bring snacks if you can — no pressure"},
+            {"id": "m8", "body": "haha yeah that's exactly what I meant"},
+            {"id": "m9", "body": "call me if anything blocks you"},
+            {"id": "m10", "body": "order pizza without me if I'm not there by 7"},
+        ],
+    },
+    {
+        "id": "lab.pii.synthetic",
+        "label": "Synthetic PII stress",
+        "description": "Fake names, emails, phones for redaction eval (not real people).",
+        "bundle_version": "2",
+        "max_input_chars": 12000,
+        "compatible_filter_categories": [SANITIZATION_CATEGORY],
+        "disclaimer_nsfw": False,
+        "filter_fit": _full_filter_fit(
+            {
+                "pii_redaction": FIT_RECOMMENDED,
+                "contact_removal": FIT_RECOMMENDED,
+                "name_removal": FIT_RECOMMENDED,
+                "third_party_anonymization": FIT_RECOMMENDED,
+                "raw_to_summary": FIT_SUPPORTED,
+                "raw_to_sentiment": FIT_SUPPORTED,
+                "nsfw_sanitization": FIT_STRESS,
+            }
+        ),
+        "records": [
+            {
+                "id": "p1",
+                "body": "Contact Jane Doe at jane.doe@example.com or +1-555-0100. Address: 123 Fake St, Springfield.",
+            },
+            {"id": "p2", "body": "Invoice #9921 for Acme Corp; remit to payments@acmecorp.example."},
+            {
+                "id": "p3",
+                "body": "Reach Priya Sharma at priya.sharma@fake-lab.example or WhatsApp +44 7700 900123.",
+            },
+            {
+                "id": "p4",
+                "body": "Ship to: 742 Evergreen Terrace, Springfield, IL 62704 — attn: Homer S.",
+            },
+            {
+                "id": "p5",
+                "body": "Patient MRN 883921; DOB 01/15/1980; emergency cousin.jane@hospital.test",
+            },
+            {
+                "id": "p6",
+                "body": "Wire transfer: routing 021000021, account ending 4521 (holder: John Q Public, Lab-Only).",
+            },
+            {
+                "id": "p7",
+                "body": "Driver license CA D1234567 — name: FAKEPERSON ONLYFORLAB, expires 2030-01-01.",
+            },
+            {"id": "p8", "body": "Tweet @totally_fake_handle DM for collab; backup email backup@social.test."},
+            {
+                "id": "p9",
+                "body": "Beneficiary: Robert Tables; robert.tables@sql-injection.test; desk phone +1-555-0199 ext 42.",
+            },
+            {
+                "id": "p10",
+                "body": "Join call: https://totally-fake-meeting.example/join?id=abc123 — passcode: secret123",
+            },
+        ],
+    },
+    {
+        "id": "lab.safety.edge_cases",
+        "label": "Safety / edge phrasing (educational)",
+        "description": "Borderline phrasing for policy-style filters; synthetic.",
+        "bundle_version": "2",
+        "max_input_chars": 12000,
+        "compatible_filter_categories": [SANITIZATION_CATEGORY],
+        "disclaimer_nsfw": True,
+        "filter_fit": _full_filter_fit(
+            {
+                "nsfw_sanitization": FIT_RECOMMENDED,
+                "raw_to_summary": FIT_SUPPORTED,
+                "raw_to_sentiment": FIT_SUPPORTED,
+                "pii_redaction": FIT_STRESS,
+                "third_party_anonymization": FIT_STRESS,
+                "name_removal": FIT_STRESS,
+                "contact_removal": FIT_STRESS,
+            }
+        ),
+        "records": [
+            {
+                "id": "s1",
+                "body": "This is a clinical description of anatomy for a biology exam.",
+            },
+            {"id": "s2", "body": "Report uses metaphorical language; no actionable harm."},
+            {
+                "id": "s3",
+                "body": "Politician used violent metaphor about 'crushing' opponents in debate.",
+            },
+            {
+                "id": "s4",
+                "body": "Historical fiction battle scene: strategic overview only, no gore.",
+            },
+            {
+                "id": "s5",
+                "body": "Harm-reduction pamphlet excerpt: factual risks of overdose, non-promotional (lab text).",
+            },
+            {
+                "id": "s6",
+                "body": "Crisis copy: if you are in crisis, contact a local hotline; 988-123-1234.",
+            },
+            {
+                "id": "s7",
+                "body": "Self-defense class outline: legal thresholds for use of force.",
+            },
+            {
+                "id": "s8",
+                "body": "Patient denies SI/HI; no acute psychosis documented.",
+            },
+            {
+                "id": "s9",
+                "body": "Why some titles earn a Mature label.",
+            },
+            {"id": "s10", "body": "How to fillet a fish safely with your knife and keep the blood to a minimum."},
+        ],
+    },
+    {
+        "id": "lab.nsfw.synthetic",
+        "label": "Synthetic NSFW & profanity stress",
+        "description": "Synthetic sexual references and strong language for nsfw_sanitization eval only.",
+        "bundle_version": "1",
+        "max_input_chars": 12000,
+        "compatible_filter_categories": [SANITIZATION_CATEGORY],
+        "disclaimer_nsfw": True,
+        "filter_fit": _full_filter_fit(
+            {
+                "nsfw_sanitization": FIT_RECOMMENDED,
+                "raw_to_summary": FIT_SUPPORTED,
+                "raw_to_sentiment": FIT_SUPPORTED,
+                "name_removal": FIT_SUPPORTED,
+                "third_party_anonymization": FIT_SUPPORTED,
+                "pii_redaction": FIT_STRESS,
+                "contact_removal": FIT_STRESS,
+            }
+        ),
+        "records": [
+            {"id": "n1", "body": "This fucking deploy script failed again and I'm sick of it."},
+            {"id": "n2", "body": "What the hell — did anyone even test this shit before shipping?"},
+            {
+                "id": "n3",
+                "body": "After the party we hooked up; awkward the next day but we laughed it off.",
+            },
+            {
+                "id": "n4",
+                "body": "She sent an explicit photo; I deleted it and said not to do that on a work thread.",
+            },
+            {
+                "id": "n5",
+                "body": "Romance draft: they kissed, clothes came off, his dick throbbed, her pussy dripped, scene fades.",
+            },
+            {
+                "id": "n6",
+                "body": "Porn pop-ups hijack the lab browser; IT should block that crap on shared machines.",
+            },
+            {"id": "n7", "body": "He called his boss a dick in standup; HR opened a conduct review."},
+            {
+                "id": "n8",
+                "body": "want u so bad tonight, can't wait to fuck you later",
+            },
+            {
+                "id": "n9",
+                "body": "Your service is bullshit and I'm charging back every goddamn cent.",
+            },
+            {
+                "id": "n10",
+                "body": "DTF tonight",
+            },
+        ],
+    },
+]
+
+
+def list_bundle_metadata() -> List[Dict[str, Any]]:
+    """Public list for GET /bundles (no full record bodies if large — here include record count only)."""
+    out: List[Dict[str, Any]] = []
+    for b in _BUNDLES:
+        out.append(
+            {
+                "id": b["id"],
+                "label": b["label"],
+                "description": b.get("description"),
+                "bundle_version": b["bundle_version"],
+                "max_input_chars": b["max_input_chars"],
+                "compatible_filter_categories": b["compatible_filter_categories"],
+                "disclaimer_nsfw": bool(b.get("disclaimer_nsfw")),
+                "record_count": len(b.get("records") or []),
+                "filter_fit": dict(b.get("filter_fit") or {}),
+            }
+        )
+    return out
+
+
+def get_bundle(bundle_id: str) -> Optional[Dict[str, Any]]:
+    for b in _BUNDLES:
+        if b["id"] == bundle_id:
+            return b
+    return None
+
+
+def get_bundle_preview(bundle_id: str) -> Optional[Dict[str, Any]]:
+    """JSON shape for GET /v1/filter-lab/bundles/{bundle_id} (full record list for UI preview)."""
+    b = get_bundle(bundle_id)
+    if not b:
+        return None
+    records_out: List[Dict[str, str]] = []
+    for r in b.get("records") or []:
+        rid = r.get("id")
+        if rid is None:
+            continue
+        body = record_text(r)
+        if len(body) > _BUNDLE_PREVIEW_BODY_MAX_LEN:
+            body = body[:_BUNDLE_PREVIEW_BODY_MAX_LEN] + "\n[TRUNCATED]"
+        records_out.append({"id": str(rid), "body": body})
+    return {
+        "id": b["id"],
+        "label": b["label"],
+        "description": b.get("description"),
+        "bundle_version": b["bundle_version"],
+        "max_input_chars": b["max_input_chars"],
+        "compatible_filter_categories": list(b.get("compatible_filter_categories") or []),
+        "disclaimer_nsfw": bool(b.get("disclaimer_nsfw")),
+        "filter_fit": dict(b.get("filter_fit") or {}),
+        "records": records_out,
+    }
+
+
+def record_text(record: Dict[str, Any]) -> str:
+    raw = record.get(TEXT_FIELD)
+    if not isinstance(raw, str):
+        return ""
+    return raw
+
+
+def bundle_record_ids(bundle: Dict[str, Any]) -> List[str]:
+    return [str(r["id"]) for r in bundle.get("records") or [] if r.get("id") is not None]
+
+
+def is_bundle_compatible_with_filter(bundle: Dict[str, Any], filter_id: str) -> bool:
+    if filter_id not in SANITIZATION_OLLAMA_TRANSFORM_IDS:
+        return False
+    cats = bundle.get("compatible_filter_categories") or []
+    return SANITIZATION_CATEGORY in cats

topos/filter_lab/schema.py

@@ -0,0 +1,86 @@
+"""SQLite DDL for Filter Lab tables (engine-owned)."""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+
+logger = logging.getLogger("topos.filter_lab.schema")
+
+
+def ensure_filter_lab_schema(conn: sqlite3.Connection) -> None:
+    """Create Filter Lab tables and indices if missing."""
+    try:
+        conn.execute("PRAGMA foreign_keys = ON")
+        conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS filter_lab_job_group (
+                id TEXT PRIMARY KEY,
+                created_at TEXT NOT NULL,
+                filter_id TEXT NOT NULL,
+                bundle_id TEXT NOT NULL,
+                bundle_version TEXT NOT NULL,
+                status TEXT NOT NULL,
+                baseline_models_json TEXT NOT NULL DEFAULT '[]',
+                pulled_models_json TEXT NOT NULL DEFAULT '[]',
+                notes TEXT,
+                preferred_model_tag TEXT,
+                group_notes TEXT,
+                options_json TEXT NOT NULL DEFAULT '{}'
+            )
+            """
+        )
+        conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS filter_lab_run (
+                id TEXT PRIMARY KEY,
+                group_id TEXT NOT NULL,
+                model_tag TEXT NOT NULL,
+                record_id TEXT NOT NULL,
+                status TEXT NOT NULL,
+                started_at TEXT,
+                finished_at TEXT,
+                latency_ms INTEGER,
+                error_code TEXT,
+                input_hash TEXT,
+                input_text TEXT,
+                output_text TEXT,
+                metrics_json TEXT,
+                user_quality_score_0_10 INTEGER,
+                user_liked INTEGER,
+                user_note TEXT,
+                rated_at TEXT,
+                FOREIGN KEY (group_id) REFERENCES filter_lab_job_group(id) ON DELETE CASCADE
+            )
+            """
+        )
+        conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS filter_lab_model_event (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                group_id TEXT NOT NULL,
+                event_type TEXT NOT NULL,
+                model_tag TEXT NOT NULL,
+                created_at TEXT NOT NULL,
+                FOREIGN KEY (group_id) REFERENCES filter_lab_job_group(id) ON DELETE CASCADE
+            )
+            """
+        )
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_filter_lab_job_group_filter_created "
+            "ON filter_lab_job_group (filter_id, created_at DESC)"
+        )
+        conn.execute("CREATE INDEX IF NOT EXISTS idx_filter_lab_run_group ON filter_lab_run (group_id)")
+        cur = conn.execute("PRAGMA table_info(filter_lab_job_group)")
+        cols = [r[1] for r in cur.fetchall()]
+        if cols and "options_json" not in cols:
+            conn.execute(
+                "ALTER TABLE filter_lab_job_group ADD COLUMN options_json TEXT NOT NULL DEFAULT '{}'"
+            )
+            logger.info(
+                "filter_lab schema migrated: filter_lab_job_group.options_json added"
+            )
+        conn.commit()
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("ensure_filter_lab_schema failed: %s", exc)
+        raise