topos-node 0.1.18__tar.gz → 1.0.0__tar.gz

{topos_node-0.1.18 → topos_node-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: topos-node
-Version: 0.1.18
+Version: 1.0.0
 Summary: Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://topos.dialogues.ai
@@ -22,17 +22,30 @@ Requires-Dist: packaging<26.2,>=26.1
 Requires-Dist: mcp~=1.27.0
 Requires-Dist: google-cloud-storage~=3.10.1
 Requires-Dist: google-cloud-run~=0.16.0
-Requires-Dist: networkx<3.7,>=3.2.1
-Requires-Dist: transformers~=5.10.2
-Requires-Dist: huggingface-hub~=1.18.0
-Requires-Dist: tqdm~=4.67.3
-Requires-Dist: hf-xet~=1.5.1
-Requires-Dist: torch<3,>=2.8.0
-Requires-Dist: python-louvain<0.17,>=0.16
 Requires-Dist: asgi-lifespan~=2.1.0
+Provides-Extra: database
+Requires-Dist: psycopg[binary]~=3.3.3; extra == "database"
 Provides-Extra: engine
 Requires-Dist: duckdb~=1.5.2; extra == "engine"
-Requires-Dist: psycopg[binary]~=3.3.3; extra == "engine"
+Requires-Dist: torch<3,>=2.8.0; extra == "engine"
+Requires-Dist: transformers~=5.10.2; extra == "engine"
+Requires-Dist: huggingface-hub~=1.18.0; extra == "engine"
+Requires-Dist: tqdm~=4.67.3; extra == "engine"
+Requires-Dist: hf-xet~=1.5.1; extra == "engine"
+Requires-Dist: sentence-transformers~=5.6.0; extra == "engine"
+Requires-Dist: networkx<3.7,>=3.2.1; extra == "engine"
+Requires-Dist: python-louvain<0.17,>=0.16; extra == "engine"
+Provides-Extra: local
+Requires-Dist: psycopg[binary]~=3.3.3; extra == "local"
+Requires-Dist: duckdb~=1.5.2; extra == "local"
+Requires-Dist: torch<3,>=2.8.0; extra == "local"
+Requires-Dist: transformers~=5.10.2; extra == "local"
+Requires-Dist: huggingface-hub~=1.18.0; extra == "local"
+Requires-Dist: tqdm~=4.67.3; extra == "local"
+Requires-Dist: hf-xet~=1.5.1; extra == "local"
+Requires-Dist: sentence-transformers~=5.6.0; extra == "local"
+Requires-Dist: networkx<3.7,>=3.2.1; extra == "local"
+Requires-Dist: python-louvain<0.17,>=0.16; extra == "local"
 Provides-Extra: signal
 Requires-Dist: pysqlcipher3~=1.2.0; extra == "signal"
 Provides-Extra: dev
@@ -109,7 +122,7 @@ The `shared/` package in this repo is part of the node runtime contract. It cont
 ### 1) Install
 
 ```bash
-uv tool install topos-node
+uv tool install 'topos-node[local]'
 ```
 
 ### 2) Configure

{topos_node-0.1.18 → topos_node-1.0.0}/README.md

@@ -66,7 +66,7 @@ The `shared/` package in this repo is part of the node runtime contract. It cont
 ### 1) Install
 
 ```bash
-uv tool install topos-node
+uv tool install 'topos-node[local]'
 ```
 
 ### 2) Configure

{topos_node-0.1.18 → topos_node-1.0.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "topos-node"
-version = "0.1.18"
+version = "1.0.0"
 description = "Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client"
 readme = "README.md"
 license = "Apache-2.0"
@@ -24,13 +24,6 @@ dependencies = [
   "mcp~=1.27.0",
   "google-cloud-storage~=3.10.1",
   "google-cloud-run~=0.16.0",
-  "networkx>=3.2.1,<3.7",
-  "transformers~=5.10.2",
-  "huggingface-hub~=1.18.0",
-  "tqdm~=4.67.3",
-  "hf-xet~=1.5.1",
-  "torch>=2.8.0,<3",
-  "python-louvain>=0.16,<0.17",
   "asgi-lifespan~=2.1.0",
 ]
 
@@ -39,9 +32,34 @@ Homepage = "https://topos.dialogues.ai"
 Source = "https://dialogues.ai"
 
 [project.optional-dependencies]
+# Lean database / sync plane (hosted topos-database Cloud Run image).
+database = [
+  "psycopg[binary]~=3.3.3",
+]
+# ML + enrichment runtime (hosted topos-engine Cloud Run image).
 engine = [
   "duckdb~=1.5.2",
+  "torch>=2.8.0,<3",
+  "transformers~=5.10.2",
+  "huggingface-hub~=1.18.0",
+  "tqdm~=4.67.3",
+  "hf-xet~=1.5.1",
+  "sentence-transformers~=5.6.0",
+  "networkx>=3.2.1,<3.7",
+  "python-louvain>=0.16,<0.17",
+]
+# Colocated local node (database + engine on one device). Default for PyPI / uv tool install.
+local = [
   "psycopg[binary]~=3.3.3",
+  "duckdb~=1.5.2",
+  "torch>=2.8.0,<3",
+  "transformers~=5.10.2",
+  "huggingface-hub~=1.18.0",
+  "tqdm~=4.67.3",
+  "hf-xet~=1.5.1",
+  "sentence-transformers~=5.6.0",
+  "networkx>=3.2.1,<3.7",
+  "python-louvain>=0.16,<0.17",
 ]
 signal = [
   "pysqlcipher3~=1.2.0",
@@ -65,8 +83,13 @@ filterwarnings = [
   "ignore:on_event is deprecated.*:DeprecationWarning:fastapi.*",
 ]
 markers = [
+  "gap: gap verification test (sprint pack); must pass after sprint lands",
+  "qq_eval: live query quality/latency eval against owner database",
+  "acceptance: Wiki MVP acceptance test (AT-1 through AT-8)",
+  "hosted_parity: hosted vs local parity (staging/nightly)",
   "public: public-safe test lane for OSS CI",
   "private: internal architecture or fixtures; excluded from public CI",
   "e2e: end-to-end tests (live Keycloak + Control Plane; require config and running CP)",
 ]
+pythonpath = ["tests/gap/p3/engine", "tests/gap/p4/engine", "tests/gap/qq/engine", "tests/acceptance/wiki_mvp"]
 testpaths = ["tests"]

{topos_node-0.1.18 → topos_node-1.0.0}/topos/__version__.py

@@ -1,6 +1,6 @@
 """Topos Control Plane Version Information."""
 
-__version__ = "0.1.18"
-__version_info__ = (0, 1, 18)
+__version__ = "1.0.0"
+__version_info__ = (1, 0, 0)
 
 __all__ = ["__version__", "__version_info__"]

{topos_node-0.1.18 → topos_node-1.0.0}/topos/api/enrichment.py

@@ -85,36 +85,25 @@ async def _backfill_browser_visits_url_classification(
     only_missing: bool = True,
     limit: Optional[int] = None,
 ) -> Dict[str, Any]:
-    """Backfill URL classification for normalized browser visits raw table rows."""
-    from ..engine import Engine, build_url_classification_task
-    from ..storage.raw.browser_flat_tables import (
-        ensure_browser_url_classification_table,
-        write_browser_url_classification,
-    )
-    from ..enrichment.progress_bar import ProgressBar
-    from ..storage.raw.raw_tables_manager import RawTablesManager
+    """Backfill Interests signal facts from canonical activity_events (not raw tables)."""
+    import uuid
 
-    source_table = "raw_chat_messages_browservisits"
+    from ..ingestion.canonical_pipeline import (
+        activity_payload_to_signal_record,
+        run_post_canonical_pipeline,
+    )
+    from ..sources.registry import BROWSER_VISITS
 
     logger.info(
-        "[PIPELINE:ENRICHMENT] Source backfill start: source=browser_visits enrichment=url_classification only_missing=%s limit=%s",
+        "[PIPELINE:SIGNAL_DERIVE] Browser visits backfill from activity_events only_missing=%s limit=%s",
         only_missing,
         limit,
     )
 
-    # Ensure/migrate the raw browser visits table to normalized-column schema first.
-    RawTablesManager(db_conn).ensure_raw_table(source_table)
-
-    # If source table does not exist yet, return an empty success result.
-    source_exists = db_conn.execute(
-        "SELECT name FROM sqlite_master WHERE type='table' AND name=?",
-        (source_table,),
+    table_exists = db_conn.execute(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name='activity_events'"
     ).fetchone()
-    if not source_exists:
-        logger.info(
-            "[PIPELINE:ENRICHMENT] Source backfill complete: source table missing (%s)",
-            source_table,
-        )
+    if not table_exists:
         return {
             "rows_scanned": 0,
             "rows_processed": 0,
@@ -123,100 +112,71 @@ async def _backfill_browser_visits_url_classification(
             "errors": [],
         }
 
-    ensure_browser_url_classification_table(db_conn)
-
-    params: List[Any] = []
+    params: List[Any] = ["browser_visits"]
+    query = """
+        SELECT event_id, activity_type, url, title, occurred_at, source_id
+        FROM activity_events
+        WHERE source_id=?
+    """
     if only_missing:
-        query = """
-            SELECT
-                (COALESCE(v.url, '') || '_' || COALESCE(v.visited_at, '')) AS derived_record_id,
-                v.dataset_id,
-                v.url,
-                v.title
-            FROM raw_chat_messages_browservisits v
-            LEFT JOIN browser_url_classification c
-              ON c.source_table = 'browser_visits'
-             AND c.record_id = (COALESCE(v.url, '') || '_' || COALESCE(v.visited_at, ''))
-            WHERE c.record_id IS NULL
-            ORDER BY v.visited_at ASC
-        """
-    else:
-        query = """
-            SELECT
-                (COALESCE(v.url, '') || '_' || COALESCE(v.visited_at, '')) AS derived_record_id,
-                v.dataset_id,
-                v.url,
-                v.title
-            FROM raw_chat_messages_browservisits v
-            ORDER BY v.visited_at ASC
+        query += """
+          AND event_id NOT IN (
+            SELECT record_id FROM signal_facts
+            WHERE source_id='browser_visits' AND dimension='interests'
+          )
         """
+    query += " ORDER BY occurred_at ASC"
     if isinstance(limit, int) and limit > 0:
         query += " LIMIT ?"
         params.append(limit)
 
     rows = db_conn.execute(query, tuple(params)).fetchall()
+    canonical_records = [
+        activity_payload_to_signal_record(
+            {
+                "event_id": row[0],
+                "activity_type": row[1],
+                "url": row[2],
+                "title": row[3],
+                "occurred_at": row[4],
+                "source_id": row[5],
+            },
+            source_id="browser_visits",
+        )
+        for row in rows
+        if isinstance(row[2], str) and row[2].strip()
+    ]
+
+    skipped = max(len(rows) - len(canonical_records), 0)
+    if not canonical_records:
+        return {
+            "rows_scanned": len(rows),
+            "rows_processed": 0,
+            "rows_skipped": skipped + len(rows),
+            "rows_failed": 0,
+            "errors": [],
+        }
 
-    processed = 0
-    skipped = 0
-    failed = 0
-    errors: List[Dict[str, Any]] = []
-
-    if rows:
-        with ProgressBar(total=len(rows), desc="url_classification backfill") as pbar:
-            for row in rows:
-                record_id = row[0]
-                dataset_id = row[1]
-                url = row[2]
-                title = row[3]
-                if not isinstance(url, str) or not url.strip():
-                    skipped += 1
-                    pbar.update(1)
-                    continue
-
-                try:
-                    task = build_url_classification_task(
-                        task_id=f"backfill_url_{record_id}",
-                        url=url,
-                        title=title,
-                        source_id="browser_visits",
-                        record_ids=[record_id],
-                    )
-                    engine = Engine()
-                    result = await asyncio.to_thread(engine.run, task)
-                    if result.status != "completed":
-                        failed += 1
-                        errors.append({"record_id": record_id, "error": result.error or result.status})
-                        continue
-                    out = result.output
-                    write_browser_url_classification(
-                        db_conn,
-                        source_table="browser_visits",
-                        record_id=record_id,
-                        dataset_id=dataset_id,
-                        url=url,
-                        title=title,
-                        category=out.get("category"),
-                        confidence=out.get("confidence"),
-                        model_name=out.get("model"),
-                        ensure_table=False,
-                        log_write=False,  # Avoid per-row log spam during bulk backfill
-                    )
-                    processed += 1
-                except Exception as exc:  # noqa: BLE001
-                    failed += 1
-                    errors.append({"record_id": record_id, "error": str(exc)})
-                finally:
-                    pbar.update(1)
+    pipeline_outcome = await run_post_canonical_pipeline(
+        source_def=BROWSER_VISITS,
+        canonical_records=canonical_records,
+        sync_batch_id=f"backfill_browser_visits_{uuid.uuid4().hex[:12]}",
+        run_enrichment=False,
+        job_names=["url_classification"],
+    )
+    derive_result = pipeline_outcome.get("signal_derivation") or {}
+    processed = sum((derive_result.get("records_created") or {}).values())
+    failed = len(derive_result.get("errors") or [])
 
     summary = {
         "rows_scanned": len(rows),
-        "rows_processed": processed,
+        "rows_processed": int(processed),
         "rows_skipped": skipped,
         "rows_failed": failed,
-        "errors": errors[:100],
+        "errors": (derive_result.get("errors") or [])[:100],
     }
     logger.info(
-        "[PIPELINE:ENRICHMENT] Source backfill complete: source=browser_visits enrichment=url_classification scanned=%d processed=%d skipped=%d failed=%d",
+        "[PIPELINE:SIGNAL_DERIVE] Browser visits backfill complete: scanned=%d processed=%d skipped=%d failed=%d",
         summary["rows_scanned"],
         summary["rows_processed"],
         summary["rows_skipped"],

topos_node-1.0.0/topos/api/ingestion_api.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from fastapi import APIRouter, Body, Depends, HTTPException, Request
+
+from ..auth import require_api_key
+from ..ingestion.reprocess import reprocess_source
+from .ingestion_sources import ingest_source
+
+router = APIRouter()
+
+
+@router.post("/ingestion/start")
+async def start_ingestion() -> dict:
+    return {"status": "stub"}
+
+
+@router.post("/ingestion/progress")
+async def report_progress() -> dict:
+    return {"status": "stub"}
+
+
+@router.post("/ingestion/reprocess", dependencies=[Depends(require_api_key)])
+async def ingestion_reprocess(payload: dict = Body(default_factory=dict)) -> dict:
+    try:
+        return await reprocess_source(
+            source_id=str(payload["source_id"]),
+            dataset_id=str(payload["dataset_id"]),
+            from_stage=payload.get("from_stage") or "raw",
+            sync_batch_id=payload.get("sync_batch_id"),
+            force=bool(payload.get("force", False)),
+        )
+    except KeyError as exc:
+        raise HTTPException(status_code=400, detail=f"Missing field: {exc}") from exc
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+
+@router.get("/ingestion/audit", dependencies=[Depends(require_api_key)])
+async def get_ingestion_audit(sync_batch_id: str) -> dict:
+    from ..core.state import get_db_connection
+    from ..pipeline.audit import SQLiteIngestAuditStore
+
+    conn = get_db_connection()
+    if not conn:
+        raise HTTPException(status_code=503, detail="Database unavailable")
+    store = SQLiteIngestAuditStore(conn)
+    return {"sync_batch_id": sync_batch_id, "stages": store.query_by_batch(sync_batch_id)}
+
+
+@router.post("/ingestion/upload", dependencies=[Depends(require_api_key)])
+async def upload_ingestion_file(request: Request) -> dict:
+    return await ingest_source("chatgpt_file_ingestion", request)
+
+
+@router.post("/ingestion/upload-local-path", dependencies=[Depends(require_api_key)])
+async def upload_ingestion_local_path(
+    request: Request, payload: dict = Body(default_factory=dict)
+) -> dict:
+    dataset_id = payload.get("dataset_id")
+    file_path = payload.get("file_path")
+    return await ingest_source(
+        "chatgpt_file_ingestion",
+        request,
+        dataset_id=dataset_id,
+        file_path=file_path,
+    )

{topos_node-0.1.18 → topos_node-1.0.0}/topos/api/ingestion_sources.py

@@ -10,7 +10,7 @@ from fastapi import APIRouter, Body, Depends, Query, Request  # noqa: F401 Body
 
 from ..auth import require_api_key
 from ..core.state import get_db_connection
-from ..ingestion.ingest_helpers import ingest_file_payload, ingest_ui_payload
+from ..ingestion.ingest_helpers import ingest_file_payload, ingest_ui_payload, resolve_file_format
 from ..ingestion.local_sync import run_imessage_sync, run_signal_sync, run_signal_upload
 from ..sources.registry import REGISTRY
 from ..storage.signal_identity import get_signal_identity, put_signal_identity
@@ -86,6 +86,8 @@ async def ingest_source(
                 dataset_id=dataset_id,
                 schema_id=source.schema_id,
                 file_path=file_path,
+                source_id=source_id,
+                file_format=resolve_file_format(source_definition=source, file_path=file_path),
             )
         payload_bytes = await file.read()
         guard = await submit_usage_guard_check(
@@ -102,6 +104,8 @@ async def ingest_source(
             dataset_id=dataset_id,
             schema_id=source.schema_id,
             file_bytes=payload_bytes,
+            source_id=source_id,
+            file_format=resolve_file_format(source_definition=source),
         )
 
     if source.source_type == "ui_stream":

topos_node-1.0.0/topos/api/privacy_disclose.py

@@ -0,0 +1,85 @@
+"""Platform Privacy Layer: batch PII disclosure via Topos Engine."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, List, Optional
+
+from fastapi import APIRouter, Body, Depends, HTTPException
+from pydantic import BaseModel, Field
+
+from ..auth import require_api_key
+from ..sanitization.privacy_filter import PRIVACY_DISCLOSE_MAX_BATCH, redact_privacy_batch
+from ..sanitization.nsfw_classifier import NSFW_CLASSIFY_MAX_BATCH, classify_nsfw_batch
+
+logger = logging.getLogger("topos.api.privacy_disclose")
+
+router = APIRouter(tags=["privacy"])
+
+
+class PrivacyDiscloseItem(BaseModel):
+    id: str = ""
+    text: str = ""
+    transform_id: Optional[str] = None
+
+
+class PrivacyDiscloseRequest(BaseModel):
+    items: List[PrivacyDiscloseItem] = Field(default_factory=list)
+    transform_id: str = "pii_redaction"
+
+
+@router.post("/v1/privacy/disclose", dependencies=[Depends(require_api_key)])
+async def post_privacy_disclose(body: PrivacyDiscloseRequest = Body(...)) -> dict[str, Any]:
+    if not body.items:
+        raise HTTPException(status_code=400, detail="items required")
+    if len(body.items) > PRIVACY_DISCLOSE_MAX_BATCH:
+        raise HTTPException(
+            status_code=413,
+            detail=f"batch exceeds limit of {PRIVACY_DISCLOSE_MAX_BATCH}",
+        )
+    payload_items = [
+        {
+            "id": item.id,
+            "text": item.text,
+            "transform_id": item.transform_id or body.transform_id,
+        }
+        for item in body.items
+    ]
+    try:
+        result = redact_privacy_batch(payload_items, transform_id=body.transform_id)
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("post_privacy_disclose failed: %s", exc)
+        raise HTTPException(status_code=500, detail="privacy disclose failed") from exc
+    status = str(result.get("status") or "ok")
+    if status == "unavailable":
+        raise HTTPException(status_code=503, detail=str(result.get("error") or "privacy filter unavailable"))
+    if status == "too_large":
+        raise HTTPException(status_code=413, detail=str(result.get("error") or "batch too large"))
+    return {"status": "ok", **result}
+
+
+class NsfwClassifyRequest(BaseModel):
+    items: List[PrivacyDiscloseItem] = Field(default_factory=list)
+
+
+@router.post("/v1/privacy/nsfw-classify", dependencies=[Depends(require_api_key)])
+async def post_nsfw_classify(body: NsfwClassifyRequest = Body(...)) -> dict[str, Any]:
+    if not body.items:
+        raise HTTPException(status_code=400, detail="items required")
+    if len(body.items) > NSFW_CLASSIFY_MAX_BATCH:
+        raise HTTPException(
+            status_code=413,
+            detail=f"batch exceeds limit of {NSFW_CLASSIFY_MAX_BATCH}",
+        )
+    payload_items = [{"id": item.id, "text": item.text} for item in body.items]
+    try:
+        result = classify_nsfw_batch(payload_items)
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("post_nsfw_classify failed: %s", exc)
+        raise HTTPException(status_code=500, detail="nsfw classify failed") from exc
+    status = str(result.get("status") or "ok")
+    if status == "disabled":
+        return {"status": "ok", **result}
+    if status == "too_large":
+        raise HTTPException(status_code=413, detail=str(result.get("error") or "batch too large"))
+    return {"status": "ok", **result}