topos-node 1.2.2__tar.gz → 1.2.4__tar.gz

{topos_node-1.2.2/topos_node.egg-info → topos_node-1.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: topos-node
-Version: 1.2.2
+Version: 1.2.4
 Summary: Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://topos.dialogues.ai

{topos_node-1.2.2 → topos_node-1.2.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "topos-node"
-version = "1.2.2"
+version = "1.2.4"
 description = "Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client"
 readme = "README.md"
 license = "Apache-2.0"

{topos_node-1.2.2 → topos_node-1.2.4}/topos/__version__.py

@@ -1,6 +1,6 @@
 """Topos Control Plane Version Information."""
 
-__version__ = "1.2.2"
-__version_info__ = (1, 2, 2)
+__version__ = "1.2.4"
+__version_info__ = (1, 2, 4)
 
 __all__ = ["__version__", "__version_info__"]

{topos_node-1.2.2 → topos_node-1.2.4}/topos/api/enrichment.py

@@ -319,7 +319,14 @@ def _enrichment_coverage_core(source_id: str) -> Dict[str, Any]:
             "output_rows": {},
         }
         coverage_table = coverage_table_for_job(job_id)
-        if coverage_table and _table_exists(conn, coverage_table):
+        if job_id == "timeline":
+            from ..features.timeline_projection import timeline_coverage_for_source
+
+            timeline_stats = timeline_coverage_for_source(conn, source_id)
+            job_info["enriched_records"] = timeline_stats["enriched_records"]
+            job_info["coverage_percent"] = timeline_stats["coverage_percent"]
+            job_info["output_rows"]["timeline"] = timeline_stats["enriched_records"]
+        elif coverage_table and _table_exists(conn, coverage_table):
             cols = _table_columns(conn, coverage_table)
             id_col = "message_id" if "message_id" in cols else "record_id"
             if "source_id" in cols and id_col in cols:
@@ -552,24 +559,50 @@ async def _generic_backfill_core(
     scanned = len(records)
 
     if only_missing:
-        coverage_table = coverage_table_for_job(job_name)
-        if coverage_table and _table_exists(db_conn, coverage_table):
-            cols = _table_columns(db_conn, coverage_table)
-            id_col = "message_id" if "message_id" in cols else "record_id"
-            if "source_id" in cols and id_col in cols:
-                done = {
-                    str(row[0])
-                    for row in db_conn.execute(
-                        f"SELECT DISTINCT {id_col} FROM {coverage_table} WHERE source_id=?",
-                        (source_id,),
-                    ).fetchall()
-                    if row[0]
+        if job_name == "timeline":
+            from ..features.timeline_projection import timeline_coverage_for_source
+
+            stats = timeline_coverage_for_source(db_conn, source_id)
+            if stats["missing_records"] == 0:
+                return {
+                    "rows_scanned": scanned,
+                    "rows_processed": 0,
+                    "rows_skipped": scanned,
+                    "rows_failed": 0,
+                    "errors": [],
                 }
-                records = [r for r in records if _record_identifier(r) not in done]
+        else:
+            coverage_table = coverage_table_for_job(job_name)
+            if coverage_table and _table_exists(db_conn, coverage_table):
+                cols = _table_columns(db_conn, coverage_table)
+                id_col = "message_id" if "message_id" in cols else "record_id"
+                if "source_id" in cols and id_col in cols:
+                    done = {
+                        str(row[0])
+                        for row in db_conn.execute(
+                            f"SELECT DISTINCT {id_col} FROM {coverage_table} WHERE source_id=?",
+                            (source_id,),
+                        ).fetchall()
+                        if row[0]
+                    }
+                    records = [r for r in records if _record_identifier(r) not in done]
 
     if isinstance(limit, int) and limit > 0:
         records = records[:limit]
 
+    if job_name == "timeline":
+        from ..features.timeline_projection import repair_timeline_for_source
+
+        report = repair_timeline_for_source(db_conn, source_id, missing_only=True, dry_run=False)
+        processed = int(report.get("totals", {}).get("written", 0))
+        return {
+            "rows_scanned": scanned,
+            "rows_processed": processed,
+            "rows_skipped": max(scanned - processed, 0),
+            "rows_failed": 0,
+            "errors": [],
+        }
+
     if not records:
         return {
             "rows_scanned": scanned,
@@ -950,6 +983,7 @@ async def _process_enrichment_core(
     job_names: Optional[List[str]] = None,
     force_reprocess: bool = False,
     include_signal: bool = True,
+    progress_updater: Optional[Any] = None,
 ) -> Dict[str, Any]:
     """Core logic for processing enrichment (reusable from HTTP and WebSocket).
     
@@ -1024,32 +1058,64 @@ async def _process_enrichment_core(
     
     # Define progress callback to update progress during execution
     progress_callback = None
-    try:
-        from ..enrichment.progress import get_progress
-        # Try to get progress object if it exists (created by handler)
-        progress_obj = get_progress(source_id)  # Use source_id as fallback lookup
-        if progress_obj:
-            def progress_callback(
-                processed_count: int, 
-                total_count: int, 
-                job_name: str, 
-                job_percent: float,
-                current_job_progress: float,
-            ):
-                """Update progress as jobs execute."""
-                estimated_messages_processed = int((job_percent / 100) * total_count)
-                jobs_complete = int((job_percent / 100) * len(jobs_to_run))
-                progress_obj.update(
-                    messages_processed=estimated_messages_processed,
-                    messages_skipped=0,
-                    current_job_name=job_name,
-                    current_job_progress_percent=current_job_progress,
-                    jobs_complete=jobs_complete,
-                    jobs_total=len(jobs_to_run),
-                )
-    except Exception:
-        pass  # Progress callback is optional
+    if progress_updater is not None:
+        def progress_callback(
+            processed_count: int,
+            total_count: int,
+            job_name: str,
+            job_percent: float,
+            current_job_progress: float,
+        ):
+            estimated_messages_processed = int((job_percent / 100) * total_count) if total_count else 0
+            jobs_complete = int((job_percent / 100) * len(jobs_to_run)) if jobs_to_run else 0
+            progress_updater(
+                {
+                    "status": "processing",
+                    "progress_percent": min(100.0, job_percent),
+                    "messages_processed": estimated_messages_processed,
+                    "messages_skipped": 0,
+                    "messages_total": total_count,
+                    "current_job_name": job_name,
+                    "current_job_progress_percent": current_job_progress,
+                    "jobs_complete": jobs_complete,
+                    "jobs_total": len(jobs_to_run),
+                    "jobs_progress_percent": (jobs_complete / len(jobs_to_run) * 100) if jobs_to_run else 0.0,
+                }
+            )
+    else:
+        try:
+            from ..enrichment.progress import get_progress
+            progress_obj = get_progress(source_id)
+            if progress_obj:
+                def progress_callback(
+                    processed_count: int,
+                    total_count: int,
+                    job_name: str,
+                    job_percent: float,
+                    current_job_progress: float,
+                ):
+                    estimated_messages_processed = int((job_percent / 100) * total_count)
+                    jobs_complete = int((job_percent / 100) * len(jobs_to_run))
+                    progress_obj.update(
+                        messages_processed=estimated_messages_processed,
+                        messages_skipped=0,
+                        current_job_name=job_name,
+                        current_job_progress_percent=current_job_progress,
+                        jobs_complete=jobs_complete,
+                        jobs_total=len(jobs_to_run),
+                    )
+        except Exception:
+            pass
     
+    if progress_updater is not None:
+        progress_updater(
+            {
+                "status": "processing",
+                "messages_total": len(unprocessed_messages),
+                "jobs_total": len(jobs_to_run),
+            }
+        )
+
     enrichment_result = await orchestrator.run_canonical(
         unprocessed_messages,
         job_names=jobs_to_run,

{topos_node-1.2.2 → topos_node-1.2.4}/topos/api/source_install.py

@@ -11,6 +11,7 @@ from ..ingestion.ingest_helpers import ingest_file_payload, ingest_ui_payload
 from ..api.enrichment import _process_enrichment_core
 from ..engine.usage_observation import emit_usage_observation
 from ..sources import install_service
+from ..sources.definitions import with_source_capabilities
 
 router = APIRouter()
 logger = logging.getLogger("topos.api.source_install")
@@ -140,7 +141,7 @@ async def _list_sources_core(payload: Dict[str, Any]) -> Dict[str, Any]:
         source_def = rec.source_definition_json if isinstance(rec.source_definition_json, dict) else {}
         if not sid or sid in active_by_source or not source_def:
             continue
-        active_by_source[sid] = source_def
+        active_by_source[sid] = with_source_capabilities(source_def)
     sources = list(active_by_source.values())
     return {"status": "ok", "sources": sources}
 

{topos_node-1.2.2 → topos_node-1.2.4}/topos/api/ui_config.py

@@ -43,6 +43,7 @@ def _normalize_ui_config(value: Any) -> dict[str, Any]:
     # Graph view prefs: the temporal window is personal (data volume/velocity
     # dependent — 2 days for some owners, 2 months for others). timeWindowDays
     # is a whole-day count; explicit null = all time; absent = unset.
+    # trackLookbackDays caps how far left the scrubber track extends.
     graph = value.get("graph")
     if isinstance(graph, dict) and "timeWindowDays" in graph:
         window = graph.get("timeWindowDays")
@@ -52,6 +53,14 @@ def _normalize_ui_config(value: Any) -> dict[str, Any]:
             days = int(window)
             if 1 <= days <= GRAPH_WINDOW_MAX_DAYS:
                 out["graph"]["timeWindowDays"] = days
+    if isinstance(graph, dict) and "trackLookbackDays" in graph:
+        lookback = graph.get("trackLookbackDays")
+        if lookback is None:
+            out["graph"]["trackLookbackDays"] = None
+        elif isinstance(lookback, (int, float)) and not isinstance(lookback, bool):
+            days = int(lookback)
+            if 1 <= days <= GRAPH_WINDOW_MAX_DAYS:
+                out["graph"]["trackLookbackDays"] = days
     if isinstance(graph, dict):
         mode = graph.get("nodeColorMode")
         if mode in ("type", "community"):

{topos_node-1.2.2 → topos_node-1.2.4}/topos/app.py

@@ -172,6 +172,20 @@ async def startup_event() -> None:
             _start_upgrades(_upgrade_conn)
     except Exception as e:
         logger.warning("Upgrade runner at startup failed (non-fatal): %s", e)
+    try:
+        from .core.state import get_db_connection as _get_conn_for_pipeline
+        from .pipeline.job_runner import recover_pipeline_jobs, start_pipeline_worker
+        from .features.entities.graph_refresh import reconcile_graph_on_startup
+
+        _pipeline_conn = state.db_conn if state.db_conn is not None else _get_conn_for_pipeline()
+        if _pipeline_conn is not None:
+            recovered = recover_pipeline_jobs(_pipeline_conn)
+            if recovered:
+                logger.info("Recovered stale pipeline jobs on startup: count=%s", recovered)
+            reconcile_graph_on_startup(_pipeline_conn)
+            start_pipeline_worker(_get_conn_for_pipeline)
+    except Exception as e:
+        logger.warning("Pipeline worker at startup failed (non-fatal): %s", e)
     try:
         from .sources import install_service
 

topos_node-1.2.4/topos/canonicalization/mappers/base.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict, List, Optional
+
+from ...ingestion.parsers.base import NormalizedRecord
+
+
+@dataclass(frozen=True)
+class CanonicalRecord:
+    record_id: str
+    payload: Dict[str, str]
+    # Explicit canonical target table for dual-lane mappers. None means "the
+    # default table for the source's canonical_group_id" (existing behavior);
+    # single-lane mappers never need to set it.
+    table: Optional[str] = None
+
+
+@dataclass(frozen=True)
+class MappingMetadata:
+    source_id: str
+    mapping_version: str
+
+
+class CanonicalMapper:
+    def map(self, normalized: NormalizedRecord) -> CanonicalRecord:
+        raise NotImplementedError
+
+    def map_many(self, normalized: NormalizedRecord) -> List[CanonicalRecord]:
+        """Multi-lane hook: one normalized record → one or more canonical rows,
+        each optionally targeting a different canonical table via
+        ``CanonicalRecord.table``. Default delegates to :meth:`map`, so
+        single-lane mappers keep working unchanged."""
+        return [self.map(normalized)]
+
+    def mapping_metadata(self, normalized: NormalizedRecord) -> MappingMetadata:
+        raise NotImplementedError

{topos_node-1.2.2 → topos_node-1.2.4}/topos/canonicalization/mappers/github_activity_mapper.py

@@ -1,9 +1,16 @@
-"""GitHub activity events → wiki activity_events canonical mapper."""
+"""GitHub activity events → dual-lane canonical mapper.
+
+Every event maps to one ``activity_events`` row (single-lane behavior kept).
+Additionally, each commit inside a PushEvent ``payload.commits[]`` maps to one
+``journal_entries`` row — a commit reads like a work-journal entry (category
+'code', content = commit message). Only PushEvent commits fan out; PRs/issues
+stay activity-only.
+"""
 
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 
 from ...ingestion.parsers.base import NormalizedRecord
 from .base import CanonicalMapper, CanonicalRecord, MappingMetadata
@@ -82,9 +89,28 @@ def _title(repo_name: Optional[str], event_type: str, activity_type: str, event_
     return f"{prefix}: {activity_type.replace('_', ' ')}"
 
 
+def _commit_timestamp(commit: Dict[str, Any]) -> Optional[str]:
+    """Commit timestamp if the payload carries one (REST events API usually
+    omits it — webhook-shaped payloads have ``timestamp``, some mirrors nest an
+    author date). None → caller falls back to the event created_at."""
+    for candidate in (commit.get("timestamp"), commit.get("date")):
+        if str(candidate or "").strip():
+            return str(candidate)
+    author = commit.get("author")
+    if isinstance(author, dict) and str(author.get("date") or "").strip():
+        return str(author["date"])
+    return None
+
+
+def _branch_from_ref(ref: Optional[str]) -> Optional[str]:
+    if ref and ref.startswith("refs/heads/"):
+        return ref[len("refs/heads/") :]
+    return None
+
+
 @dataclass
 class GithubActivityCanonicalMapper(CanonicalMapper):
-    version: str = "v1"
+    version: str = "v2"
 
     def map(self, normalized: NormalizedRecord) -> CanonicalRecord:
         payload = normalized.payload
@@ -108,5 +134,60 @@ class GithubActivityCanonicalMapper(CanonicalMapper):
         }
         return CanonicalRecord(record_id=canonical["event_id"], payload=canonical)
 
+    def map_many(self, normalized: NormalizedRecord) -> List[CanonicalRecord]:
+        """Dual lane: the activity_events row (always) + one journal_entries
+        row per PushEvent commit. Deterministic ids keep re-ingest idempotent:
+        entry_id 'github:{repo}:{sha}', source_record_id '{event_id}:{sha}'."""
+        records = [self.map(normalized)]
+        payload = normalized.payload
+        if str(payload.get("type") or "") != "PushEvent":
+            return records
+
+        event_record_id = str(payload.get("id") or normalized.record_id)
+        created_at = payload.get("created_at") or payload.get("occurred_at")
+        repo = payload.get("repo")
+        repo_name = str(repo.get("name")) if isinstance(repo, dict) and repo.get("name") else None
+        event_payload = payload.get("payload") if isinstance(payload.get("payload"), dict) else {}
+        ref = str(event_payload.get("ref") or "").strip() or None
+        branch = _branch_from_ref(ref)
+        commits = event_payload.get("commits")
+        if not isinstance(commits, list):
+            return records
+
+        for commit in commits:
+            if not isinstance(commit, dict):
+                continue
+            sha = str(commit.get("sha") or "").strip()
+            if not sha:
+                continue  # no deterministic identity without a sha
+            entry_id = f"github:{repo_name}:{sha}" if repo_name else f"github:{sha}"
+            entry_at = _commit_timestamp(commit) or created_at
+            metadata: Dict[str, Any] = {
+                "repo": repo_name,
+                "sha": sha,
+                "ref": ref,
+                "branch": branch,
+                "event_id": f"github:{event_record_id}",
+            }
+            journal = {
+                "entry_id": entry_id,
+                "entry_at": entry_at,
+                # A commit is a point-in-time entry: no invented duration/start.
+                "starts_at": None,
+                "ends_at": entry_at,
+                "duration": None,
+                "mood_tag": None,
+                "category": "code",
+                "content": str(commit.get("message") or ""),
+                "people": None,
+                "place_name": None,
+                "source_record_id": f"{event_record_id}:{sha}",
+                "metadata_json": {k: v for k, v in metadata.items() if v is not None},
+            }
+            records.append(
+                CanonicalRecord(record_id=entry_id, payload=journal, table="journal_entries")
+            )
+        return records
+
     def mapping_metadata(self, normalized: NormalizedRecord) -> MappingMetadata:
         return MappingMetadata(source_id="github_activity", mapping_version=self.version)

{topos_node-1.2.2 → topos_node-1.2.4}/topos/config/settings.py

@@ -30,7 +30,7 @@ class Settings(BaseSettings):
 
     environment: str = Field("development", env="ENVIRONMENT")
     log_format: Optional[str] = Field(None, env="LOG_FORMAT")
-    log_level: str = Field("DEBUG", env="LOG_LEVEL")
+    log_level: str = Field("INFO", env="LOG_LEVEL")
 
     openai_base_url: str = Field("https://api.openai.com/v1", env="OPENAI_BASE_URL")
     openai_model: str = Field("gpt-4o-mini", env="OPENAI_MODEL")
@@ -48,7 +48,12 @@ class Settings(BaseSettings):
         "https://api.nodes.griptape.ai", env="GRIPTAPE_NODES_API_BASE_URL"
     )
 
-    allowed_origins_raw: str = Field("http://localhost:3000", env="ALLOWED_ORIGINS")
+    allowed_origins_raw: str = Field(
+        "http://localhost:3000",
+        # pydantic v2 ignores Field(env=); without the alias this field only
+        # read ALLOWED_ORIGINS_RAW and the documented ALLOWED_ORIGINS was dead.
+        validation_alias=AliasChoices("ALLOWED_ORIGINS", "ALLOWED_ORIGINS_RAW"),
+    )
     allowed_origin_regex: Optional[str] = Field(None, env="ALLOWED_ORIGIN_REGEX")
     enable_health_auth: bool = Field(False, env="ENABLE_HEALTH_AUTH")
 

{topos_node-1.2.2 → topos_node-1.2.4}/topos/control_plane_client.py

@@ -6,6 +6,7 @@ from collections import deque
 import json
 import logging
 import ssl
+import threading
 from typing import Any, Awaitable, Callable, Dict
 
 import certifi
@@ -81,6 +82,9 @@ class ControlPlaneClient:
         self._presence_outbox_size = max(1, int(settings.control_plane_presence_outbox_size))
         self._presence_outbox: deque[dict[str, Any]] = deque(maxlen=self._presence_outbox_size)
         self._outbox_lock = asyncio.Lock()
+        # Thread-safe outbox for sync callers (Engine.run in worker threads via to_thread).
+        self._sync_outbox: deque[dict[str, Any]] = deque(maxlen=self._presence_outbox_size)
+        self._sync_outbox_lock = threading.Lock()
 
         self._backoff = ExponentialBackoff(
             ResilienceConfig(
@@ -290,6 +294,7 @@ class ControlPlaneClient:
             await self._handle_message(ws, data)
 
     async def _flush_presence_outbox(self) -> None:
+        await self._drain_sync_outbox_into_presence()
         async with self._outbox_lock:
             pending = list(self._presence_outbox)
             self._presence_outbox.clear()
@@ -301,6 +306,13 @@ class ControlPlaneClient:
                 await self._enqueue_presence_message(message)
                 break
 
+    async def _drain_sync_outbox_into_presence(self) -> None:
+        with self._sync_outbox_lock:
+            pending = list(self._sync_outbox)
+            self._sync_outbox.clear()
+        for message in pending:
+            await self._enqueue_presence_message(message)
+
     async def _enqueue_presence_message(self, message: Dict[str, Any]) -> None:
         async with self._outbox_lock:
             at_capacity = len(self._presence_outbox) >= self._presence_outbox_size
@@ -312,6 +324,36 @@ class ControlPlaneClient:
                 )
             self._presence_outbox.append(dict(message))
 
+    def enqueue_unsolicited_message_threadsafe(self, message: Dict[str, Any]) -> None:
+        """Queue an unsolicited WS message from a sync/worker thread.
+
+        Enrichment runs Engine.run via asyncio.to_thread; those threads cannot safely
+        touch the websockets connection. Queue here and flush on the client's loop.
+        """
+        with self._sync_outbox_lock:
+            at_capacity = len(self._sync_outbox) >= self._presence_outbox_size
+            if at_capacity:
+                dropped = self._sync_outbox.popleft()
+                logger.warning(
+                    "Sync outbox full; dropping oldest message type=%s",
+                    dropped.get("type"),
+                )
+            self._sync_outbox.append(dict(message))
+
+        client_task = self._task
+        if client_task is None:
+            return
+        try:
+            loop = client_task.get_loop()
+        except Exception:
+            return
+        if not loop.is_running():
+            return
+        try:
+            asyncio.run_coroutine_threadsafe(self._flush_presence_outbox(), loop)
+        except Exception:
+            logger.debug("Failed to schedule sync outbox flush", exc_info=True)
+
     async def _send_ws_json(self, ws, payload: Dict[str, Any]) -> bool:
         if not ws:
             return False