topos-node 1.3.49__tar.gz → 1.3.51__tar.gz

{topos_node-1.3.49/topos_node.egg-info → topos_node-1.3.51}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: topos-node
-Version: 1.3.49
+Version: 1.3.51
 Summary: Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://topos.dialogues.ai

{topos_node-1.3.49 → topos_node-1.3.51}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "topos-node"
-version = "1.3.49"
+version = "1.3.51"
 description = "Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client"
 readme = "README.md"
 license = "Apache-2.0"

{topos_node-1.3.49 → topos_node-1.3.51}/topos/__version__.py

@@ -1,6 +1,6 @@
 """Topos Control Plane Version Information."""
 
-__version__ = "1.3.49"
-__version_info__ = (1, 3, 49)
+__version__ = "1.3.51"
+__version_info__ = (1, 3, 51)
 
 __all__ = ["__version__", "__version_info__"]

{topos_node-1.3.49 → topos_node-1.3.51}/topos/api/enrichment.py

@@ -53,6 +53,7 @@ _CANONICAL_TABLE_BY_GROUP: Dict[str, tuple[str, str]] = {
     "financial": ("financial_transactions", "transaction_id"),
     "places": ("location_events", "event_id"),
     "contacts": ("contacts", "contact_id"),
+    "transcripts": ("transcript_segments", "segment_id"),
 }
 
 # Marker helpers live in `enrichment/record_progress.py` so BOTH lanes share one

{topos_node-1.3.49 → topos_node-1.3.51}/topos/canonicalization/mappers/__init__.py

@@ -18,6 +18,7 @@ from .github_activity_mapper import GithubActivityCanonicalMapper
 from .google_calendar_mapper import GoogleCalendarMapper
 from .grok_mapper import GrokCanonicalMapper
 from .messenger_mapper import ImessageCanonicalMapper, SignalCanonicalMapper, VoxtermTranscriptCanonicalMapper
+from .transcript_mapper import TranscriptCanonicalMapper
 
 MAPPER_REGISTRY = {
     "chatgpt": ChatGPTCanonicalMapper,
@@ -37,6 +38,7 @@ MAPPER_REGISTRY = {
     "journal_time_log": JournalTimeLogMapper,
     "documents": DocumentsCanonicalMapper,
     "google_calendar": GoogleCalendarMapper,
+    "transcript": TranscriptCanonicalMapper,
 }
 
 __all__ = [
@@ -49,5 +51,6 @@ __all__ = [
     "GrokCanonicalMapper",
     "ImessageCanonicalMapper",
     "SignalCanonicalMapper",
+    "TranscriptCanonicalMapper",
     "MAPPER_REGISTRY",
 ]

topos_node-1.3.51/topos/canonicalization/mappers/transcript_mapper.py

@@ -0,0 +1,204 @@
+"""Session-shaped transcript → transcripts / speakers / segments.
+
+``map_many`` fans one normalized session into a header row, optional roster
+rows from owner-supplied ``participants``, and one segment per caption item.
+
+Attribution is fail-closed:
+- ``participation_mode`` is always ``ambient``
+- every segment is ``actor_role=ambient``, ``is_from_self=0``
+- roster names never set ``is_owner`` or ``contact_id``
+- unlabeled captions stay unlabeled (roster is a prior, not diarization)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from typing import Any, Dict, List, Optional
+
+from ...ingestion.parsers.base import NormalizedRecord
+from .base import CanonicalMapper, CanonicalRecord, MappingMetadata
+
+_AMBIENT = "ambient"
+
+
+def _parse_started_at(raw: Any) -> Optional[datetime]:
+    text = str(raw or "").strip()
+    if not text:
+        return None
+    if text.endswith("Z"):
+        text = text[:-1] + "+00:00"
+    try:
+        dt = datetime.fromisoformat(text)
+    except ValueError:
+        return None
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    return dt
+
+
+def _event_at(started_at: Optional[datetime], start_sec: Any) -> Optional[str]:
+    if started_at is None:
+        return None
+    try:
+        offset = float(start_sec or 0)
+    except (TypeError, ValueError):
+        offset = 0.0
+    return (started_at + timedelta(seconds=offset)).isoformat()
+
+
+def _item_text(item: Dict[str, Any]) -> str:
+    return str(item.get("text") or item.get("content") or item.get("caption") or "").strip()
+
+
+def _item_speaker_label(item: Dict[str, Any]) -> str:
+    return str(
+        item.get("speaker_label") or item.get("speaker") or item.get("speaker_id") or ""
+    ).strip()
+
+
+@dataclass
+class TranscriptCanonicalMapper(CanonicalMapper):
+    version: str = "v1"
+
+    def map(self, normalized: NormalizedRecord) -> CanonicalRecord:
+        header = self._header(normalized)
+        return CanonicalRecord(
+            record_id=str(header["transcript_id"]),
+            payload=header,
+            table="transcripts",
+        )
+
+    def map_many(self, normalized: NormalizedRecord) -> List[CanonicalRecord]:
+        p = normalized.payload if isinstance(normalized.payload, dict) else {}
+        header = self._header(normalized)
+        transcript_id = str(header["transcript_id"])
+        dataset_id = p.get("dataset_id")
+        started = _parse_started_at(header.get("started_at"))
+        out: List[CanonicalRecord] = [
+            CanonicalRecord(record_id=transcript_id, payload=header, table="transcripts")
+        ]
+
+        label_to_speaker: Dict[str, str] = {}
+        participants = p.get("participants") if isinstance(p.get("participants"), list) else []
+        for index, person in enumerate(participants):
+            if not isinstance(person, dict):
+                continue
+            name = str(person.get("name") or "").strip()
+            if not name:
+                continue
+            speaker_id = f"{transcript_id}:roster:{index}"
+            out.append(
+                CanonicalRecord(
+                    record_id=speaker_id,
+                    payload={
+                        "speaker_id": speaker_id,
+                        "transcript_id": transcript_id,
+                        "dataset_id": dataset_id,
+                        "label": name,
+                        "display_name": name,
+                        "contact_id": None,
+                        "is_owner": 0,
+                        "attribution_source": "owner_roster",
+                        "attribution_confidence": 1.0,
+                        "source_record_id": speaker_id,
+                    },
+                    table="transcript_speakers",
+                )
+            )
+
+        items = p.get("items") if isinstance(p.get("items"), list) else []
+        for index, item in enumerate(items):
+            if not isinstance(item, dict):
+                continue
+            text = _item_text(item)
+            if not text:
+                continue
+            label = _item_speaker_label(item)
+            speaker_id = None
+            if label:
+                speaker_id = label_to_speaker.get(label)
+                if speaker_id is None:
+                    speaker_id = f"{transcript_id}:label:{len(label_to_speaker)}"
+                    label_to_speaker[label] = speaker_id
+                    out.append(
+                        CanonicalRecord(
+                            record_id=speaker_id,
+                            payload={
+                                "speaker_id": speaker_id,
+                                "transcript_id": transcript_id,
+                                "dataset_id": dataset_id,
+                                "label": label,
+                                "display_name": None,
+                                "contact_id": None,
+                                "is_owner": 0,
+                                "attribution_source": "source_label",
+                                "attribution_confidence": 0.0,
+                                "source_record_id": speaker_id,
+                            },
+                            table="transcript_speakers",
+                        )
+                    )
+            try:
+                start_sec = float(item.get("start") if item.get("start") is not None else item.get("start_sec") or 0)
+            except (TypeError, ValueError):
+                start_sec = 0.0
+            try:
+                duration_sec = item.get("duration")
+                if duration_sec is None:
+                    duration_sec = item.get("duration_sec")
+                duration_sec = float(duration_sec) if duration_sec is not None else None
+            except (TypeError, ValueError):
+                duration_sec = None
+            segment_id = str(item.get("segment_id") or f"{transcript_id}:{index}")
+            asr_confidence = item.get("asr_confidence")
+            try:
+                asr_confidence = float(asr_confidence) if asr_confidence is not None else None
+            except (TypeError, ValueError):
+                asr_confidence = None
+            out.append(
+                CanonicalRecord(
+                    record_id=segment_id,
+                    payload={
+                        "segment_id": segment_id,
+                        "transcript_id": transcript_id,
+                        "dataset_id": dataset_id,
+                        "speaker_id": speaker_id,
+                        "speaker_label": label or None,
+                        "content": text,
+                        "start_sec": start_sec,
+                        "duration_sec": duration_sec,
+                        "event_at": _event_at(started, start_sec),
+                        "actor_role": _AMBIENT,
+                        "is_from_self": 0,
+                        "asr_confidence": asr_confidence,
+                        "source_record_id": segment_id,
+                    },
+                    table="transcript_segments",
+                )
+            )
+        return out
+
+    def mapping_metadata(self, normalized: NormalizedRecord) -> MappingMetadata:
+        return MappingMetadata(source_id="transcript", mapping_version=self.version)
+
+    def _header(self, normalized: NormalizedRecord) -> Dict[str, Any]:
+        p = normalized.payload if isinstance(normalized.payload, dict) else {}
+        transcript_id = str(p.get("transcript_id") or normalized.record_id)
+        return {
+            "transcript_id": transcript_id,
+            "dataset_id": p.get("dataset_id"),
+            "title": p.get("title"),
+            "origin_url": p.get("origin_url"),
+            "origin_kind": p.get("origin_kind") or "other",
+            "started_at": p.get("started_at"),
+            "ended_at": p.get("ended_at"),
+            "duration_sec": p.get("duration_sec"),
+            "language_code": p.get("language_code"),
+            "asr_model": p.get("asr_model"),
+            "asr_quality": p.get("asr_quality") or "unknown",
+            "is_generated": p.get("is_generated"),
+            "media_ref": p.get("media_ref"),
+            "participation_mode": _AMBIENT,
+            "source_record_id": transcript_id,
+        }

{topos_node-1.3.49 → topos_node-1.3.51}/topos/data_explorer_tables.py

@@ -19,6 +19,9 @@ CANONICAL_SCHEMA_TABLES: frozenset[str] = frozenset(
         "financial_transactions",
         "location_events",
         "documents",
+        "transcripts",
+        "transcript_speakers",
+        "transcript_segments",
     }
 )
 
@@ -50,6 +53,9 @@ CANONICAL_ROW_ID_COLUMN: dict[str, str] = {
     "financial_transactions": "transaction_id",
     "location_events": "event_id",
     "documents": "doc_id",
+    "transcripts": "transcript_id",
+    "transcript_speakers": "speaker_id",
+    "transcript_segments": "segment_id",
 }
 
 

{topos_node-1.3.49 → topos_node-1.3.51}/topos/engine/engine.py

@@ -270,6 +270,12 @@ class Engine:
                         "completion_tokens": completion_tokens,
                         "total_tokens": total_tokens,
                     },
+                    # Rebuilding usage from three keys DROPPED the provider's
+                    # load/prefill/decode split, so it reached the browser but
+                    # never the database — leaving the per-model latency table
+                    # empty. Same flatten-and-discard shape as the control
+                    # plane's parse_usage_tokens, missed on this path.
+                    timings=(usage.get("timings") if isinstance(usage.get("timings"), dict) else None),
                     origin=origin,
                     source_id=normalized.source_id,
                     duration_ms=duration_ms,

{topos_node-1.3.49 → topos_node-1.3.51}/topos/engine/usage_observation.py

@@ -304,6 +304,21 @@ def emit_usage_observation_sync(
     return envelope
 
 
+#: The phase keys lifted into usage metadata. A closed set, mirroring the
+#: control plane's TIMING_KEYS: metadata_json is free-form, and an open
+#: passthrough would let a future provider field become a billing input.
+_TIMING_KEYS = (
+    "load_ms",
+    "prefill_ms",
+    "decode_ms",
+    "total_ms",
+    "prefill_tok_s",
+    "decode_tok_s",
+    "cold",
+    "prefill_cached",
+)
+
+
 def emit_engine_llm_usage_observation(
     *,
     task_id: str,
@@ -318,6 +333,7 @@ def emit_engine_llm_usage_observation(
     ttfb_ms: Optional[int] = None,
     pack_id: Optional[str] = None,
     role: Optional[str] = None,
+    timings: Optional[Dict[str, Any]] = None,
 ) -> Optional[Dict[str, Any]]:
     """Record + emit llm.generate usage for a completed generative Engine task.
 
@@ -380,6 +396,11 @@ def emit_engine_llm_usage_observation(
     # number is one you cannot.
     resolved_duration = int(duration_ms) if duration_ms is not None else None
     resolved_ttfb = int(ttfb_ms) if ttfb_ms is not None else None
+    # The load/prefill/decode split, if the provider reported one. This is the
+    # WRITE path for the per-model latency table: the engine records these rows
+    # directly, so a split that only reaches the browser never reaches the
+    # aggregation. Absent for providers that report nothing — never zero-filled.
+    phase_meta = {k: timings[k] for k in _TIMING_KEYS if isinstance(timings, dict) and k in timings}
     metadata: Dict[str, Any] = {
         "purpose": purpose,
         "provider": provider_key,
@@ -391,6 +412,7 @@ def emit_engine_llm_usage_observation(
         "subtype": subtype_key,
         "task_type": str(task_type or ""),
         "source_id": str(source_id or ""),
+        **phase_meta,
     }
     if resolved_pack_id:
         metadata["pack_id"] = resolved_pack_id

{topos_node-1.3.49 → topos_node-1.3.51}/topos/enrichment/jobs/canonical/brief_fallback.py

@@ -245,6 +245,7 @@ def prepare_signal_record(record: Dict[str, Any]) -> Dict[str, Any]:
     out = dict(record)
     record_id = (
         out.get("message_id")
+        or out.get("segment_id")
         or out.get("entry_id")
         or out.get("record_id")
         or out.get("transaction_id")

{topos_node-1.3.49 → topos_node-1.3.51}/topos/enrichment/jobs/canonical/derivation_job.py

@@ -322,6 +322,9 @@ def _iter_history(conn, limit=2000):
             f" ORDER BY event_at DESC LIMIT {int(limit)}"):
         out.append({"table": tbl, "record_id": rid, "text": (text or "")[:6000],
                     "date": str(at or "")[:10], "role": role or "observed", "source_id": ""})
+    # Ambient captions are not taste/habit evidence. any_with_label catchup
+    # must not mint owner-subject facts from overheard speech. A gated
+    # overheard reader is the only path that may surface captions.
     return out
 
 
@@ -424,6 +427,10 @@ def _self_gate(conn, *, all_packs, enabled, filters, llm, judge, model, vmodel,
             """SELECT 'journal_entries', entry_id, content, entry_at, 'authored'
                FROM journal_entries WHERE content IS NOT NULL AND LENGTH(content)>15
                AND entry_at >= datetime('now','-14 day') ORDER BY entry_at DESC LIMIT 100""")
+        recent += _rows(
+            """SELECT 'transcript_segments', segment_id, content, event_at, actor_role
+               FROM transcript_segments WHERE content IS NOT NULL AND LENGTH(content)>15
+               AND event_at >= datetime('now','-14 day') ORDER BY event_at DESC LIMIT 200""")
         for tbl, rid, text, at, role in recent:
             role = role or "observed"
             if role in pack.allowed_roles() and filters[pid].passes(text):

{topos_node-1.3.49 → topos_node-1.3.51}/topos/enrichment/jobs/canonical/embeddings_job.py

@@ -42,6 +42,8 @@ def _record_type(msg: Dict[str, Any]) -> Optional[str]:
         "conversation_messages": "conversation_message",
         "profile_records": "profile_record",
         "calendar_events": "calendar_event",
+        "transcript_segments": "transcript_segment",
+        "transcripts": "transcript_segment",
     }
     if table in mapping:
         return mapping[table]
@@ -93,8 +95,13 @@ class EmbeddingsJob(BaseEnrichmentJob):
         # vector row or it is unreachable through retrieval.
         seen_activity_hashes: set = set()
         for msg in canonical_messages:
-            message_id = msg.get("message_id") or msg.get("id")
-            record_id = msg.get("event_id") or msg.get("record_id") or message_id
+            message_id = msg.get("message_id") or msg.get("segment_id") or msg.get("id")
+            record_id = (
+                msg.get("event_id")
+                or msg.get("record_id")
+                or msg.get("segment_id")
+                or message_id
+            )
             content = embeddable_content(msg)
             if not record_id or not content or str(record_id) in excluded:
                 processed += 1

{topos_node-1.3.49 → topos_node-1.3.51}/topos/enrichment/jobs/canonical/entities_job.py

@@ -19,7 +19,15 @@ _MIN_RESOLVE_CONFIDENCE = 0.60
 # activity events: event_id/occurred_at, journal entries: entry_id/entry_at, …).
 # One key contract here, or whole sources silently skip extraction — the
 # 2026-07-14 backfill lost browser_visits/grow_* exactly this way.
-_RECORD_ID_FIELDS = ("message_id", "id", "record_id", "event_id", "entry_id", "transaction_id")
+_RECORD_ID_FIELDS = (
+    "message_id",
+    "segment_id",
+    "id",
+    "record_id",
+    "event_id",
+    "entry_id",
+    "transaction_id",
+)
 _EVENT_AT_FIELDS = ("event_at", "ts", "occurred_at", "entry_at", "starts_at", "created_at")
 
 

{topos_node-1.3.49 → topos_node-1.3.51}/topos/enrichment/jobs/canonical/relationship_edges_job.py

@@ -26,6 +26,12 @@ class RelationshipEdgesJob(BaseEnrichmentJob):
         pairs: Counter[Tuple[str, str]] = Counter()
         source_id = None
         for msg in canonical_messages:
+            table = str(msg.get("_table") or msg.get("canonical_table") or "")
+            role = str(msg.get("actor_role") or "").strip().lower()
+            if table in {"transcript_segments", "transcripts", "transcript_speakers"}:
+                continue
+            if role == "ambient":
+                continue
             source_id = msg.get("source_id") or source_id
             conv = str(msg.get("conversation_id") or msg.get("thread_id") or "default")
             sender = str(msg.get("sender_id") or msg.get("contact_id") or msg.get("from") or "").strip()

{topos_node-1.3.49 → topos_node-1.3.51}/topos/features/complexity/projection.py

@@ -35,6 +35,7 @@ SOURCE_INTENT: Dict[str, float] = {
     "time_log": 0.9,
     "github_activity": 0.9,
     "voxterm_transcripts": 0.8,
+    "youtube_transcripts": 0.2,
     "chatgpt": 0.7,
     "chatgpt_ui_conversation": 0.7,
     "chatgpt_file_ingestion": 0.7,
@@ -50,6 +51,7 @@ TABLE_INTENT: Dict[str, float] = {
     "conversation_messages": 0.5,
     "location_events": 0.3,
     "activity_events": 0.3,
+    "transcript_segments": 0.2,
 }
 
 DEFAULT_INTENT = 0.4