topos-node 0.1.0__py3-none-any.whl

topos/ingestion/parsers/chatgpt_parser.py

@@ -0,0 +1,67 @@
+"""ChatGPT parser for ingestion layer."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import Any, Dict
+
+from ..log_preview import field_preview
+from ..sources.base import RawRecord
+from ..validation.base import ValidationResult
+from ..validation.schema_registry import validate_schema
+from .base import NormalizedRecord, Parser
+
+logger = logging.getLogger("topos.ingestion.parser.chatgpt")
+
+
+@dataclass
+class ChatGPTParser(Parser):
+    dataset_id: str
+    _schema_id: str = "chatgpt.conversation.v1"  # Default to v1, can be overridden
+
+    def parse(self, raw: RawRecord) -> NormalizedRecord:
+        payload = raw.payload
+        role = payload.get("role", "").lower()
+        sender_type = "human" if role == "user" else "assistant"
+        created_at = payload.get("created_at")
+        ts = ""
+        if isinstance(created_at, (int, float)):
+            from datetime import datetime, timezone
+
+            ts = datetime.fromtimestamp(created_at, tz=timezone.utc).isoformat()
+        elif isinstance(created_at, str):
+            ts = created_at
+        normalized = {
+            "message_id": payload.get("id", raw.record_id),
+            "dataset_id": self.dataset_id,
+            "thread_id": payload.get("thread_id", ""),
+            "ts": ts,
+            "sender_type": sender_type,
+            "content": payload.get("content", ""),
+        }
+        # Preserve _metadata if present (for conversation tree reconstruction)
+        if "_metadata" in payload:
+            normalized["_metadata"] = payload["_metadata"]
+        logger.debug(
+            "[PIPELINE:PARSER] Parsed record: message_id=%s, sender_type=%s, thread_id=%s, content_preview=%s",
+            normalized["message_id"],
+            normalized["sender_type"],
+            normalized["thread_id"],
+            field_preview(normalized.get("content")),
+        )
+        return NormalizedRecord(record_id=normalized["message_id"], payload=normalized)
+
+    def validate(self, record: RawRecord) -> ValidationResult:
+        is_valid, error = validate_schema(record.payload, self.schema_id())
+        errors = [] if is_valid else [error or "Invalid record"]
+        logger.debug(
+            "[PIPELINE:PARSER] Validation result: record_id=%s, is_valid=%s, errors=%s",
+            record.record_id,
+            is_valid,
+            errors,
+        )
+        return ValidationResult(is_valid=is_valid, errors=errors, metadata={})
+
+    def schema_id(self) -> str:
+        return self._schema_id

topos/ingestion/parsers/grok_parser.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from ..sources.base import RawRecord
+from ..validation.base import ValidationResult
+from .base import NormalizedRecord, Parser
+
+
+@dataclass
+class GrokParser(Parser):
+    dataset_id: str
+
+    def parse(self, raw: RawRecord) -> NormalizedRecord:
+        return NormalizedRecord(record_id=raw.record_id, payload=raw.payload)
+
+    def validate(self, record: RawRecord) -> ValidationResult:
+        return ValidationResult(is_valid=True, errors=[], metadata={})
+
+    def schema_id(self) -> str:
+        return "grok.conversation.v1"

topos/ingestion/parsers/messenger_parser.py

@@ -0,0 +1,97 @@
+"""Minimal parsers for messenger ingestion (iMessage, Signal).
+
+Maps raw dict/row to normalized chat shape (message_id, thread_id, sender_type, content, ts).
+Full implementation (reading from chat.db / Signal DB) is in Sprints 03 and 04.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import Any, Dict
+
+from ..sources.base import RawRecord
+from ..validation.base import ValidationResult
+from ..validation.schema_registry import validate_schema
+from .base import NormalizedRecord, Parser
+
+logger = logging.getLogger("topos.ingestion.parser.messenger")
+
+
+def _normalize_messenger_payload(payload: Dict[str, Any], record_id: str, dataset_id: str) -> Dict[str, Any]:
+    """Convert raw messenger record to normalized shape for conversation_messages."""
+    role = (payload.get("role") or payload.get("sender_type") or "user").lower()
+    sender_type = "human"  # Preserve legacy semantics; identity is carried in sender_id.
+    created_at = payload.get("created_at") or payload.get("ts")
+    ts = ""
+    if isinstance(created_at, (int, float)):
+        from datetime import datetime, timezone
+        try:
+            ts = datetime.fromtimestamp(created_at, tz=timezone.utc).isoformat()
+        except (OverflowError, OSError, ValueError):
+            # Keep ingestion resilient if a source record has an out-of-range timestamp.
+            ts = ""
+    elif isinstance(created_at, str):
+        ts = created_at
+    normalized = {
+        "message_id": str(payload.get("id") or payload.get("message_id") or record_id),
+        "dataset_id": dataset_id,
+        "thread_id": str(payload.get("thread_id") or payload.get("conversation_id") or ""),
+        "conversation_id": str(payload.get("thread_id") or payload.get("conversation_id") or ""),
+        "ts": ts,
+        "sender_type": sender_type,
+        "content": (payload.get("content") or "") or "",
+    }
+    if "_metadata" in payload:
+        normalized["_metadata"] = payload["_metadata"]
+    if payload.get("sender_id") is not None:
+        normalized["sender_id"] = str(payload["sender_id"])
+    if payload.get("reply_to_message_id") is not None:
+        normalized["reply_to_message_id"] = str(payload["reply_to_message_id"])
+    if payload.get("message_type") is not None:
+        normalized["message_type"] = str(payload["message_type"])
+    if payload.get("event_type") is not None:
+        normalized["event_type"] = str(payload["event_type"])
+    return normalized
+
+
+@dataclass
+class ImessageParser(Parser):
+    """Parser for iMessage normalized records (imessage.messages.v1)."""
+
+    dataset_id: str
+    _schema_id: str = "imessage.messages.v1"
+
+    def parse(self, raw: RawRecord) -> NormalizedRecord:
+        payload = raw.payload
+        normalized = _normalize_messenger_payload(payload, raw.record_id, self.dataset_id)
+        return NormalizedRecord(record_id=normalized["message_id"], payload=normalized)
+
+    def validate(self, record: RawRecord) -> ValidationResult:
+        is_valid, error = validate_schema(record.payload, self._schema_id)
+        errors = [] if is_valid else [error or "Invalid record"]
+        return ValidationResult(is_valid=is_valid, errors=errors, metadata={})
+
+    def schema_id(self) -> str:
+        return self._schema_id
+
+
+@dataclass
+class SignalParser(Parser):
+    """Parser for Signal normalized records (signal.messages.v1)."""
+
+    dataset_id: str
+    _schema_id: str = "signal.messages.v1"
+
+    def parse(self, raw: RawRecord) -> NormalizedRecord:
+        payload = raw.payload
+        normalized = _normalize_messenger_payload(payload, raw.record_id, self.dataset_id)
+        return NormalizedRecord(record_id=normalized["message_id"], payload=normalized)
+
+    def validate(self, record: RawRecord) -> ValidationResult:
+        is_valid, error = validate_schema(record.payload, self._schema_id)
+        errors = [] if is_valid else [error or "Invalid record"]
+        return ValidationResult(is_valid=is_valid, errors=errors, metadata={})
+
+    def schema_id(self) -> str:
+        return self._schema_id

topos/ingestion/progress.py

@@ -0,0 +1,54 @@
+"""Progress tracking for ingestion jobs."""
+
+from __future__ import annotations
+
+import time
+from typing import Optional
+
+
+class IngestionProgress:
+    def __init__(self, job_id: str, records_total: Optional[int] = None):
+        self.job_id = job_id
+        self.records_total = records_total
+        self.records_processed = 0
+        self.start_time = time.time()
+        self.last_update_time = self.start_time
+        self.current_step = "parsing"
+        self.errors_count = 0
+
+    def update(self, records_processed: int, current_step: Optional[str] = None) -> None:
+        self.records_processed = records_processed
+        self.last_update_time = time.time()
+        if current_step:
+            self.current_step = current_step
+
+    def get_progress_percent(self) -> float:
+        if not self.records_total:
+            return 0.0
+        return min(100.0, (self.records_processed / self.records_total) * 100.0)
+
+    def get_estimated_seconds_remaining(self) -> Optional[int]:
+        if not self.records_total or self.records_processed == 0:
+            return None
+        elapsed = time.time() - self.start_time
+        if elapsed <= 0:
+            return None
+        rate = self.records_processed / elapsed
+        if rate <= 0:
+            return None
+        remaining = self.records_total - self.records_processed
+        return int(remaining / rate)
+
+    def should_report(self, min_interval: float = 1.0) -> bool:
+        return (time.time() - self.last_update_time) >= min_interval
+
+    def to_dict(self) -> dict:
+        return {
+            "job_id": self.job_id,
+            "progress_percent": self.get_progress_percent(),
+            "records_processed": self.records_processed,
+            "records_total": self.records_total,
+            "estimated_seconds_remaining": self.get_estimated_seconds_remaining(),
+            "current_step": self.current_step,
+            "errors_count": self.errors_count,
+        }

topos/ingestion/sources/__init__.py

@@ -0,0 +1,20 @@
+"""Source connector registry."""
+
+from .base import SourceConnector
+from .calendar import CalendarSourceConnector
+from .chatgpt import ChatGPTSourceConnector
+from .grok import GrokSourceConnector
+
+SOURCE_REGISTRY = {
+    "chatgpt": ChatGPTSourceConnector,
+    "grok": GrokSourceConnector,
+    "calendar": CalendarSourceConnector,
+}
+
+__all__ = [
+    "SourceConnector",
+    "ChatGPTSourceConnector",
+    "GrokSourceConnector",
+    "CalendarSourceConnector",
+    "SOURCE_REGISTRY",
+]

topos/ingestion/sources/base.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict, Literal
+
+
+@dataclass(frozen=True)
+class SourcePayload:
+    payload: Dict[str, str]
+
+
+@dataclass(frozen=True)
+class RawRecord:
+    record_id: str
+    payload: Dict[str, str]
+
+
+@dataclass(frozen=True)
+class SourceIdentity:
+    source_system: str
+    source_record_id: str
+    source_export_id: str
+
+
+class SourceConnector:
+    source_name: str
+    source_type: Literal["file", "sqlite"]
+
+    def ingest(self, payload: SourcePayload) -> str:
+        raise NotImplementedError
+
+    def schema(self) -> Dict[str, str]:
+        raise NotImplementedError
+
+    def identity(self, record: RawRecord) -> SourceIdentity:
+        raise NotImplementedError
+
+    def canonical_eligible(self) -> bool:
+        raise NotImplementedError

topos/ingestion/sources/calendar.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict, Literal
+
+from .base import RawRecord, SourceConnector, SourceIdentity, SourcePayload
+
+
+@dataclass
+class CalendarSourceConnector(SourceConnector):
+    source_name: str = "calendar"
+    source_type: Literal["file", "sqlite"] = "file"
+
+    def ingest(self, payload: SourcePayload) -> str:
+        _ = payload
+        return "calendar.events.v1"
+
+    def schema(self) -> Dict[str, str]:
+        return {"schema_id": "calendar.events.v1"}
+
+    def identity(self, record: RawRecord) -> SourceIdentity:
+        return SourceIdentity(
+            source_system="calendar",
+            source_record_id=record.record_id,
+            source_export_id=record.record_id,
+        )
+
+    def canonical_eligible(self) -> bool:
+        return False

topos/ingestion/sources/chatgpt.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict, Literal
+
+from .base import RawRecord, SourceConnector, SourceIdentity, SourcePayload
+
+
+@dataclass
+class ChatGPTSourceConnector(SourceConnector):
+    source_name: str = "chatgpt"
+    source_type: Literal["file", "sqlite"] = "file"
+
+    def ingest(self, payload: SourcePayload) -> str:
+        _ = payload
+        return "chatgpt.conversation.v1"
+
+    def schema(self) -> Dict[str, str]:
+        return {"schema_id": "chatgpt.conversation.v1"}
+
+    def identity(self, record: RawRecord) -> SourceIdentity:
+        return SourceIdentity(
+            source_system="chatgpt",
+            source_record_id=record.record_id,
+            source_export_id=record.record_id,
+        )
+
+    def canonical_eligible(self) -> bool:
+        return True

topos/ingestion/sources/contact_importers.py

@@ -0,0 +1,274 @@
+from __future__ import annotations
+
+import json
+import logging
+import platform
+import re
+import shutil
+import ssl
+import subprocess
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from typing import Any, Dict, List, Optional
+
+
+GOOGLE_DEVICE_CODE_URL = "https://oauth2.googleapis.com/device/code"
+GOOGLE_TOKEN_URL = "https://oauth2.googleapis.com/token"
+GOOGLE_PEOPLE_CONNECTIONS_URL = "https://people.googleapis.com/v1/people/me/connections"
+
+logger = logging.getLogger("topos.ingestion.sources.contact_importers")
+
+
+def _build_ssl_context() -> ssl.SSLContext:
+    """
+    Build a verified TLS context for outbound Google API calls.
+
+    On some macOS Python runtimes, default trust roots are missing and
+    certificate validation fails. Prefer certifi when available.
+    """
+    try:
+        import certifi
+
+        return ssl.create_default_context(cafile=certifi.where())
+    except Exception:
+        return ssl.create_default_context()
+
+
+def _http_post_form(url: str, body: Dict[str, Any]) -> Dict[str, Any]:
+    payload = urllib.parse.urlencode({k: v for k, v in body.items() if v is not None}).encode("utf-8")
+    req = urllib.request.Request(
+        url=url,
+        data=payload,
+        method="POST",
+        headers={"Content-Type": "application/x-www-form-urlencoded"},
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=30, context=_build_ssl_context()) as resp:
+            return json.loads(resp.read().decode("utf-8"))
+    except urllib.error.HTTPError as e:
+        raw = e.read().decode("utf-8", errors="replace")
+        try:
+            return json.loads(raw)
+        except Exception:
+            return {"error": f"http_{e.code}", "error_description": raw}
+
+
+def _http_get_json(url: str, bearer_token: str) -> Dict[str, Any]:
+    req = urllib.request.Request(
+        url=url,
+        method="GET",
+        headers={"Authorization": f"Bearer {bearer_token}"},
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=30, context=_build_ssl_context()) as resp:
+            return json.loads(resp.read().decode("utf-8"))
+    except urllib.error.HTTPError as e:
+        raw = e.read().decode("utf-8", errors="replace")
+        try:
+            return json.loads(raw)
+        except Exception:
+            raise RuntimeError(f"Google API request failed: HTTP {e.code}: {raw[:500]}") from e
+    except urllib.error.URLError as e:
+        raise RuntimeError(
+            f"Google API TLS/network error: {e}. "
+            "If this is a certificate verify failure, ensure certifi is installed in the engine environment."
+        ) from e
+
+
+def _normalize_phone(value: Any) -> str:
+    s = str(value or "").strip()
+    if not s:
+        return ""
+    # Preserve a leading + where present, drop formatting characters.
+    plus = s.startswith("+")
+    digits = re.sub(r"[^\d]", "", s)
+    if not digits:
+        return ""
+    return f"+{digits}" if plus else digits
+
+
+def import_apple_contacts_local() -> List[Dict[str, Any]]:
+    """
+    Read Apple Contacts locally on macOS via JXA (osascript JavaScript bridge).
+    Returns normalized records: [{"display_name": str, "identifiers": [{"type","identifier"}]}]
+    """
+    current_platform = platform.system().lower()
+    logger.info("[CONTACT_IMPORT] Apple import start: platform=%s", current_platform)
+    if current_platform != "darwin":
+        raise RuntimeError("Apple Contacts import is only available on macOS")
+    osascript_bin = shutil.which("osascript")
+    logger.info("[CONTACT_IMPORT] Apple import environment: osascript=%s", osascript_bin or "missing")
+    if not osascript_bin:
+        raise RuntimeError("osascript not found in PATH; Apple Contacts import requires macOS host runtime")
+
+    jxa_script = r"""
+const app = Application("Contacts");
+const people = app.people();
+const out = [];
+for (let i = 0; i < people.length; i++) {
+  const p = people[i];
+  const name = (() => { try { return String(p.name() || "").trim(); } catch (_) { return ""; } })();
+  const phones = [];
+  const emails = [];
+  try {
+    const ph = p.phones();
+    for (let j = 0; j < ph.length; j++) {
+      const val = String(ph[j].value() || "").trim();
+      if (val) phones.push(val);
+    }
+  } catch (_) {}
+  try {
+    const em = p.emails();
+    for (let j = 0; j < em.length; j++) {
+      const val = String(em[j].value() || "").trim();
+      if (val) emails.push(val);
+    }
+  } catch (_) {}
+  if (name || phones.length || emails.length) out.push({ name, phones, emails });
+}
+JSON.stringify(out);
+"""
+    proc = subprocess.run(
+        ["osascript", "-l", "JavaScript", "-e", jxa_script],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if proc.returncode != 0:
+        stderr = (proc.stderr or "").strip()
+        stdout = (proc.stdout or "").strip()
+        logger.error(
+            "[CONTACT_IMPORT] Apple import osascript failed: returncode=%s stderr=%s stdout=%s",
+            proc.returncode,
+            stderr[:500],
+            stdout[:500],
+        )
+        raise RuntimeError((stderr or stdout or "Failed to read Apple Contacts").strip())
+    try:
+        raw_items = json.loads(proc.stdout or "[]")
+    except Exception as e:
+        raise RuntimeError(f"Failed to parse Apple Contacts output: {e}") from e
+
+    imported: List[Dict[str, Any]] = []
+    for item in raw_items:
+        name = str((item or {}).get("name") or "").strip()
+        identifiers: List[Dict[str, str]] = []
+        for p in (item or {}).get("phones") or []:
+            phone = _normalize_phone(p)
+            if phone:
+                identifiers.append({"type": "phone", "identifier": phone})
+        for e in (item or {}).get("emails") or []:
+            email = str(e or "").strip().lower()
+            if email:
+                identifiers.append({"type": "email", "identifier": email})
+        if identifiers:
+            imported.append({"display_name": name or None, "identifiers": identifiers})
+    logger.info(
+        "[CONTACT_IMPORT] Apple import complete: raw_contacts=%d imported_contacts=%d",
+        len(raw_items),
+        len(imported),
+    )
+    return imported
+
+
+def start_google_device_auth(client_id: str) -> Dict[str, Any]:
+    if not str(client_id or "").strip():
+        raise RuntimeError("google_client_id is required")
+    logger.info("[CONTACT_IMPORT] Google device auth start requested")
+    result = _http_post_form(
+        GOOGLE_DEVICE_CODE_URL,
+        {
+            "client_id": client_id.strip(),
+            "scope": "openid https://www.googleapis.com/auth/contacts.readonly",
+        },
+    )
+    if result.get("error"):
+        logger.warning(
+            "[CONTACT_IMPORT] Google device auth start failed: error=%s description=%s",
+            result.get("error"),
+            result.get("error_description"),
+        )
+    else:
+        logger.info("[CONTACT_IMPORT] Google device auth start succeeded")
+    return result
+
+
+def finish_google_device_auth(
+    *,
+    client_id: str,
+    device_code: str,
+    interval_seconds: int = 5,
+    timeout_seconds: int = 120,
+) -> Dict[str, Any]:
+    logger.info(
+        "[CONTACT_IMPORT] Google device auth finish polling start: interval=%s timeout=%s",
+        interval_seconds,
+        timeout_seconds,
+    )
+    started = time.time()
+    interval = max(2, int(interval_seconds or 5))
+    while True:
+        result = _http_post_form(
+            GOOGLE_TOKEN_URL,
+            {
+                "client_id": client_id.strip(),
+                "device_code": device_code,
+                "grant_type": "urn:ietf:params:oauth:grant-type:device_code",
+            },
+        )
+        if result.get("access_token"):
+            logger.info("[CONTACT_IMPORT] Google device auth finish succeeded")
+            return result
+        if result.get("error") not in {"authorization_pending", "slow_down"}:
+            logger.warning(
+                "[CONTACT_IMPORT] Google device auth finish failed: error=%s description=%s",
+                result.get("error"),
+                result.get("error_description"),
+            )
+            return result
+        if time.time() - started > max(30, int(timeout_seconds)):
+            return {"error": "authorization_timeout", "error_description": "Timed out waiting for Google authorization"}
+        if result.get("error") == "slow_down":
+            interval += 2
+        time.sleep(interval)
+
+
+def import_google_contacts(access_token: str) -> List[Dict[str, Any]]:
+    logger.info("[CONTACT_IMPORT] Google contacts fetch start")
+    imported: List[Dict[str, Any]] = []
+    page_token: Optional[str] = None
+    for _ in range(20):
+        params = {
+            "pageSize": "1000",
+            "personFields": "names,emailAddresses,phoneNumbers",
+            "sortOrder": "LAST_MODIFIED_ASCENDING",
+        }
+        if page_token:
+            params["pageToken"] = page_token
+        url = f"{GOOGLE_PEOPLE_CONNECTIONS_URL}?{urllib.parse.urlencode(params)}"
+        data = _http_get_json(url, access_token)
+        for person in data.get("connections") or []:
+            name = None
+            for n in person.get("names") or []:
+                disp = str(n.get("displayName") or "").strip()
+                if disp:
+                    name = disp
+                    break
+            identifiers: List[Dict[str, str]] = []
+            for p in person.get("phoneNumbers") or []:
+                phone = _normalize_phone(p.get("value"))
+                if phone:
+                    identifiers.append({"type": "phone", "identifier": phone})
+            for e in person.get("emailAddresses") or []:
+                email = str(e.get("value") or "").strip().lower()
+                if email:
+                    identifiers.append({"type": "email", "identifier": email})
+            if identifiers:
+                imported.append({"display_name": name, "identifiers": identifiers})
+        page_token = data.get("nextPageToken")
+        if not page_token:
+            break
+    logger.info("[CONTACT_IMPORT] Google contacts fetch complete: imported_contacts=%d", len(imported))
+    return imported

topos/ingestion/sources/grok.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict, Literal
+
+from .base import RawRecord, SourceConnector, SourceIdentity, SourcePayload
+
+
+@dataclass
+class GrokSourceConnector(SourceConnector):
+    source_name: str = "grok"
+    source_type: Literal["file", "sqlite"] = "file"
+
+    def ingest(self, payload: SourcePayload) -> str:
+        _ = payload
+        return "grok.conversation.v1"
+
+    def schema(self) -> Dict[str, str]:
+        return {"schema_id": "grok.conversation.v1"}
+
+    def identity(self, record: RawRecord) -> SourceIdentity:
+        return SourceIdentity(
+            source_system="grok",
+            source_record_id=record.record_id,
+            source_export_id=record.record_id,
+        )
+
+    def canonical_eligible(self) -> bool:
+        return False