topos-node 0.1.0__py3-none-any.whl

topos/ingestion/parser.py

@@ -0,0 +1,174 @@
+"""File parsers for ingestion files (JSONL, JSON, CSV)."""
+
+from __future__ import annotations
+
+import csv
+import json
+import logging
+from typing import Any, AsyncIterator, Dict
+
+from .parsers.chatgpt_conversation_flattener import (
+    flatten_conversation_array,
+    is_conversation_format,
+)
+
+logger = logging.getLogger("topos.ingestion.parser")
+
+
+def _strip_json_comments(content: str) -> str:
+    """Remove // and /* */ comments while preserving string literals."""
+    out: list[str] = []
+    i = 0
+    n = len(content)
+    in_string = False
+    escaped = False
+
+    while i < n:
+        ch = content[i]
+        nxt = content[i + 1] if i + 1 < n else ""
+
+        if in_string:
+            out.append(ch)
+            if escaped:
+                escaped = False
+            elif ch == "\\":
+                escaped = True
+            elif ch == '"':
+                in_string = False
+            i += 1
+            continue
+
+        if ch == '"':
+            in_string = True
+            out.append(ch)
+            i += 1
+            continue
+
+        if ch == "/" and nxt == "/":
+            i += 2
+            while i < n and content[i] not in "\r\n":
+                i += 1
+            continue
+
+        if ch == "/" and nxt == "*":
+            i += 2
+            while i < n - 1 and not (content[i] == "*" and content[i + 1] == "/"):
+                if content[i] in "\r\n":
+                    out.append(content[i])
+                i += 1
+            i += 2 if i < n - 1 else 0
+            continue
+
+        out.append(ch)
+        i += 1
+
+    return "".join(out)
+
+
+def _load_json_with_optional_comments(content: str) -> Any:
+    try:
+        return json.loads(content)
+    except json.JSONDecodeError as exc:
+        stripped = _strip_json_comments(content)
+        if stripped != content:
+            try:
+                logger.info("Parsed JSON payload after stripping comments")
+                return json.loads(stripped)
+            except json.JSONDecodeError as commented_exc:
+                raise ValueError(
+                    "Failed to parse JSON file: "
+                    f"{commented_exc.msg} (line {commented_exc.lineno}, column {commented_exc.colno})"
+                ) from commented_exc
+        raise ValueError(
+            f"Failed to parse JSON file: {exc.msg} (line {exc.lineno}, column {exc.colno})"
+        ) from exc
+
+
+async def parse_jsonl_stream(file_stream: AsyncIterator[bytes]) -> AsyncIterator[Dict[str, Any]]:
+    buffer = b""
+    line_num = 0
+    async for chunk in file_stream:
+        buffer += chunk
+        while b"\n" in buffer:
+            line, buffer = buffer.split(b"\n", 1)
+            line = line.strip()
+            if not line:
+                continue
+            line_num += 1
+            try:
+                record = json.loads(line.decode("utf-8"))
+                yield record
+            except json.JSONDecodeError as exc:
+                logger.warning("Failed to parse JSONL line %d: %s", line_num, exc)
+                continue
+    if buffer.strip():
+        line_num += 1
+        try:
+            record = json.loads(buffer.decode("utf-8"))
+            yield record
+        except json.JSONDecodeError as exc:
+            logger.warning("Failed to parse JSONL line %d: %s", line_num, exc)
+
+
+async def parse_json_stream(file_stream: AsyncIterator[bytes]) -> AsyncIterator[Dict[str, Any]]:
+    chunks = []
+    async for chunk in file_stream:
+        chunks.append(chunk)
+    content = b"".join(chunks).decode("utf-8")
+    data = _load_json_with_optional_comments(content)
+    if isinstance(data, list):
+        # Check if this is a ChatGPT conversation array
+        if data and is_conversation_format(data[0]):
+            logger.info("Detected ChatGPT conversation format, flattening conversations")
+            # Flatten conversation array to individual message records
+            for record in flatten_conversation_array(data, include_system=False):
+                yield record
+        else:
+            # Regular array - yield records as-is
+            for record in data:
+                yield record
+    elif isinstance(data, dict):
+        # Check if single conversation object
+        if is_conversation_format(data):
+            logger.info("Detected ChatGPT conversation format (single object), flattening")
+            from .parsers.chatgpt_conversation_flattener import flatten_conversation
+            for record in flatten_conversation(data, include_system=False):
+                yield record
+        elif isinstance(data.get("browsing_history"), list):
+            # Demo browser-history payloads wrap visit rows under a top-level key.
+            # Flatten to per-visit records so source parsers can validate normally.
+            owner_user_id = data.get("user_id")
+            for record in data.get("browsing_history") or []:
+                if isinstance(record, dict):
+                    if owner_user_id and "user_id" not in record:
+                        record = {**record, "user_id": owner_user_id}
+                    yield record
+        else:
+            yield data
+    else:
+        raise ValueError(f"JSON must be array or object, got {type(data)}")
+
+
+async def parse_csv_stream(file_stream: AsyncIterator[bytes], delimiter: str = ",") -> AsyncIterator[Dict[str, Any]]:
+    chunks = []
+    async for chunk in file_stream:
+        chunks.append(chunk)
+    content = b"".join(chunks).decode("utf-8")
+    reader = csv.DictReader(content.splitlines(), delimiter=delimiter)
+    for row in reader:
+        yield row
+
+
+async def parse_file(file_stream: AsyncIterator[bytes], file_format: str) -> AsyncIterator[Dict[str, Any]]:
+    format_lower = file_format.lower()
+    if format_lower in {"jsonl", "ndjson"}:
+        async for record in parse_jsonl_stream(file_stream):
+            yield record
+    elif format_lower == "json":
+        async for record in parse_json_stream(file_stream):
+            yield record
+    elif format_lower == "csv":
+        async for record in parse_csv_stream(file_stream):
+            yield record
+    else:
+        raise ValueError(f"Unsupported file format: {file_format}")

topos/ingestion/parsers/__init__.py

@@ -0,0 +1,32 @@
+"""Parser registry."""
+
+from .base import Parser
+from .browser_parser import BrowserParser, BrowserEventsParser
+from .calendar_parser import CalendarParser
+from .chatgpt_parser import ChatGPTParser
+from .grok_parser import GrokParser
+from .messenger_parser import ImessageParser, SignalParser
+
+PARSER_REGISTRY = {
+    "chatgpt.conversation.v1": ChatGPTParser,
+    "chatgpt.conversation.v2": ChatGPTParser,  # Same parser, flattened records match v1 format
+    "grok.conversation.v1": GrokParser,
+    "calendar.events.v1": CalendarParser,
+    "browser.visits.v1": BrowserParser,  # Sprint 3: Browser plugin visits
+    "managed.file.browser_history_dem.v1": BrowserParser,  # Hosted browser demo source alias
+    "browser.events.v1": BrowserEventsParser,  # Clicks, highlights, star_page, VIDEO_PLAY
+    "imessage.messages.v1": ImessageParser,  # Sprint 02: Messenger ingestion
+    "signal.messages.v1": SignalParser,
+}
+
+__all__ = [
+    "Parser",
+    "BrowserParser",
+    "BrowserEventsParser",
+    "ChatGPTParser",
+    "GrokParser",
+    "CalendarParser",
+    "ImessageParser",
+    "SignalParser",
+    "PARSER_REGISTRY",
+]

topos/ingestion/parsers/base.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Dict
+
+from ..sources.base import RawRecord
+from ..validation.base import ValidationResult
+
+
+@dataclass(frozen=True)
+class NormalizedRecord:
+    record_id: str
+    payload: Dict[str, str]
+
+
+class Parser:
+    def parse(self, raw: RawRecord) -> NormalizedRecord:
+        raise NotImplementedError
+
+    def validate(self, record: RawRecord) -> ValidationResult:
+        raise NotImplementedError
+
+    def schema_id(self) -> str:
+        raise NotImplementedError

topos/ingestion/parsers/browser_parser.py

@@ -0,0 +1,171 @@
+"""Browser visits parser for ingestion layer (Sprint 3)."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import Any, Dict
+from datetime import datetime, timezone
+
+from ..sources.base import RawRecord
+from ..validation.base import ValidationResult
+from .base import NormalizedRecord, Parser
+
+logger = logging.getLogger("topos.ingestion.parser.browser")
+
+
+@dataclass
+class BrowserParser(Parser):
+    """Parser for browser_visits records. Stores records as-is with minimal normalization."""
+    
+    dataset_id: str
+    _schema_id: str = "browser.visits.v1"
+
+    def parse(self, raw: RawRecord) -> NormalizedRecord:
+        """Parse browser visit record. For MVP, we store records as-is."""
+        payload = raw.payload
+        
+        # Normalize timestamp if present
+        visited_at = payload.get("visited_at") or payload.get("timestamp")
+        if isinstance(visited_at, str):
+            # Already ISO format, keep as-is
+            ts = visited_at
+        elif visited_at is None:
+            # Use current time if missing
+            ts = datetime.now(timezone.utc).isoformat()
+        else:
+            ts = str(visited_at)
+        
+        # Create normalized record with all browser fields preserved
+        normalized = {
+            "record_id": payload.get("url", raw.record_id) + "_" + ts,  # Unique ID
+            "dataset_id": self.dataset_id,
+            "url": payload.get("url", ""),
+            "visited_at": ts,
+            "title": payload.get("title", ""),
+            "favicon_url": payload.get("favicon_url"),
+            "hostname": payload.get("hostname", ""),
+            "device_name": payload.get("device_name", ""),
+            "tab_id": payload.get("tab_id"),
+            "window_id": payload.get("window_id"),
+            "incognito": payload.get("incognito"),
+            "transition_type": payload.get("transition_type", ""),
+            "pinned": payload.get("pinned"),
+            "audible": payload.get("audible"),
+            "muted": payload.get("muted"),
+            "opener_tab_id": payload.get("opener_tab_id"),
+            "referred_by": payload.get("referred_by"),
+        }
+        
+        # Remove None values to keep payload clean
+        normalized = {k: v for k, v in normalized.items() if v is not None}
+        
+        logger.debug(
+            "[PIPELINE:PARSER] Parsed browser visit: url=%s, visited_at=%s",
+            normalized.get("url", "")[:50],
+            normalized.get("visited_at", ""),
+        )
+        return NormalizedRecord(record_id=normalized["record_id"], payload=normalized)
+
+    def validate(self, record: RawRecord) -> ValidationResult:
+        """Validate browser visit record. For MVP, minimal validation."""
+        payload = record.payload
+        if not isinstance(payload, dict):
+            return ValidationResult(
+                is_valid=False,
+                errors=["Record must be a dict"],
+                metadata={},
+            )
+        
+        # Required fields for browser_visits
+        if "url" not in payload:
+            return ValidationResult(
+                is_valid=False,
+                errors=["Missing required field: url"],
+                metadata={},
+            )
+        
+        if "visited_at" not in payload and "timestamp" not in payload:
+            return ValidationResult(
+                is_valid=False,
+                errors=["Missing required field: visited_at (or timestamp)"],
+                metadata={},
+            )
+        
+        return ValidationResult(is_valid=True, errors=[], metadata={})
+
+    def schema_id(self) -> str:
+        return self._schema_id
+
+
+@dataclass
+class BrowserEventsParser(Parser):
+    """Parser for browser_events: clicks, highlights, star_page, VIDEO_PLAY. Stores event_type + payload."""
+
+    dataset_id: str
+    _schema_id: str = "browser.events.v1"
+
+    def parse(self, raw: RawRecord) -> NormalizedRecord:
+        """Parse browser event record. Preserves event_type and full payload."""
+        payload = raw.payload
+        event_type = payload.get("event_type") or "unknown"
+        ts = (
+            payload.get("visited_at")
+            or payload.get("starred_at")
+            or payload.get("created_at")
+            or datetime.now(timezone.utc).isoformat()
+        )
+        if isinstance(ts, (int, float)):
+            ts = datetime.fromtimestamp(ts, tz=timezone.utc).isoformat()
+        else:
+            ts = str(ts)
+        url = payload.get("url") or ""
+        record_id = f"{event_type}_{(url or raw.record_id)[:80]}_{ts[:24]}"
+        normalized = {
+            "record_id": record_id,
+            "dataset_id": self.dataset_id,
+            "event_type": event_type,
+            "url": url,
+            "visited_at": ts,
+            "title": payload.get("title"),
+            "favicon_url": payload.get("favicon_url"),
+            "hostname": payload.get("hostname"),
+            "device_name": payload.get("device_name"),
+            "transition_type": payload.get("transition_type"),
+            "content": payload.get("content"),
+            "tab_id": payload.get("tab_id"),
+            "window_id": payload.get("window_id"),
+            "incognito": payload.get("incognito"),
+            "pinned": payload.get("pinned"),
+            "audible": payload.get("audible"),
+            "muted": payload.get("muted"),
+            "opener_tab_id": payload.get("opener_tab_id"),
+            "starred_at": payload.get("starred_at"),
+        }
+        normalized = {k: v for k, v in normalized.items() if v is not None}
+        logger.debug(
+            "[PIPELINE:PARSER] Parsed browser event: event_type=%s, url=%s",
+            event_type,
+            url[:50] if url else None,
+        )
+        return NormalizedRecord(record_id=record_id, payload=normalized)
+
+    def validate(self, record: RawRecord) -> ValidationResult:
+        """Validate browser event: require event_type and at least url or content."""
+        payload = record.payload
+        if not isinstance(payload, dict):
+            return ValidationResult(
+                is_valid=False,
+                errors=["Record must be a dict"],
+                metadata={},
+            )
+        if not payload.get("event_type"):
+            return ValidationResult(
+                is_valid=False,
+                errors=["Missing required field: event_type"],
+                metadata={},
+            )
+        return ValidationResult(is_valid=True, errors=[], metadata={})
+
+    def schema_id(self) -> str:
+        return self._schema_id

topos/ingestion/parsers/calendar_parser.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from ..sources.base import RawRecord
+from ..validation.base import ValidationResult
+from .base import NormalizedRecord, Parser
+
+
+@dataclass
+class CalendarParser(Parser):
+    dataset_id: str
+
+    def parse(self, raw: RawRecord) -> NormalizedRecord:
+        return NormalizedRecord(record_id=raw.record_id, payload=raw.payload)
+
+    def validate(self, record: RawRecord) -> ValidationResult:
+        return ValidationResult(is_valid=True, errors=[], metadata={})
+
+    def schema_id(self) -> str:
+        return "calendar.events.v1"

topos/ingestion/parsers/chatgpt_conversation_flattener.py

@@ -0,0 +1,266 @@
+"""ChatGPT conversation flattener.
+
+Converts nested conversation objects from ChatGPT export format into flat message records.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, Iterator, List, Optional
+
+logger = logging.getLogger("topos.ingestion.parser.chatgpt_flattener")
+
+
+def flatten_conversation(conversation: Dict[str, Any], include_system: bool = False) -> Iterator[Dict[str, Any]]:
+    """Flatten a ChatGPT conversation object into individual message records.
+    
+    Args:
+        conversation: ChatGPT conversation object with 'mapping' field
+        include_system: Whether to include system messages (default: False)
+        
+    Yields:
+        Flattened message records compatible with chatgpt.conversation.v1 format
+    """
+    mapping = conversation.get("mapping", {})
+    if not mapping:
+        logger.warning("Conversation has no mapping field")
+        return  # Generator function - return without yielding means empty
+    
+    conv_id = conversation.get("conversation_id") or conversation.get("id", "")
+    conv_title = conversation.get("title")
+    conv_create_time = conversation.get("create_time")
+    
+    # Traverse the message tree and extract messages
+    visited = set()
+    
+    def traverse_node(node_id: str) -> Iterator[Dict[str, Any]]:
+        """Recursively traverse message nodes and yield records."""
+        if node_id in visited or node_id not in mapping:
+            return
+        
+        visited.add(node_id)
+        node = mapping[node_id]
+        message = node.get("message")
+        
+        # Skip nodes without messages (root nodes, etc.)
+        if not message:
+            # Still traverse children
+            children = node.get("children")
+            if children:
+                for child_id in children:
+                    yield from traverse_node(child_id)
+            return
+        
+        # Extract message data
+        role = message.get("author", {}).get("role", "").lower()
+        
+        # Skip system messages unless explicitly included
+        if role == "system" and not include_system:
+            # Still traverse children
+            children = node.get("children")
+            if children:
+                for child_id in children:
+                    yield from traverse_node(child_id)
+            return
+        
+        # Extract content
+        content_obj = message.get("content", {})
+        content_type = content_obj.get("content_type", "text")
+        parts = content_obj.get("parts", [])
+        
+        # Handle different content types
+        content = extract_content(content_obj, content_type)
+        
+        # Skip messages with empty content (unless they're tool calls)
+        if not content and content_type == "text":
+            # Still traverse children (might be tool execution results)
+            children = node.get("children")
+            if children:
+                for child_id in children:
+                    yield from traverse_node(child_id)
+            return
+        
+        # Extract timestamp
+        create_time = message.get("create_time")
+        if create_time is None:
+            create_time = conv_create_time
+        
+        # Map role to expected format
+        # ChatGPT uses: user, assistant, system, tool
+        # We need: user -> "user", assistant -> "assistant", tool -> "assistant"
+        mapped_role = role
+        if role == "tool":
+            mapped_role = "assistant"  # Tool messages are from assistant
+        
+        # Create flattened record
+        record = {
+            "id": message.get("id", node_id),
+            "thread_id": conv_id,
+            "role": mapped_role,
+            "content": content,
+            "created_at": create_time,
+            # Additional metadata (optional, for debugging)
+            "_metadata": {
+                "conversation_title": conv_title,
+                "node_id": node_id,
+                "parent_id": node.get("parent"),
+                "content_type": content_type,
+                "original_role": role,
+            },
+        }
+        
+        yield record
+        
+        # Traverse children
+        children = node.get("children")
+        if children:
+            for child_id in children:
+                yield from traverse_node(child_id)
+    
+    # Find root nodes (nodes with no parent or parent not in mapping)
+    root_nodes = []
+    for node_id, node in mapping.items():
+        parent = node.get("parent")
+        if not parent or parent not in mapping:
+            root_nodes.append(node_id)
+    
+    # Start traversal from root nodes
+    for root_id in root_nodes:
+        try:
+            yield from traverse_node(root_id)
+        except Exception as e:
+            logger.warning(f"Error traversing root node {root_id}: {e}", exc_info=True)
+            continue
+
+
+def extract_content(content_obj: Dict[str, Any], content_type: str) -> str:
+    """Extract text content from content object based on content type.
+    
+    Args:
+        content_obj: Content object from message
+        content_type: Type of content (text, thoughts, reasoning_recap, etc.)
+        
+    Returns:
+        Extracted text content
+    """
+    if content_type == "text":
+        parts = content_obj.get("parts", [])
+        if isinstance(parts, list):
+            # Join parts, filtering out empty strings
+            return " ".join(str(p) for p in parts if p and str(p).strip())
+        return str(parts) if parts else ""
+    
+    elif content_type == "thoughts":
+        # Extract thoughts content
+        thoughts = content_obj.get("thoughts", [])
+        if isinstance(thoughts, list):
+            # Extract summary or content from each thought
+            thought_texts = []
+            for thought in thoughts:
+                if isinstance(thought, dict):
+                    summary = thought.get("summary", "")
+                    content = thought.get("content", "")
+                    if summary:
+                        thought_texts.append(summary)
+                    elif content:
+                        thought_texts.append(content)
+                elif isinstance(thought, str):
+                    thought_texts.append(thought)
+            return " ".join(thought_texts)
+        return str(thoughts) if thoughts else ""
+    
+    elif content_type == "reasoning_recap":
+        # Extract reasoning recap content
+        recap = content_obj.get("reasoning_recap", "")
+        if recap:
+            return str(recap)
+        # Fallback to parts if available
+        parts = content_obj.get("parts", [])
+        if isinstance(parts, list):
+            return " ".join(str(p) for p in parts if p)
+        return ""
+    
+    elif content_type == "code":
+        # Extract code content
+        code = content_obj.get("code", "")
+        if code:
+            return f"```\n{code}\n```"
+        # Fallback to parts
+        parts = content_obj.get("parts", [])
+        if isinstance(parts, list):
+            return " ".join(str(p) for p in parts if p)
+        return ""
+    
+    elif content_type == "multimodal_text":
+        # Extract multimodal text (may have images, etc.)
+        parts = content_obj.get("parts", [])
+        if isinstance(parts, list):
+            # Filter out non-text parts
+            text_parts = [str(p) for p in parts if isinstance(p, str) and p.strip()]
+            return " ".join(text_parts)
+        return ""
+    
+    elif content_type == "execution_output":
+        # Extract execution output
+        output = content_obj.get("output", "")
+        if output:
+            return str(output)
+        parts = content_obj.get("parts", [])
+        if isinstance(parts, list):
+            return " ".join(str(p) for p in parts if p)
+        return ""
+    
+    else:
+        # Unknown content type - try to extract parts
+        logger.warning(f"Unknown content type: {content_type}")
+        parts = content_obj.get("parts", [])
+        if isinstance(parts, list):
+            return " ".join(str(p) for p in parts if p)
+        return ""
+
+
+def is_conversation_format(record: Dict[str, Any]) -> bool:
+    """Check if a record is a ChatGPT conversation object.
+    
+    Args:
+        record: Record to check
+        
+    Returns:
+        True if record appears to be a conversation object
+    """
+    return (
+        isinstance(record, dict) and
+        "mapping" in record and
+        isinstance(record.get("mapping"), dict) and
+        ("conversation_id" in record or "id" in record)
+    )
+
+
+def flatten_conversation_array(conversations: List[Dict[str, Any]], include_system: bool = False) -> Iterator[Dict[str, Any]]:
+    """Flatten an array of conversation objects.
+    
+    Args:
+        conversations: List of conversation objects
+        include_system: Whether to include system messages
+        
+    Yields:
+        Flattened message records
+    """
+    if not conversations:
+        return
+    
+    for conv in conversations:
+        if not conv or not isinstance(conv, dict):
+            logger.warning(f"Skipping invalid conversation object: {type(conv)}")
+            continue
+            
+        if not is_conversation_format(conv):
+            logger.warning(f"Skipping non-conversation object: {type(conv)}")
+            continue
+        
+        try:
+            for record in flatten_conversation(conv, include_system=include_system):
+                yield record
+        except Exception as e:
+            logger.error(f"Error flattening conversation {conv.get('conversation_id', conv.get('id', 'unknown'))}: {e}", exc_info=True)
+            continue