contextsynapse 1.0.0__py3-none-any.whl

contextsynapse/ingestion/universal/parsers/turn_parser.py

@@ -0,0 +1,329 @@
+"""Multi-provider turn parser.
+
+Supports:
+- Plain text  (``User:`` / ``Assistant:`` or ``Human:`` / ``AI:`` markers)
+- ChatGPT export  (``mapping`` dict keyed by message id)
+- Claude export   (``chat_messages`` list)
+- OpenAI-compatible  (``messages`` list with role/content dicts)
+
+All formats are normalised into a flat ``(SessionMeta, List[Turn])`` pair.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+from contextsynapse.ingestion.universal.ingest_content import Chunk
+
+# ---------------------------------------------------------------------------
+# Data classes
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class Turn:
+    """A single conversational turn."""
+    role: str
+    content: str
+    index: int
+    timestamp: Optional[str] = None
+    model: Optional[str] = None
+
+
+@dataclass
+class SessionMeta:
+    """Metadata about the parsed conversation session."""
+    title: str = ""
+    source: str = ""
+    model: str = ""
+    message_count: int = 0
+    created_at: Optional[str] = None
+    conversation_id: str = ""
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+_ROLE_MAP = {
+    "human": "user",
+    "ai": "assistant",
+    "bot": "assistant",
+}
+
+_TEXT_TURN_RE = re.compile(
+    r"^(User|Assistant|Human|AI)\s*:\s*",
+    re.IGNORECASE | re.MULTILINE,
+)
+
+
+def _normalize_role(raw: str) -> str:
+    return _ROLE_MAP.get(raw.lower(), raw.lower())
+
+
+# ---------------------------------------------------------------------------
+# TurnParser
+# ---------------------------------------------------------------------------
+
+
+class TurnParser:
+    """Parse various chat export formats into uniform Turn sequences."""
+
+    # -- public interface ---------------------------------------------------
+
+    def parse(self, input_data: Union[str, dict, list]) -> Tuple[SessionMeta, List[Turn]]:
+        """Parse *input_data* and return ``(meta, turns)``.
+
+        Handles: single conversation dict, list of conversations (ChatGPT export),
+        JSON string, or plain text with role markers.
+        Raises ``ValueError`` when the format cannot be determined.
+        """
+        if isinstance(input_data, str):
+            # Try JSON first
+            try:
+                parsed = json.loads(input_data)
+                if isinstance(parsed, dict):
+                    return self._dispatch_dict(parsed)
+                if isinstance(parsed, list):
+                    return self._parse_list(parsed)
+            except (json.JSONDecodeError, ValueError):
+                pass
+            return self._parse_text(input_data)
+
+        if isinstance(input_data, dict):
+            return self._dispatch_dict(input_data)
+
+        if isinstance(input_data, list):
+            return self._parse_list(input_data)
+
+        raise ValueError(f"Unsupported input type: {type(input_data).__name__}")
+
+    def _parse_list(self, data: list) -> Tuple[SessionMeta, List[Turn]]:
+        """Parse a list of conversations (e.g. ChatGPT full export).
+
+        Merges all conversations into a single (meta, turns) result.
+        """
+        all_turns: List[Turn] = []
+        titles: List[str] = []
+        source = ""
+        model = ""
+
+        for conv in data:
+            if not isinstance(conv, dict):
+                continue
+            try:
+                meta, turns = self._dispatch_dict(conv)
+                # Re-index turns to be sequential across all conversations
+                offset = len(all_turns)
+                for t in turns:
+                    all_turns.append(Turn(
+                        role=t.role, content=t.content,
+                        index=offset + t.index,
+                        timestamp=t.timestamp, model=t.model,
+                    ))
+                if meta.title:
+                    titles.append(meta.title)
+                if meta.source:
+                    source = meta.source
+                if meta.model:
+                    model = meta.model
+            except Exception:
+                continue
+
+        combined_meta = SessionMeta(
+            title=f"{len(titles)} conversations" if titles else "",
+            source=source or "chatgpt",
+            model=model,
+            message_count=len(all_turns),
+        )
+        return combined_meta, all_turns
+
+    def to_chunks(self, turns: List[Turn], conversation_title: str = "") -> List[Chunk]:
+        """Convert Turn objects into Chunk objects suitable for the pipeline."""
+        chunks: List[Chunk] = []
+        for turn in turns:
+            chunks.append(Chunk(
+                content=turn.content,
+                index=turn.index,
+                chunk_type="turn",
+                metadata={
+                    "role": turn.role,
+                    **({"conversation_title": conversation_title} if conversation_title else {}),
+                    **({"timestamp": turn.timestamp} if turn.timestamp else {}),
+                    **({"model": turn.model} if turn.model else {}),
+                },
+            ))
+        return chunks
+
+    # -- format dispatch ----------------------------------------------------
+
+    def _dispatch_dict(self, data: dict) -> Tuple[SessionMeta, List[Turn]]:
+        if "mapping" in data:
+            return self._parse_chatgpt(data)
+        if "chat_messages" in data:
+            return self._parse_claude(data)
+        if "messages" in data:
+            return self._parse_generic(data)
+        raise ValueError(
+            "Unknown dict format: expected 'mapping', 'chat_messages', or 'messages' key"
+        )
+
+    # -- plain text ---------------------------------------------------------
+
+    def _parse_text(self, text: str) -> Tuple[SessionMeta, List[Turn]]:
+        splits = _TEXT_TURN_RE.split(text)
+        # splits looks like: [preamble, role1, content1, role2, content2, ...]
+        turns: List[Turn] = []
+        idx = 0
+
+        # Skip preamble (index 0) then process pairs
+        i = 1
+        while i < len(splits) - 1:
+            raw_role = splits[i].strip()
+            content = splits[i + 1].strip()
+            role = _normalize_role(raw_role)
+            if role != "system" and content:
+                turns.append(Turn(role=role, content=content, index=idx))
+                idx += 1
+            i += 2
+
+        meta = SessionMeta(
+            source="text",
+            message_count=len(turns),
+        )
+        return meta, turns
+
+    # -- ChatGPT export (mapping dict) --------------------------------------
+
+    def _parse_chatgpt(self, data: dict) -> Tuple[SessionMeta, List[Turn]]:
+        mapping: Dict[str, Any] = data["mapping"]
+
+        # Collect messages with create_time for sorting
+        raw: List[Tuple[float, str, str]] = []
+        for _id, node in mapping.items():
+            msg = node.get("message")
+            if not msg:
+                continue
+            role = _normalize_role(msg.get("author", {}).get("role", ""))
+            if role == "system":
+                continue
+            content_obj = msg.get("content", {})
+            parts = content_obj.get("parts", [])
+            text = "\n".join(str(p) for p in parts if p).strip()
+            if not text:
+                continue
+            create_time = msg.get("create_time") or 0.0
+            raw.append((float(create_time), role, text))
+
+        raw.sort(key=lambda t: t[0])
+
+        turns = [
+            Turn(
+                role=role,
+                content=content,
+                index=idx,
+                timestamp=datetime.fromtimestamp(ts).isoformat() if ts else None,
+            )
+            for idx, (ts, role, content) in enumerate(raw)
+        ]
+
+        meta = SessionMeta(
+            title=data.get("title", ""),
+            source="chatgpt",
+            model=data.get("model", "") or data.get("default_model_slug", ""),
+            message_count=len(turns),
+            created_at=data.get("create_time"),
+            conversation_id=data.get("id", data.get("conversation_id", "")),
+        )
+        return meta, turns
+
+    # -- Claude export (chat_messages list) ---------------------------------
+
+    def _parse_claude(self, data: dict) -> Tuple[SessionMeta, List[Turn]]:
+        messages: List[Dict[str, Any]] = data["chat_messages"]
+        turns: List[Turn] = []
+        idx = 0
+
+        for msg in messages:
+            role = _normalize_role(msg.get("sender", msg.get("role", "")))
+            if role == "system":
+                continue
+
+            # Content may be a plain string or a list of content blocks
+            raw_content = msg.get("text", msg.get("content", ""))
+            if isinstance(raw_content, list):
+                # Content blocks: [{type: "text", text: "..."}, ...]
+                parts = [
+                    block.get("text", "") for block in raw_content
+                    if isinstance(block, dict) and block.get("type") == "text"
+                ]
+                text = "\n".join(parts).strip()
+            else:
+                text = str(raw_content).strip()
+
+            if not text:
+                continue
+
+            turns.append(Turn(
+                role=role,
+                content=text,
+                index=idx,
+                timestamp=msg.get("created_at") or msg.get("timestamp"),
+                model=msg.get("model"),
+            ))
+            idx += 1
+
+        meta = SessionMeta(
+            title=data.get("name", data.get("title", "")),
+            source="claude",
+            model=data.get("model", ""),
+            message_count=len(turns),
+            created_at=data.get("created_at"),
+            conversation_id=data.get("uuid", data.get("id", "")),
+        )
+        return meta, turns
+
+    # -- Generic / OpenAI-compatible (messages list) ------------------------
+
+    def _parse_generic(self, data: dict) -> Tuple[SessionMeta, List[Turn]]:
+        messages: List[Dict[str, Any]] = data["messages"]
+        turns: List[Turn] = []
+        idx = 0
+
+        for msg in messages:
+            role = _normalize_role(msg.get("role", ""))
+            if role == "system":
+                continue
+            content = msg.get("content", "")
+            if isinstance(content, list):
+                parts = [
+                    block.get("text", "") for block in content
+                    if isinstance(block, dict) and block.get("type") == "text"
+                ]
+                content = "\n".join(parts).strip()
+            else:
+                content = str(content).strip()
+            if not content:
+                continue
+
+            turns.append(Turn(
+                role=role,
+                content=content,
+                index=idx,
+                timestamp=msg.get("timestamp"),
+                model=msg.get("model"),
+            ))
+            idx += 1
+
+        meta = SessionMeta(
+            title=data.get("title", ""),
+            source="generic",
+            message_count=len(turns),
+            model=data.get("model", ""),
+            conversation_id=data.get("id", ""),
+        )
+        return meta, turns

contextsynapse/ingestion/universal/shared_signals.py

@@ -0,0 +1,161 @@
+"""Shared signal detection for the universal ingestion pipeline.
+
+Regex-based detection of decision, action, question, problem, solution,
+and clinical signal types.  Each hit is returned as a Signal dataclass
+with type, matched text, and confidence score.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Pattern, Set
+
+# ---------------------------------------------------------------------------
+# Data classes
+# ---------------------------------------------------------------------------
+
+@dataclass
+class Signal:
+    """A single detected signal in a piece of text."""
+    type: str
+    text: str
+    confidence: float
+
+
+# ---------------------------------------------------------------------------
+# Pattern registry  (compiled once at import time)
+# ---------------------------------------------------------------------------
+
+_SIGNAL_PATTERNS: Dict[str, Pattern] = {
+    "decision": re.compile(
+        r"(?i)\b(?:decided\s+to|let'?s\s+go\s+with|we'?ll\s+use|going\s+with"
+        r"|chose|chosen|the\s+plan\s+is\s+to|we\s+agreed|final\s+decision"
+        r"|settled\s+on)\b"
+    ),
+    "action": re.compile(
+        r"(?i)\b(?:need\s+to|TODO|will\s+do|should|must|have\s+to|going\s+to"
+        r"|plan\s+to|action\s+item|next\s+step|deadline)\b"
+    ),
+    "question": re.compile(r"\?\s*$", re.MULTILINE),
+    "problem": re.compile(
+        r"(?i)\b(?:error|failed|issue|bug|broken|doesn'?t\s+work|can'?t"
+        r"|exception|crash(?:ed)?|timeout|404|500)\b"
+    ),
+    "solution": re.compile(
+        r"(?i)\b(?:fix(?:ed)?|solved|workaround|the\s+solution|try\s+this"
+        r"|you\s+can\s+use|resolved|the\s+answer|here'?s\s+how|to\s+fix\s+this)\b"
+    ),
+    "clinical_finding": re.compile(
+        r"(?i)\b(?:diagnosed\s+with|lab\s+results|HbA1c|blood\s+pressure|BMI"
+        r"|positive\s+for|negative\s+for|elevated|reduced)\b"
+    ),
+    "contraindication": re.compile(
+        r"(?i)\b(?:contraindicated|do\s+not\s+use|avoid|not\s+recommended|adverse)\b"
+    ),
+    "guideline": re.compile(
+        r"(?i)\b(?:guidelines?\s+recommend|standard\s+of\s+care"
+        r"|per\s+ADA|per\s+WHO|per\s+CDC)\b"
+    ),
+}
+
+# Confidence per signal type (higher = more decisive)
+_CONFIDENCE: Dict[str, float] = {
+    "decision": 0.90,
+    "action": 0.80,
+    "question": 0.75,
+    "problem": 0.85,
+    "solution": 0.85,
+    "clinical_finding": 0.90,
+    "contraindication": 0.95,
+    "guideline": 0.90,
+}
+
+# ---------------------------------------------------------------------------
+# Acknowledgment pattern (for significance scoring)
+# ---------------------------------------------------------------------------
+
+_ACK_PATTERN = re.compile(
+    r"^(?:ok|okay|thanks|thank\s+you|got\s+it|yes|no|sure|right|yep|nope"
+    r"|agreed|ack|roger|k|yup|nah|cool|fine|alright|understood)[.!]?$",
+    re.IGNORECASE,
+)
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def detect_signals(text: str, types: Optional[List[str]] = None) -> List[Signal]:
+    """Detect signals in *text* using regex patterns.
+
+    Parameters
+    ----------
+    text:
+        The text to scan.
+    types:
+        Optional list of signal type names to restrict detection to.
+        When ``None`` all known types are checked.
+
+    Returns
+    -------
+    List[Signal]
+        Detected signals ordered by occurrence in text.
+    """
+    if not text:
+        return []
+
+    allowed: Optional[Set[str]] = set(types) if types else None
+    signals: List[Signal] = []
+
+    for sig_type, pattern in _SIGNAL_PATTERNS.items():
+        if allowed is not None and sig_type not in allowed:
+            continue
+        for m in pattern.finditer(text):
+            signals.append(Signal(
+                type=sig_type,
+                text=m.group(0),
+                confidence=_CONFIDENCE.get(sig_type, 0.7),
+            ))
+
+    # Sort by position in the original text
+    signals.sort(key=lambda s: text.index(s.text))
+    return signals
+
+
+def significance_score(text: str) -> float:
+    """Score the significance of *text* on a 0.0 – 1.0 scale.
+
+    Heuristics
+    ----------
+    * Pure acknowledgment tokens  → 0.1
+    * Very short text (< 20 chars) → 0.2
+    * Base                         → 0.5
+    * Contains a question mark     → +0.2
+    * Contains a decision pattern  → +0.2
+    * Contains an action pattern   → +0.1
+    * Capped at 1.0
+    """
+    if not text or not text.strip():
+        return 0.0
+
+    stripped = text.strip()
+
+    # Pure acknowledgment
+    if _ACK_PATTERN.match(stripped):
+        return 0.1
+
+    # Very short
+    if len(stripped) < 20:
+        return 0.2
+
+    score = 0.5
+
+    if "?" in stripped:
+        score += 0.2
+    if _SIGNAL_PATTERNS["decision"].search(stripped):
+        score += 0.2
+    if _SIGNAL_PATTERNS["action"].search(stripped):
+        score += 0.1
+
+    return min(score, 1.0)