pdm-memory 0.1.2__py3-none-any.whl

pdm_memory/ingest/auto_signature.py

@@ -0,0 +1,202 @@
+"""
+Auto-Signature Generator — Task 3.2
+
+Uses an LLM to auto-generate:
+  1. compressed_fact   — ≤500 char compressed memory from raw text
+  2. intent_tags       — exactly 3 mandatory tags (+ up to 2 optional)
+  3. p_magnitude       — suggested importance score (0–100)
+
+Works with both OpenAI and Anthropic clients (duck-typed).
+
+Usage:
+    from pdm_memory.ingest.auto_signature import AutoSignatureGenerator
+    gen = AutoSignatureGenerator(openai_client)
+    result = gen.generate("User said they prefer dark mode and smaller fonts.")
+    print(result.compressed_fact)
+    print(result.intent_tags)
+    print(result.p_magnitude)
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass
+from typing import Any, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+GENERATION_PROMPT = """You are a memory compression engine for an AI assistant.
+
+Your job: compress the raw input into a PDM (Pressure-Driven Memory) signature.
+
+Rules:
+1. compressed_fact: A concise factual statement (max 500 characters). 
+   Strip conversational filler. Keep only the durable, important information.
+2. intent_tags: Exactly 3 to 5 lowercase tags (strings) that categorise this memory.
+   Tags must be specific and searchable (e.g. ["units", "formatting", "preferences"]).
+   The first 3 are mandatory.
+3. p_magnitude: An integer 0-100 representing importance.
+   - 80-100: Critical, user-defining preference or fact
+   - 60-79:  Important, frequently relevant
+   - 40-59:  Useful, occasionally relevant
+   - 20-39:  Low signal, rarely matters
+   - 0-19:   Near-trivial
+
+Return ONLY valid JSON in this exact format (no markdown):
+{
+  "compressed_fact": "...",
+  "intent_tags": ["tag1", "tag2", "tag3"],
+  "p_magnitude": 65
+}
+
+Raw input to compress:
+"""
+
+
+@dataclass
+class AutoSignatureResult:
+    compressed_fact: str
+    intent_tags: List[str]
+    p_magnitude: float
+    raw_response: str = ""
+
+
+class AutoSignatureGenerator:
+    """
+    Generates PDM signatures from raw text using an LLM.
+
+    Args:
+        llm_client: An OpenAI or Anthropic client instance.
+                    The generator will duck-type to detect which one.
+        model:      Model name override (defaults per provider).
+        max_tokens: Max tokens for the LLM response.
+    """
+
+    OPENAI_DEFAULT_MODEL = "gpt-4o-mini"
+    ANTHROPIC_DEFAULT_MODEL = "claude-3-haiku-20240307"
+
+    def __init__(
+        self,
+        llm_client: Any,
+        model: Optional[str] = None,
+        max_tokens: int = 256,
+    ) -> None:
+        self._client = llm_client
+        self._model = model
+        self._max_tokens = max_tokens
+        self._provider = self._detect_provider()
+
+    def generate(self, raw_text: str) -> Optional[AutoSignatureResult]:
+        """
+        Generate a PDM signature for raw_text.
+
+        Args:
+            raw_text: The text to compress into a memory.
+
+        Returns:
+            AutoSignatureResult or None on failure.
+        """
+        prompt = GENERATION_PROMPT + raw_text.strip()[:2000]  # Cap input length
+
+        try:
+            if self._provider == "openai":
+                return self._call_openai(prompt)
+            elif self._provider == "anthropic":
+                return self._call_anthropic(prompt)
+            else:
+                logger.warning("[PDM-AutoSig] Unknown LLM provider; trying OpenAI API shape")
+                return self._call_openai(prompt)
+        except Exception as e:
+            logger.warning("[PDM-AutoSig] Generation failed: %s", e)
+            return None
+
+    # ------------------------------------------------------------------
+    # Provider-specific calls
+    # ------------------------------------------------------------------
+
+    def _call_openai(self, prompt: str) -> Optional[AutoSignatureResult]:
+        model = self._model or self.OPENAI_DEFAULT_MODEL
+        resp = self._client.chat.completions.create(
+            model=model,
+            messages=[{"role": "user", "content": prompt}],
+            max_tokens=self._max_tokens,
+            temperature=0.2,
+        )
+        raw = resp.choices[0].message.content or ""
+        return self._parse_response(raw)
+
+    def _call_anthropic(self, prompt: str) -> Optional[AutoSignatureResult]:
+        model = self._model or self.ANTHROPIC_DEFAULT_MODEL
+        resp = self._client.messages.create(
+            model=model,
+            max_tokens=self._max_tokens,
+            messages=[{"role": "user", "content": prompt}],
+        )
+        raw = resp.content[0].text if resp.content else ""
+        return self._parse_response(raw)
+
+    # ------------------------------------------------------------------
+    # Parsing
+    # ------------------------------------------------------------------
+
+    def _parse_response(self, raw: str) -> Optional[AutoSignatureResult]:
+        raw = raw.strip()
+        # Strip markdown code fences if present
+        if raw.startswith("```"):
+            lines = raw.split("\n")
+            raw = "\n".join(lines[1:-1] if lines[-1].strip() == "```" else lines[1:])
+
+        try:
+            data = json.loads(raw)
+        except json.JSONDecodeError:
+            # Try to extract JSON from mixed content
+            import re
+            match = re.search(r"\{.*\}", raw, re.DOTALL)
+            if not match:
+                logger.warning("[PDM-AutoSig] Could not parse JSON from: %s", raw[:200])
+                return None
+            try:
+                data = json.loads(match.group(0))
+            except json.JSONDecodeError:
+                return None
+
+        fact = str(data.get("compressed_fact", "")).strip()[:500]
+        tags = data.get("intent_tags", [])
+        if isinstance(tags, list):
+            tags = [str(t).lower().strip() for t in tags if t][:5]
+        else:
+            tags = []
+
+        if len(tags) < 3:
+            logger.warning("[PDM-AutoSig] LLM returned fewer than 3 tags: %s", tags)
+
+        p_mag = float(data.get("p_magnitude", 50))
+        p_mag = max(0.0, min(100.0, p_mag))
+
+        if not fact:
+            return None
+
+        return AutoSignatureResult(
+            compressed_fact=fact,
+            intent_tags=tags,
+            p_magnitude=p_mag,
+            raw_response=raw,
+        )
+
+    def _detect_provider(self) -> str:
+        """Duck-type detection of OpenAI vs Anthropic client."""
+        client_type = type(self._client).__name__.lower()
+        module = getattr(type(self._client), "__module__", "") or ""
+
+        if "anthropic" in module or "anthropic" in client_type:
+            return "anthropic"
+        if "openai" in module or "openai" in client_type:
+            return "openai"
+        # Check for common attribute shapes
+        if hasattr(self._client, "messages") and hasattr(self._client.messages, "create"):
+            return "anthropic"
+        if hasattr(self._client, "chat") and hasattr(self._client.chat, "completions"):
+            return "openai"
+        return "openai"  # default fallback

pdm_memory/ingest/batch.py

@@ -0,0 +1,164 @@
+"""
+Batch Processing Handler — Task 3.3
+
+Splits large data sources into chunks and processes them with rate limiting.
+Prevents OOM and API overload when ingesting large datasets (10,000+ rows).
+
+Usage:
+    processor = BatchProcessor(batch_size=50, delay_seconds=0.5)
+    counts = processor.process(data_source, ingester, on_progress=print_progress)
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Any, Callable, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class BatchProcessor:
+    """
+    Processes large data sources in batches.
+
+    Args:
+        batch_size:       Number of records per batch (default 50).
+        delay_seconds:    Pause between batches in seconds (default 0.2).
+        max_retries:      Retries per batch on transient errors (default 3).
+        retry_delay:      Wait before retry, doubles each attempt (default 1.0s).
+    """
+
+    def __init__(
+        self,
+        batch_size: int = 50,
+        delay_seconds: float = 0.2,
+        max_retries: int = 3,
+        retry_delay: float = 1.0,
+    ) -> None:
+        self.batch_size = max(1, batch_size)
+        self.delay_seconds = max(0.0, delay_seconds)
+        self.max_retries = max(0, max_retries)
+        self.retry_delay = max(0.0, retry_delay)
+
+    def process(
+        self,
+        data_source: Any,
+        ingester: Any,  # DataIngester instance
+        on_progress: Optional[Callable[[int, int], None]] = None,
+    ) -> Dict[str, int]:
+        """
+        Process a data source through the ingester in batches.
+
+        Args:
+            data_source: Any of:
+                         - list[dict]
+                         - str (CSV file path or raw CSV content)
+                         - list[str] (plain text lines → auto-converted to dicts)
+            ingester:    DataIngester instance.
+            on_progress: Optional callback(processed: int, total: int).
+
+        Returns:
+            Aggregated counts: {saved, skipped, errors}.
+        """
+        rows = self._normalise_source(data_source)
+        total = len(rows)
+        counts = {"saved": 0, "skipped": 0, "errors": 0}
+        processed = 0
+
+        logger.info("[PDM-Batch] Starting: %d records, batch_size=%d", total, self.batch_size)
+
+        for i in range(0, total, self.batch_size):
+            batch = rows[i : i + self.batch_size]
+            batch_counts = self._process_batch(batch, ingester)
+
+            counts["saved"] += batch_counts["saved"]
+            counts["skipped"] += batch_counts["skipped"]
+            counts["errors"] += batch_counts["errors"]
+            processed += len(batch)
+
+            if on_progress:
+                try:
+                    on_progress(processed, total)
+                except Exception:
+                    pass
+
+            logger.debug(
+                "[PDM-Batch] Batch %d/%d processed | saved=%d skipped=%d errors=%d",
+                processed, total,
+                batch_counts["saved"], batch_counts["skipped"], batch_counts["errors"],
+            )
+
+            # Rate-limit between batches (not after the last one)
+            if i + self.batch_size < total and self.delay_seconds > 0:
+                time.sleep(self.delay_seconds)
+
+        logger.info(
+            "[PDM-Batch] Done: saved=%d skipped=%d errors=%d",
+            counts["saved"], counts["skipped"], counts["errors"],
+        )
+        return counts
+
+    # ------------------------------------------------------------------
+    # Private
+    # ------------------------------------------------------------------
+
+    def _process_batch(
+        self,
+        batch: List[Dict[str, Any]],
+        ingester: Any,
+    ) -> Dict[str, int]:
+        """Process one batch with retry logic."""
+        last_error = None
+        delay = self.retry_delay
+
+        for attempt in range(self.max_retries + 1):
+            try:
+                return ingester.ingest_rows(batch)
+            except Exception as e:
+                last_error = e
+                if attempt < self.max_retries:
+                    logger.warning(
+                        "[PDM-Batch] Batch attempt %d failed (%s). Retrying in %.1fs…",
+                        attempt + 1, e, delay,
+                    )
+                    time.sleep(delay)
+                    delay *= 2  # Exponential backoff
+
+        logger.error("[PDM-Batch] Batch failed after %d attempts: %s", self.max_retries + 1, last_error)
+        return {"saved": 0, "skipped": 0, "errors": len(batch)}
+
+    @staticmethod
+    def _normalise_source(data_source: Any) -> List[Dict[str, Any]]:
+        """Convert any supported source type to list[dict]."""
+        if isinstance(data_source, list):
+            if not data_source:
+                return []
+            if isinstance(data_source[0], dict):
+                return data_source
+            # Assume list of strings
+            return [{"text": str(item)} for item in data_source]
+
+        if isinstance(data_source, str):
+            # Try as CSV file path or CSV content
+            import csv
+            from io import StringIO
+            try:
+                with open(data_source, newline="", encoding="utf-8") as f:
+                    content = f.read()
+            except (FileNotFoundError, OSError):
+                content = data_source  # Treat as raw CSV string
+            reader = csv.DictReader(StringIO(content))
+            return list(reader)
+
+        # Try iterator/generator
+        try:
+            rows = list(data_source)
+            if rows and isinstance(rows[0], dict):
+                return rows
+            return [{"text": str(r)} for r in rows]
+        except Exception:
+            raise ValueError(
+                f"Unsupported data_source type: {type(data_source)}. "
+                "Expected list[dict], list[str], or a CSV file path/string."
+            )

pdm_memory/ingest/ingester.py

@@ -0,0 +1,272 @@
+"""
+Data Ingester — Task 3.1
+
+Convert any legacy data source (list[dict], CSV, raw messages) into
+PDM SignatureRecords.
+
+Usage (via Memory):
+    mem.ingest(
+        data_source=[
+            {"text": "User dislikes long responses", "importance": 70},
+            {"message": "Prefers Python over JS"},
+        ],
+        mapping={"text": "compressed_fact", "importance": "p_magnitude"},
+    )
+
+Usage (standalone):
+    from pdm_memory.ingest.ingester import DataIngester
+    ingester = DataIngester(storage=driver, user="alice")
+    count = ingester.ingest_rows(rows)
+"""
+
+from __future__ import annotations
+
+import csv
+import logging
+from io import StringIO
+from typing import Any, Dict, List, Optional
+
+from pdm_memory.core.math import calculate_effective_spike, infer_domain
+from pdm_memory.core.signature import SignatureRecord
+from pdm_memory.storage.base import BaseStorage
+
+logger = logging.getLogger(__name__)
+
+
+# Auto-detect common field name aliases
+_FIELD_ALIASES: Dict[str, str] = {
+    # compressed_fact aliases
+    "text": "compressed_fact",
+    "content": "compressed_fact",
+    "message": "compressed_fact",
+    "body": "compressed_fact",
+    "fact": "compressed_fact",
+    "memory": "compressed_fact",
+    "summary": "compressed_fact",
+    # p_magnitude aliases
+    "importance": "p_magnitude",
+    "priority": "p_magnitude",
+    "pressure": "p_magnitude",
+    "score": "p_magnitude",
+    "weight": "p_magnitude",
+    # intent_tags aliases
+    "tags": "intent_tags",
+    "labels": "intent_tags",
+    "categories": "intent_tags",
+    "keywords": "intent_tags",
+    # drawer aliases
+    "category": "drawer_domain",
+    "drawer": "drawer_domain",
+    "topic": "drawer_domain",
+    "domain": "drawer_domain",
+    # source aliases
+    "origin": "source",
+    "channel": "source",
+    # regime aliases
+    "context": "question_regime",
+    "regime": "question_regime",
+}
+
+
+class DataIngester:
+    """
+    Converts structured rows into SignatureRecords and saves them.
+
+    Args:
+        storage:    The active BaseStorage backend.
+        user:       User identifier.
+        mapping:    Optional explicit field mapping (source_key → pdm_field).
+                    If None, auto-detect via _FIELD_ALIASES.
+        llm_client: Optional LLM client for auto-signature generation.
+                    If provided, missing fields are filled via the LLM.
+    """
+
+    def __init__(
+        self,
+        storage: BaseStorage,
+        user: str = "default",
+        mapping: Optional[Dict[str, str]] = None,
+        llm_client: Optional[Any] = None,
+    ) -> None:
+        self._storage = storage
+        self._user = user
+        self._mapping = mapping
+        self._llm_client = llm_client
+
+    def ingest_rows(self, rows: List[Dict[str, Any]]) -> Dict[str, int]:
+        """
+        Ingest a list of dicts.  Returns {'saved': N, 'skipped': N, 'errors': N}.
+        """
+        counts = {"saved": 0, "skipped": 0, "errors": 0}
+
+        for i, row in enumerate(rows):
+            try:
+                sig = self._row_to_signature(row)
+                if sig is None:
+                    counts["skipped"] += 1
+                    continue
+                self._storage.save(sig)
+                counts["saved"] += 1
+            except Exception as e:
+                logger.warning("[PDM-Ingest] Row %d failed: %s", i, e)
+                counts["errors"] += 1
+
+        return counts
+
+    def ingest_csv(self, csv_path_or_content: str) -> Dict[str, int]:
+        """
+        Ingest from a CSV file path or raw CSV string.
+        """
+        try:
+            with open(csv_path_or_content, newline="", encoding="utf-8") as f:
+                content = f.read()
+        except (FileNotFoundError, OSError):
+            content = csv_path_or_content  # Treat as raw CSV string
+
+        reader = csv.DictReader(StringIO(content))
+        return self.ingest_rows(list(reader))
+
+    # ------------------------------------------------------------------
+    # Private
+    # ------------------------------------------------------------------
+
+    def _row_to_signature(self, row: Dict[str, Any]) -> Optional[SignatureRecord]:
+        """Map one row to a SignatureRecord using auto-detect or explicit mapping."""
+        # Build effective mapping
+        field_map = self._build_field_map(row)
+
+        # Extract compressed_fact (required)
+        text = field_map.get("compressed_fact", "")
+        if not text:
+            # Try LLM fallback
+            if self._llm_client:
+                raw_data = " | ".join(f"{k}: {v}" for k, v in row.items())
+                text = self._generate_fact_via_llm(raw_data)
+            if not text:
+                logger.debug("[PDM-Ingest] Skipping row with no text: %s", row)
+                return None
+
+        text = str(text).strip()[:500]
+
+        # Extract other fields with defaults
+        p_mag = _safe_float(field_map.get("p_magnitude"), default=50.0, min_v=0, max_v=100)
+        t_pers = _safe_float(field_map.get("t_persistence"), default=30.0, min_v=1, max_v=365)
+        phase = _safe_float(field_map.get("phase_privilege"), default=1.0)
+        tags = _parse_tags(field_map.get("intent_tags"))
+        drawer = str(field_map.get("drawer_domain") or "general").strip()
+        source = str(field_map.get("source") or "csv").strip()
+        regime = str(field_map.get("question_regime") or "neutral").strip()
+
+        # Auto-generate tags via LLM if none found
+        if not tags and self._llm_client:
+            tags = self._generate_tags_via_llm(text)
+
+        # Auto-generate tags (minimum 3 recommended)
+        if not tags:
+            tags = self._auto_tags_from_text(text)
+
+        domain = infer_domain(tags)
+        eff_spike = calculate_effective_spike(p_mag, t_pers, phase)
+
+        return SignatureRecord(
+            user=self._user,
+            compressed_fact=text,
+            source=source,
+            p_magnitude=p_mag,
+            t_persistence=t_pers,
+            phase_privilege=phase,
+            effective_spike=eff_spike,
+            intent_tags=tags,
+            question_regime=regime,
+            domain=domain,
+            drawer_domain=drawer,
+        )
+
+    def _build_field_map(self, row: Dict[str, Any]) -> Dict[str, Any]:
+        """Apply explicit or auto mapping to a row."""
+        result: Dict[str, Any] = {}
+
+        if self._mapping:
+            # Explicit mapping
+            for src_key, pdm_key in self._mapping.items():
+                if src_key in row:
+                    result[pdm_key] = row[src_key]
+            # Also pass through keys that directly match PDM fields
+            for k, v in row.items():
+                if k not in result:
+                    result[k] = v
+        else:
+            # Auto-detect
+            for k, v in row.items():
+                canonical = _FIELD_ALIASES.get(k.lower(), k.lower())
+                result[canonical] = v
+
+        return result
+
+    def _generate_fact_via_llm(self, raw: str) -> str:
+        """Ask the LLM to compress raw data into a ≤500 char fact."""
+        from pdm_memory.ingest.auto_signature import AutoSignatureGenerator
+        gen = AutoSignatureGenerator(self._llm_client)
+        result = gen.generate(raw)
+        return result.compressed_fact if result else ""
+
+    def _generate_tags_via_llm(self, text: str) -> List[str]:
+        """Ask the LLM to generate 3+ tags for a text."""
+        from pdm_memory.ingest.auto_signature import AutoSignatureGenerator
+        gen = AutoSignatureGenerator(self._llm_client)
+        result = gen.generate(text)
+        return result.intent_tags if result else []
+
+    @staticmethod
+    def _auto_tags_from_text(text: str) -> List[str]:
+        """Extract simple keyword tags from text without an LLM."""
+        import re
+        words = re.findall(r"\b[a-zA-Z]{4,}\b", text.lower())
+        stopwords = {
+            "this", "that", "with", "from", "have", "been", "will",
+            "which", "when", "they", "their", "there", "about",
+        }
+        seen = set()
+        tags = []
+        for w in words:
+            if w not in stopwords and w not in seen:
+                seen.add(w)
+                tags.append(w)
+            if len(tags) >= 5:
+                break
+        return tags[:5]
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _safe_float(val: Any, default: float = 0.0, min_v: float = None, max_v: float = None) -> float:
+    try:
+        v = float(val)
+        if min_v is not None:
+            v = max(min_v, v)
+        if max_v is not None:
+            v = min(max_v, v)
+        return v
+    except (TypeError, ValueError):
+        return default
+
+
+def _parse_tags(val: Any) -> List[str]:
+    if val is None:
+        return []
+    if isinstance(val, list):
+        return [str(t).strip() for t in val if t]
+    if isinstance(val, str):
+        import json
+        try:
+            parsed = json.loads(val)
+            if isinstance(parsed, list):
+                return [str(t).strip() for t in parsed if t]
+        except (json.JSONDecodeError, ValueError):
+            pass
+        # Comma or space separated
+        return [t.strip() for t in val.replace(",", " ").split() if t.strip()]
+    return []

pdm_memory/integrations/__init__.py

@@ -0,0 +1,21 @@
+"""pdm_memory.integrations package."""
+from pdm_memory.integrations.openai_adapter import wrap_openai, PDMOpenAIClient
+from pdm_memory.integrations.anthropic_adapter import wrap_anthropic, PDMAnthropicClient
+from pdm_memory.integrations.context_manager import ContextWindowManager
+from pdm_memory.integrations.gemini_adapter import wrap_gemini, PDMGeminiClient
+from pdm_memory.integrations.ollama_adapter import wrap_ollama, PDMOllamaClient
+from pdm_memory.integrations.groq_adapter import wrap_groq, PDMGroqClient
+
+__all__ = [
+    "wrap_openai",
+    "wrap_anthropic",
+    "wrap_gemini",
+    "wrap_ollama",
+    "wrap_groq",
+    "PDMOpenAIClient",
+    "PDMAnthropicClient",
+    "PDMGeminiClient",
+    "PDMOllamaClient",
+    "PDMGroqClient",
+    "ContextWindowManager",
+]