ihmt-memory 1.2.0__py3-none-any.whl

ihmt/__init__.py

@@ -0,0 +1,134 @@
+"""IHMT — Infinite Hierarchical Memory Tree.
+
+A domain-agnostic, dependency-free long-term memory for LLMs. Knowledge is
+stored as a recursive tree on the local filesystem: raw ``.txt`` leaves at
+layer 0, JSON "summaries of summaries" in layers 1..N, and a single
+``root.json`` trunk. Retrieval walks that tree instead of scanning it, so the
+cost of a query grows with the *depth* of the tree, not with its size.
+
+Typical use::
+
+    from ihmt import IHMT
+
+    memory = IHMT.initialize("./workspace")
+    memory.ingest_file("notes/journal.txt")
+    memory.flush()
+    print(memory.search("vacaciones en Benidorm").best.excerpt)
+"""
+
+from __future__ import annotations
+
+from .api import IHMT
+from .project_index import CodeHit, FindResult, ProjectIndex, SyncReport
+from .chunkers import (
+    Block,
+    Chunk,
+    Chunker,
+    CodeAwareChunker,
+    GenericChunker,
+    NarrativeChunker,
+    ProcessChunker,
+    TemporalChunker,
+    get_chunker,
+)
+from .config import IHMTConfig
+from .conflict_resolver import ConflictResolver, FactPattern
+from .detectors import Detection, DomainDetector
+from .models import (
+    BranchNode,
+    ChildRef,
+    Contradiction,
+    DataType,
+    DomainEntry,
+    Fact,
+    IHMTError,
+    LeafStatus,
+    MalformedLeafError,
+    MemoryLeaf,
+    NodeKind,
+    NodeNotFoundError,
+    RootIndex,
+    StoreNotInitializedError,
+)
+from .recursive_summarizer import ConsolidationReport, RecursiveSummarizer, SummarizationEvent
+from .semantic_navigator import (
+    ClueOption,
+    ClueRequest,
+    SearchResponse,
+    SearchResult,
+    SemanticNavigator,
+)
+from .storage import Catalog, CatalogEntry, MemoryStore
+from .summarizers import (
+    AnthropicSummarizer,
+    HeuristicSummarizer,
+    NodeSummary,
+    SummarizerBackend,
+    SummaryInput,
+    get_summarizer,
+)
+from .universal_ingestor import IngestReport, UniversalIngestor
+
+__version__ = "1.1.0"
+
+__all__ = [
+    "ProjectIndex",
+    "SyncReport",
+    "CodeHit",
+    "FindResult",
+    "__version__",
+    # facade
+    "IHMT",
+    "IHMTConfig",
+    # storage
+    "MemoryStore",
+    "Catalog",
+    "CatalogEntry",
+    # model
+    "MemoryLeaf",
+    "BranchNode",
+    "ChildRef",
+    "RootIndex",
+    "DomainEntry",
+    "Fact",
+    "Contradiction",
+    "DataType",
+    "NodeKind",
+    "LeafStatus",
+    "IHMTError",
+    "NodeNotFoundError",
+    "MalformedLeafError",
+    "StoreNotInitializedError",
+    # pipeline
+    "DomainDetector",
+    "Detection",
+    "UniversalIngestor",
+    "IngestReport",
+    "RecursiveSummarizer",
+    "ConsolidationReport",
+    "SummarizationEvent",
+    "SemanticNavigator",
+    "SearchResponse",
+    "SearchResult",
+    "ClueRequest",
+    "ClueOption",
+    "ConflictResolver",
+    "FactPattern",
+    # chunkers
+    "Chunker",
+    "Chunk",
+    "Block",
+    "CodeAwareChunker",
+    "NarrativeChunker",
+    "TemporalChunker",
+    "ProcessChunker",
+    "GenericChunker",
+    "get_chunker",
+    # summarizers
+    "SummarizerBackend",
+    "SummaryInput",
+    "NodeSummary",
+    "HeuristicSummarizer",
+    "AnthropicSummarizer",
+    "get_summarizer",
+]

ihmt/api.py

@@ -0,0 +1,166 @@
+"""High-level facade wiring the five components together.
+
+Most callers — the CLI, a notebook, or an LLM tool layer — only need this::
+
+    from ihmt import IHMT
+
+    memory = IHMT.initialize("./workspace")     # or IHMT("./workspace")
+    memory.ingest_file("InventoryService.java")
+    memory.flush()                              # close the tree
+    answer = memory.search("reserveStock concurrency")
+    answer = memory.ask("Luis", clue_provider=input_clue)
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional
+
+from .config import IHMTConfig
+from .conflict_resolver import ConflictResolver
+from .detectors import DomainDetector
+from .models import Contradiction, DataType, MemoryLeaf
+from .recursive_summarizer import ConsolidationReport, RecursiveSummarizer
+from .semantic_navigator import ClueRequest, SearchResponse, SemanticNavigator
+from .storage import MemoryStore
+from .summarizers import SummarizerBackend
+from .universal_ingestor import IngestReport, UniversalIngestor
+
+
+class IHMT:
+    """One initialized memory tree, with every component pre-wired."""
+
+    def __init__(
+        self,
+        base_dir: Path | str = ".",
+        *,
+        backend: Optional[SummarizerBackend] = None,
+        auto_consolidate: bool = True,
+        require_init: bool = True,
+    ) -> None:
+        """
+        Args:
+            base_dir: Directory containing (or to contain) ``ihmt_memory``.
+            backend: Summarization backend override.
+            auto_consolidate: Fire Summarization Events during ingestion.
+            require_init: Raise if the store does not exist yet.
+        """
+        self.store = MemoryStore.open(base_dir, require_init=require_init)
+        self.config: IHMTConfig = self.store.config
+        self.detector = DomainDetector()
+        self.resolver = ConflictResolver(self.store)
+        self.summarizer = RecursiveSummarizer(self.store, backend=backend)
+        self.ingestor = UniversalIngestor(
+            self.store,
+            detector=self.detector,
+            summarizer=self.summarizer,
+            conflict_resolver=self.resolver,
+            auto_consolidate=auto_consolidate,
+        )
+        self.navigator = SemanticNavigator(self.store, conflict_resolver=self.resolver)
+
+    # ------------------------------------------------------------- lifecycle
+    @classmethod
+    def initialize(
+        cls,
+        base_dir: Path | str = ".",
+        *,
+        force: bool = False,
+        **overrides: Any,
+    ) -> "IHMT":
+        """Create the store on disk (idempotent) and return a ready facade.
+
+        Args:
+            base_dir: Where to create ``ihmt_memory``.
+            force: Discard all *derived* state — trunk, branch layers, catalog
+                and fact timeline — and detach the surviving leaves so the next
+                consolidation rebuilds the hierarchy. Leaf content is kept.
+            **overrides: Configuration fields, e.g. ``branch_factor=4``.
+        """
+        config = IHMTConfig.load(base_dir)
+        for key, value in overrides.items():
+            if not hasattr(config, key):
+                raise ValueError(f"unknown configuration field: {key}")
+            setattr(config, key, value)
+        config.validate()
+        MemoryStore(config).initialize(force=force)
+        return cls(base_dir)
+
+    # -------------------------------------------------------------- ingestion
+    def ingest_text(self, text: str, **kwargs: Any) -> IngestReport:
+        """Ingest an in-memory document. See :meth:`UniversalIngestor.ingest_text`."""
+        return self.ingestor.ingest_text(text, **kwargs)
+
+    def ingest_file(self, path: Path | str, **kwargs: Any) -> IngestReport:
+        """Ingest a single file."""
+        return self.ingestor.ingest_file(path, **kwargs)
+
+    def ingest_directory(self, path: Path | str, **kwargs: Any) -> List[IngestReport]:
+        """Ingest every text file under a directory."""
+        return self.ingestor.ingest_directory(path, **kwargs)
+
+    # ---------------------------------------------------------- consolidation
+    def consolidate(self, *, force: bool = False) -> ConsolidationReport:
+        """Run pending Summarization Events and rewrite the trunk."""
+        return self.summarizer.consolidate(force=force)
+
+    def flush(self) -> ConsolidationReport:
+        """Close the tree completely, promoting partial groups as well."""
+        return self.summarizer.flush()
+
+    # ------------------------------------------------------------- retrieval
+    def search(self, query: str, **kwargs: Any) -> SearchResponse:
+        """Walk the tree for ``query``."""
+        return self.navigator.search(query, **kwargs)
+
+    def ask(
+        self,
+        query: str,
+        clue_provider: Callable[[ClueRequest], Optional[str]],
+        **kwargs: Any,
+    ) -> SearchResponse:
+        """Search with the interactive clue loop enabled."""
+        return self.navigator.search_interactive(query, clue_provider, **kwargs)
+
+    def get_leaf(self, leaf_id: str) -> MemoryLeaf:
+        """Fetch one leaf by identifier."""
+        return self.store.load_leaf(leaf_id)
+
+    def outline(self, **kwargs: Any) -> List[str]:
+        """Readable outline of the tree."""
+        return self.navigator.outline(**kwargs)
+
+    # -------------------------------------------------------------- timeline
+    def record_fact(self, subject: str, attribute: str, value: str, **kwargs: Any):
+        """Add an assertion to the timeline."""
+        return self.resolver.record_fact(subject, attribute, value, **kwargs)
+
+    def conflicts(self, **kwargs: Any) -> List[Contradiction]:
+        """Every attribute whose value changed over time."""
+        return self.resolver.detect_conflicts(**kwargs)
+
+    def supersede(self, old_leaf_id: str, new_leaf_id: str) -> str:
+        """Record that one memory corrects another; return the user-facing notice."""
+        return self.resolver.supersede_leaf(old_leaf_id, new_leaf_id)
+
+    def notices(self, **kwargs: Any) -> List[str]:
+        """User-facing contradiction notices."""
+        return [c.render_notice() for c in self.resolver.detect_conflicts(**kwargs)]
+
+    # ------------------------------------------------------------ inspection
+    def stats(self) -> Dict[str, Any]:
+        """Counters describing the store."""
+        stats = self.store.stats()
+        stats["facts"] = len(self.resolver.facts)
+        stats["conflicts"] = len(self.resolver.detect_conflicts())
+        return stats
+
+    def rebuild(self) -> Dict[str, Any]:
+        """Rebuild the catalog, the timeline and the trunk from the files."""
+        self.store.rebuild_catalog()
+        facts = self.resolver.rebuild()
+        self.summarizer.rebuild_root()
+        return {"catalog_entries": len(self.store.catalog.entries), "facts": facts}
+
+
+__all__ = ["IHMT"]

ihmt/chunkers/__init__.py

@@ -0,0 +1,55 @@
+"""Type-aware splitters and the factory that selects one.
+
+Adding support for a new material type means adding a :class:`~ihmt.chunkers.base.Chunker`
+subclass and one entry in :func:`get_chunker` — nothing else in the framework
+changes.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from ..config import IHMTConfig
+from ..models import DataType
+from .base import Block, Chunk, Chunker
+from .code import BraceScanner, CodeAwareChunker
+from .generic import GenericChunker
+from .narrative import NarrativeChunker
+from .process import ProcessChunker
+from .temporal import TemporalChunker, extract_date, leading_date
+
+__all__ = [
+    "Block",
+    "Chunk",
+    "Chunker",
+    "BraceScanner",
+    "CodeAwareChunker",
+    "GenericChunker",
+    "NarrativeChunker",
+    "ProcessChunker",
+    "TemporalChunker",
+    "extract_date",
+    "get_chunker",
+    "leading_date",
+]
+
+
+def get_chunker(data_type: DataType, config: IHMTConfig, *, source: str = "") -> Chunker:
+    """Return the splitter responsible for ``data_type``.
+
+    Args:
+        data_type: Type decided by :class:`~ihmt.detectors.DomainDetector`.
+        config: Store configuration (token budgets).
+        source: Optional filename, used to pin the programming language.
+    """
+    if data_type is DataType.CODE:
+        return CodeAwareChunker(config, language=CodeAwareChunker.language_for(source))
+    if data_type is DataType.NARRATIVE:
+        return NarrativeChunker(config)
+    if data_type is DataType.CLINICAL:
+        return TemporalChunker(config, clinical=True)
+    if data_type is DataType.PERSONAL:
+        return TemporalChunker(config, clinical=False)
+    if data_type is DataType.PROCESS:
+        return ProcessChunker(config)
+    return GenericChunker(config)

ihmt/chunkers/base.py

@@ -0,0 +1,297 @@
+"""Chunker interface and the block-packing algorithm shared by all splitters.
+
+Every splitter works in two phases:
+
+1. **Block detection** — cut the document only at boundaries that are
+   *meaningful for its type*: a syntactic block for code, a paragraph or scene
+   for narrative, a dated entry for a journal or clinical note. A block is the
+   atomic unit: IHMT never cuts inside one.
+2. **Packing** (:meth:`Chunker.pack`) — greedily merge consecutive blocks up to
+   ``target_tokens``. If a single block exceeds ``max_tokens`` it is emitted
+   whole and flagged ``oversized``: block integrity outranks the token budget.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Sequence
+
+from ..config import IHMTConfig
+from ..textutils import estimate_tokens, truncate
+
+
+@dataclass
+class Block:
+    """An indivisible span of a source document.
+
+    Attributes:
+        text: Exact source text, byte-for-byte, including trailing newlines.
+        title: Human-readable label (``"class InventoryService"``, a scene
+            heading, a journal date...).
+        start_line: 1-based inclusive first line in the source.
+        end_line: 1-based inclusive last line in the source.
+        kind: Block flavour, e.g. ``"class"``, ``"method"``, ``"paragraph"``.
+        tags: Structural tags propagated to the resulting leaf.
+        timestamp: Date extracted from the content, when the type carries one.
+        hard_boundary: If true, this block never merges with the previous one
+            (a new scene, a new clinical encounter, a new journal date).
+        context: Enclosing context that is *not* part of the text, such as the
+            class header of an extracted method. Stored in leaf metadata so the
+            LLM sees it without corrupting the byte-exact content.
+    """
+
+    text: str
+    title: str = ""
+    start_line: int = 1
+    end_line: int = 1
+    kind: str = "block"
+    tags: List[str] = field(default_factory=list)
+    timestamp: Optional[str] = None
+    hard_boundary: bool = False
+    context: str = ""
+
+    @property
+    def tokens(self) -> int:
+        return estimate_tokens(self.text)
+
+    @property
+    def is_blank(self) -> bool:
+        return not self.text.strip()
+
+
+@dataclass
+class Chunk:
+    """A packed group of blocks, ready to become a :class:`~ihmt.models.MemoryLeaf`."""
+
+    text: str
+    title: str = ""
+    start_line: int = 1
+    end_line: int = 1
+    kind: str = "chunk"
+    tags: List[str] = field(default_factory=list)
+    timestamp: Optional[str] = None
+    oversized: bool = False
+    context: str = ""
+    meta: Dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def tokens(self) -> int:
+        return estimate_tokens(self.text)
+
+
+def split_lines(text: str) -> List[str]:
+    """Split into lines while keeping the line terminators (exact round-trip)."""
+    return text.splitlines(keepends=True)
+
+
+def join_lines(lines: Sequence[str], start_line: int, end_line: int) -> str:
+    """Join 1-based inclusive line range ``[start_line, end_line]``."""
+    return "".join(lines[start_line - 1 : end_line])
+
+
+class Chunker(ABC):
+    """Base class for every splitter.
+
+    Subclasses implement :meth:`blocks`; :meth:`chunk` packs those blocks into
+    leaf-sized units. Overriding :meth:`chunk` directly is allowed when a type
+    needs custom grouping (see the temporal splitter).
+    """
+
+    #: Short identifier recorded in leaf metadata.
+    name: str = "base"
+
+    def __init__(self, config: IHMTConfig) -> None:
+        self.config = config
+
+    @abstractmethod
+    def blocks(self, text: str, *, source: str = "") -> List[Block]:
+        """Detect the indivisible blocks of ``text``."""
+
+    def chunk(self, text: str, *, source: str = "") -> List[Chunk]:
+        """Split ``text`` into leaf-sized chunks. The public entry point."""
+        if not text.strip():
+            return []
+        return self.pack(self.blocks(text, source=source))
+
+    # ------------------------------------------------------------------ packing
+    def pack(self, blocks: Sequence[Block]) -> List[Chunk]:
+        """Merge consecutive blocks into chunks of roughly ``target_tokens``.
+
+        The packing rules, in priority order:
+
+        1. A block larger than ``max_tokens`` is emitted alone and flagged
+           ``oversized`` — it is never cut.
+        2. A block marked ``hard_boundary`` always starts a new chunk.
+        3. Otherwise blocks accumulate until adding one would exceed
+           ``target_tokens``, provided the current chunk already carries at
+           least ``min_tokens`` (this avoids emitting slivers).
+        """
+        target = self.config.target_tokens
+        maximum = self.config.max_tokens
+        minimum = self.config.min_tokens
+
+        chunks: List[Chunk] = []
+        current: List[Block] = []
+        current_tokens = 0
+
+        def flush() -> None:
+            nonlocal current, current_tokens
+            if current:
+                chunks.append(self._merge(current))
+                current = []
+                current_tokens = 0
+
+        for block in blocks:
+            if block.is_blank:
+                # Whitespace-only runs ride along with the preceding block so
+                # that concatenating chunks reproduces the source exactly.
+                if current:
+                    current.append(block)
+                else:
+                    current = [block]
+                    current_tokens = 0
+                continue
+
+            tokens = block.tokens
+            if tokens > maximum:
+                flush()
+                chunk = self._merge([block])
+                chunk.oversized = True
+                chunk.meta["oversized_reason"] = (
+                    f"indivisible {block.kind} of ~{tokens} tokens exceeds max_tokens={maximum}"
+                )
+                chunks.append(chunk)
+                continue
+
+            if current and (block.hard_boundary or (current_tokens + tokens > target and current_tokens >= minimum)):
+                flush()
+
+            current.append(block)
+            current_tokens += tokens
+
+        flush()
+        return chunks
+
+    def pack_each(self, blocks: Sequence[Block]) -> List[Chunk]:
+        """One chunk per block (symbol granularity).
+
+        Whitespace-only blocks ride along with the preceding block, exactly as
+        in :meth:`pack`, so concatenating the chunks reproduces the source.
+        Runs of *small* blocks of the same kind — import lines, fields, trivial
+        accessors — are grouped up to ``target_tokens``, so a lookup never has
+        to wade through a dozen one-line leaves.
+        """
+        small = max(24, self.config.target_tokens // 4)
+
+        def solid(group: List[Block]) -> List[Block]:
+            return [b for b in group if not b.is_blank]
+
+        groups: List[List[Block]] = []
+        leading: List[Block] = []
+        for block in blocks:
+            if block.is_blank:
+                if groups:
+                    groups[-1].append(block)
+                else:
+                    leading.append(block)
+                continue
+            if groups and not leading and block.tokens < small:
+                previous = solid(groups[-1])
+                if (
+                    previous
+                    and previous[-1].tokens < small
+                    and previous[-1].kind == block.kind
+                    and sum(b.tokens for b in previous) + block.tokens <= self.config.target_tokens
+                ):
+                    groups[-1].append(block)
+                    continue
+            groups.append(leading + [block])
+            leading = []
+        if leading:
+            groups.append(leading)
+
+        chunks: List[Chunk] = []
+        for group in groups:
+            chunk = self._merge(group)
+            big = [b for b in group if not b.is_blank and b.tokens > self.config.max_tokens]
+            if big:
+                chunk.oversized = True
+                chunk.meta["oversized_reason"] = (
+                    f"indivisible {big[0].kind} of ~{big[0].tokens} tokens exceeds max_tokens={self.config.max_tokens}"
+                )
+            chunks.append(chunk)
+        return chunks
+
+    def _merge(self, blocks: Sequence[Block]) -> Chunk:
+        """Fuse a run of blocks into a single chunk, preserving exact text."""
+        named = [b for b in blocks if b.title and not b.is_blank] or list(blocks)
+        if len(named) == 1:
+            title = named[0].title
+        else:
+            title = truncate(f"{named[0].title} → {named[-1].title}", 90)
+
+        tags: List[str] = []
+        for block in blocks:
+            for tag in block.tags:
+                if tag not in tags:
+                    tags.append(tag)
+
+        timestamps = [b.timestamp for b in blocks if b.timestamp]
+        contexts = [b.context for b in blocks if b.context]
+        kinds = {b.kind for b in blocks if not b.is_blank}
+
+        return Chunk(
+            text="".join(b.text for b in blocks),
+            title=title or "untitled",
+            start_line=blocks[0].start_line,
+            end_line=blocks[-1].end_line,
+            kind=kinds.pop() if len(kinds) == 1 else "mixed",
+            tags=tags,
+            timestamp=timestamps[0] if timestamps else None,
+            context=contexts[0] if contexts else "",
+            meta={"blocks": len(blocks), "chunker": self.name},
+        )
+
+    # ------------------------------------------------------------------ helpers
+    def _fallback_blocks(self, text: str) -> List[Block]:
+        """Line-based blocks used when no structure can be detected.
+
+        Cuts only at blank lines, so even the degenerate path never splits a
+        line in half.
+        """
+        lines = split_lines(text)
+        blocks: List[Block] = []
+        start = 1
+        for index, line in enumerate(lines, start=1):
+            if not line.strip() and index > start:
+                blocks.append(
+                    Block(
+                        text=join_lines(lines, start, index),
+                        title=self._first_meaningful_line(lines[start - 1 : index]),
+                        start_line=start,
+                        end_line=index,
+                        kind="segment",
+                    )
+                )
+                start = index + 1
+        if start <= len(lines):
+            blocks.append(
+                Block(
+                    text=join_lines(lines, start, len(lines)),
+                    title=self._first_meaningful_line(lines[start - 1 :]),
+                    start_line=start,
+                    end_line=len(lines),
+                    kind="segment",
+                )
+            )
+        return blocks
+
+    @staticmethod
+    def _first_meaningful_line(lines: Sequence[str], limit: int = 70) -> str:
+        """Title heuristic: the first non-empty line, trimmed."""
+        for line in lines:
+            stripped = line.strip()
+            if stripped:
+                return truncate(stripped, limit)
+        return ""