memgit 0.1.1__py3-none-any.whl

memgit/scorer.py

@@ -0,0 +1,123 @@
+"""BM25-style relevance scoring for memory search."""
+
+from __future__ import annotations
+import math
+import re
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .models import Mnemonic
+
+
+_PRIORITY_BOOST = {1: 0.8, 2: 1.0, 3: 1.3}
+
+
+@dataclass
+class ScoredMnemonic:
+    mnemonic: "Mnemonic"
+    score: float
+    matched_fields: list[str]
+
+
+def _tokenize(text: str) -> list[str]:
+    return re.findall(r"[a-z0-9]+", text.lower())
+
+
+def _field_tokens(m: "Mnemonic") -> dict[str, list[str]]:
+    """Return tokenized fields with their weights."""
+    return {
+        "slug": _tokenize(m.slug),
+        "rule": _tokenize(m.rule or ""),
+        "why": _tokenize(m.why or ""),
+        "when": _tokenize(m.when or ""),
+        "tags": _tokenize(" ".join(m.tags)),
+        "desc": _tokenize(m.desc or ""),
+    }
+
+
+# Field importance multipliers
+_FIELD_WEIGHT = {
+    "slug": 2.0,
+    "rule": 1.5,
+    "tags": 1.8,
+    "why": 1.0,
+    "when": 0.8,
+    "desc": 0.6,
+}
+
+# BM25 parameters
+_K1 = 1.5
+_B = 0.75
+
+
+def _avg_doc_len(mnemonics: list["Mnemonic"]) -> float:
+    if not mnemonics:
+        return 1.0
+    total = sum(
+        sum(len(toks) for toks in _field_tokens(m).values())
+        for m in mnemonics
+    )
+    return total / len(mnemonics)
+
+
+def score(
+    query: str,
+    mnemonics: list["Mnemonic"],
+    top_k: int = 10,
+) -> list[ScoredMnemonic]:
+    """Score mnemonics against query and return top-k by relevance."""
+    if not query.strip() or not mnemonics:
+        return []
+
+    query_terms = set(_tokenize(query))
+    if not query_terms:
+        return []
+
+    N = len(mnemonics)
+    avg_len = _avg_doc_len(mnemonics)
+
+    # Compute IDF per term across the corpus
+    df: dict[str, int] = {}
+    for m in mnemonics:
+        seen = set()
+        for toks in _field_tokens(m).values():
+            for tok in toks:
+                if tok in query_terms and tok not in seen:
+                    df[tok] = df.get(tok, 0) + 1
+                    seen.add(tok)
+
+    idf: dict[str, float] = {}
+    for term in query_terms:
+        n_t = df.get(term, 0)
+        idf[term] = math.log((N - n_t + 0.5) / (n_t + 0.5) + 1)
+
+    results: list[ScoredMnemonic] = []
+
+    for m in mnemonics:
+        fields = _field_tokens(m)
+        doc_len = sum(len(toks) for toks in fields.values())
+        score_val = 0.0
+        matched: list[str] = []
+
+        for term in query_terms:
+            for field_name, toks in fields.items():
+                tf = toks.count(term)
+                if tf == 0:
+                    continue
+                if field_name not in matched:
+                    matched.append(field_name)
+                weight = _FIELD_WEIGHT.get(field_name, 1.0)
+                norm_tf = (tf * (_K1 + 1)) / (
+                    tf + _K1 * (1 - _B + _B * doc_len / avg_len)
+                )
+                score_val += weight * idf.get(term, 0.0) * norm_tf
+
+        # Priority boost
+        score_val *= _PRIORITY_BOOST.get(m.priority, 1.0)
+
+        if score_val > 0:
+            results.append(ScoredMnemonic(m, round(score_val, 4), matched))
+
+    results.sort(key=lambda r: r.score, reverse=True)
+    return results[:top_k]

memgit/store.py

@@ -0,0 +1,176 @@
+"""Content-addressed object store.
+
+Objects are stored at .memgit/objects/{sha[0:2]}/{sha[2:4]}/{sha[4:]}
+Each file is gzip-compressed: first line is type, rest is TOON content.
+
+SHA computation per spec:
+  Mnemonic  → SHA-256(canonical TOON text)
+  MindState → SHA-256(sorted "slug:sha\\n" pairs)
+  Checkpoint → SHA-256("CKPT1\\n" + JSON of core fields)
+"""
+
+from __future__ import annotations
+import gzip
+import hashlib
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Union
+
+from .models import Checkpoint, DiffSummary, MindState, MindStateEntry, Mnemonic
+from .toon import (
+    format_ts,
+    parse_toon,
+    serialize_checkpoint,
+    serialize_mindstate,
+    serialize_mnemonic,
+)
+
+
+class ObjectStore:
+    def __init__(self, root: Path):
+        self.root = root
+        self.objects_dir = root / 'objects'
+
+    def _obj_path(self, sha: str) -> Path:
+        return self.objects_dir / sha[:2] / sha[2:4] / sha[4:]
+
+    def resolve_sha(self, abbrev: str) -> str | None:
+        """Resolve an abbreviated SHA (≥4 chars) to the full 64-char SHA.
+
+        Returns the full SHA if exactly one match, None if not found or ambiguous.
+        If abbrev is already 64 chars, returns it as-is.
+        """
+        if len(abbrev) >= 64:
+            return abbrev
+        if len(abbrev) < 4:
+            return None
+        prefix2 = abbrev[:2]
+        prefix4 = abbrev[2:4]
+        rest_prefix = abbrev[4:]
+        search_dir = self.objects_dir / prefix2 / prefix4
+        if not search_dir.exists():
+            return None
+        matches = [
+            prefix2 + prefix4 + p.name
+            for p in search_dir.iterdir()
+            if p.is_file() and p.name.startswith(rest_prefix)
+        ]
+        return matches[0] if len(matches) == 1 else None
+
+    def _write(self, sha: str, type_name: str, toon_content: str):
+        path = self._obj_path(sha)
+        if path.exists():
+            return  # content-addressed: same SHA = same content
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with gzip.open(path, 'wt', encoding='utf-8') as f:
+            f.write(f'{type_name}\n{toon_content}')
+
+    def _read(self, sha: str) -> tuple[str, str]:
+        path = self._obj_path(sha)
+        if not path.exists() and len(sha) < 64:
+            full = self.resolve_sha(sha)
+            if full:
+                path = self._obj_path(full)
+        with gzip.open(path, 'rt', encoding='utf-8') as f:
+            data = f.read()
+        idx = data.index('\n')
+        return data[:idx], data[idx + 1:]
+
+    def exists(self, sha: str) -> bool:
+        if self._obj_path(sha).exists():
+            return True
+        if len(sha) < 64:
+            return self.resolve_sha(sha) is not None
+        return False
+
+    # ── Mnemonic ──────────────────────────────────────────────────────────────
+
+    def mnemonic_sha(self, m: Mnemonic) -> str:
+        canonical = serialize_mnemonic(m, canonical=True)
+        return hashlib.sha256(canonical.encode('utf-8')).hexdigest()
+
+    def write_mnemonic(self, m: Mnemonic) -> str:
+        sha = self.mnemonic_sha(m)
+        m.sha = sha
+        canonical = serialize_mnemonic(m, canonical=True)
+        self._write(sha, 'mnem', canonical)
+        return sha
+
+    def read_mnemonic(self, sha: str) -> Mnemonic:
+        type_name, content = self._read(sha)
+        assert type_name == 'mnem', f'Expected mnem, got {type_name}'
+        objs = parse_toon(content)
+        if not objs:
+            raise ValueError(f'Failed to parse mnemonic {sha[:8]}')
+        m = objs[0]
+        assert isinstance(m, Mnemonic), f'Expected Mnemonic, got {type(m)}'
+        m.sha = sha
+        return m
+
+    # ── MindState ─────────────────────────────────────────────────────────────
+
+    def mindstate_sha(self, ms: MindState) -> str:
+        entries = sorted(ms.entries, key=lambda e: e.slug)
+        lines = [f'{e.slug}:{e.mnem_sha}' for e in entries]
+        content = '\n'.join(lines)
+        return hashlib.sha256(content.encode('utf-8')).hexdigest()
+
+    def write_mindstate(self, ms: MindState) -> str:
+        sha = self.mindstate_sha(ms)
+        ms.sha = sha
+        toon = serialize_mindstate(ms)
+        self._write(sha, 'ms', toon)
+        return sha
+
+    def read_mindstate(self, sha: str) -> MindState:
+        type_name, content = self._read(sha)
+        assert type_name == 'ms', f'Expected ms, got {type_name}'
+        objs = parse_toon(content)
+        if not objs:
+            return MindState(timestamp=datetime.now(timezone.utc), sha=sha)
+        ms = objs[0]
+        assert isinstance(ms, MindState), f'Expected MindState, got {type(ms)}'
+        ms.sha = sha
+        return ms
+
+    # ── Checkpoint ────────────────────────────────────────────────────────────
+
+    def checkpoint_sha(self, ck: Checkpoint) -> str:
+        data = {
+            'parent_sha': ck.parent_sha,
+            'mindstate_sha': ck.mindstate_sha,
+            'timestamp': format_ts(ck.timestamp),
+            'trigger': ck.trigger,
+            'message': ck.message,
+            'author': ck.author,
+        }
+        content = 'CKPT1\n' + json.dumps(data, sort_keys=True)
+        return hashlib.sha256(content.encode('utf-8')).hexdigest()
+
+    def write_checkpoint(self, ck: Checkpoint) -> str:
+        sha = self.checkpoint_sha(ck)
+        ck.sha = sha
+        toon = serialize_checkpoint(ck)
+        self._write(sha, 'ck', toon)
+        return sha
+
+    def read_checkpoint(self, sha: str) -> Checkpoint:
+        type_name, content = self._read(sha)
+        assert type_name == 'ck', f'Expected ck, got {type_name}'
+        objs = parse_toon(content)
+        if not objs:
+            raise ValueError(f'Failed to parse checkpoint {sha[:8]}')
+        ck = objs[0]
+        assert isinstance(ck, Checkpoint), f'Expected Checkpoint, got {type(ck)}'
+        ck.sha = sha  # override with full sha from the store path
+        return ck
+
+    # ── Stats ─────────────────────────────────────────────────────────────────
+
+    def object_count(self) -> int:
+        count = 0
+        for p in self.objects_dir.rglob('*'):
+            if p.is_file():
+                count += 1
+        return count

memgit/tokens.py

@@ -0,0 +1,48 @@
+"""Token counting utilities — approximation only, no external dependencies.
+
+Uses a character-based model calibrated against GPT-4 tokenizer averages:
+  - ~4 chars/token for English prose
+  - Code/slugs are slightly denser (~3.5 chars/token)
+  - Good enough for the 3–5x comparisons we display in `memgit stats`
+"""
+
+from __future__ import annotations
+import re
+
+
+def count_tokens(text: str) -> int:
+    """Approximate token count for `text` using a char-density model."""
+    if not text:
+        return 0
+    # Strip whitespace normalization
+    text = text.strip()
+    # Count whitespace-separated tokens (rough word count)
+    words = len(re.findall(r'\S+', text))
+    # Each word averages ~1.3 tokens (handles punctuation, subwords, numbers)
+    return max(1, round(words * 1.3))
+
+
+def memory_tokens(m) -> int:
+    """Token cost of a single Mnemonic as context."""
+    from .toon import serialize_mnemonic
+    return count_tokens(serialize_mnemonic(m))
+
+
+def all_memories_tokens(mnemonics: list) -> int:
+    """Token cost of loading ALL memories (the claude.md / dump approach)."""
+    return sum(memory_tokens(m) for m in mnemonics)
+
+
+def search_tokens(scored: list, query: str) -> int:
+    """Token cost of a search result set (top-k relevance approach)."""
+    return sum(memory_tokens(r.mnemonic) for r in scored)
+
+
+# GPT-4o pricing (input, per million tokens) as of 2026
+_GPT4O_PER_MTK = 5.0   # $5/1M tokens
+_CLAUDE_SONNET_PER_MTK = 3.0  # $3/1M tokens
+
+
+def token_cost_usd(tokens: int, model: str = 'gpt4o') -> float:
+    rate = _CLAUDE_SONNET_PER_MTK if model == 'claude' else _GPT4O_PER_MTK
+    return tokens * rate / 1_000_000

memgit/toon.py

@@ -0,0 +1,356 @@
+"""TOON format parser and serializer.
+
+TOON — Thought Object Observation Notation
+Line-oriented, sigil-prefixed format purpose-built for AI memory objects.
+~45-55% fewer tokens than equivalent markdown.
+"""
+
+from __future__ import annotations
+import re
+from datetime import datetime, timezone
+from typing import Union
+
+from .models import Mnemonic, MindState, MindStateEntry, Checkpoint, DiffSummary
+
+USER_TYPE_CODES = {"fb", "us", "pj", "rf", "cn", "lx"}
+
+
+def _parse_ts(ts_str: str) -> datetime:
+    """Parse ISO 8601 compact UTC timestamp."""
+    s = ts_str.rstrip('Z')
+    try:
+        if 'T' in s:
+            # Normalize: 2026-06-14T08:22 → 2026-06-14T08:22:00
+            if len(s) == 16:
+                s += ':00'
+            return datetime.fromisoformat(s).replace(tzinfo=timezone.utc)
+    except ValueError:
+        pass
+    return datetime.now(timezone.utc)
+
+
+def format_ts(dt: datetime) -> str:
+    """Format datetime to TOON compact UTC: 2026-06-14T08:22Z"""
+    return dt.strftime('%Y-%m-%dT%H:%MZ')
+
+
+def parse_toon(text: str) -> list[Union[Mnemonic, MindState, Checkpoint]]:
+    """Parse a TOON file into a list of objects."""
+    text = text.replace('\r\n', '\n').replace('\r', '\n')
+    blocks = re.split(r'\n{2,}', text.strip())
+    results = []
+    for block in blocks:
+        block = block.strip()
+        if not block:
+            continue
+        obj = _parse_block(block)
+        if obj is not None:
+            results.append(obj)
+    return results
+
+
+def _parse_block(block: str) -> Union[Mnemonic, MindState, Checkpoint, None]:
+    lines = block.split('\n')
+    if not lines:
+        return None
+
+    header = lines[0]
+    if not header.startswith('TOON1|'):
+        return None
+
+    parts = header.split('|')
+    if len(parts) < 4:
+        return None
+
+    type_code = parts[1]
+    slug = parts[2]
+    timestamp = _parse_ts(parts[3])
+    flags_str = parts[4] if len(parts) > 4 else ''
+    priority = 2
+    if flags_str.startswith('!'):
+        try:
+            priority = int(flags_str[1:])
+        except ValueError:
+            pass
+
+    field_lines = lines[1:]
+
+    if type_code == 'ms':
+        return _parse_ms(field_lines, timestamp, slug)
+    elif type_code == 'ck':
+        return _parse_ck(field_lines, timestamp, slug)
+    elif type_code in USER_TYPE_CODES:
+        return _parse_mnemonic(field_lines, type_code, slug, timestamp, priority)
+    return None
+
+
+def _parse_ms(lines: list[str], timestamp: datetime, slug: str) -> MindState:
+    entries = []
+    for line in lines:
+        line = line.strip()
+        if line.startswith('ENTRY:'):
+            rest = line[6:]
+            if ':' in rest:
+                idx = rest.index(':')
+                s = rest[:idx].strip()
+                h = rest[idx+1:].strip()
+                entries.append(MindStateEntry(slug=s, mnem_sha=h))
+    ms = MindState(timestamp=timestamp, entries=entries)
+    ms.sha = slug  # slug field stores sha[:16] for internal objects
+    return ms
+
+
+def _parse_ck(lines: list[str], timestamp: datetime, slug: str) -> Checkpoint:
+    kv: dict[str, str] = {}
+    added, updated, removed = [], [], []
+
+    for line in lines:
+        line = line.strip()
+        if not line:
+            continue
+        if line.startswith('+'):
+            rest = line[1:]
+            if ':' in rest:
+                k, v = rest.split(':', 1)
+                k = k.strip().upper()
+                v = v.strip()
+                if k == 'ADD':
+                    added.append(v)
+                elif k == 'UPD':
+                    updated.append(v)
+                elif k == 'REM':
+                    removed.append(v)
+        elif ':' in line:
+            k, v = line.split(':', 1)
+            kv[k.strip().upper()] = v.strip()
+
+    ck = Checkpoint(
+        mindstate_sha=kv.get('MSTATE', ''),
+        timestamp=timestamp,
+        trigger=kv.get('TRIGGER', 'explicit'),
+        message=kv.get('MSG', ''),
+        author=kv.get('AUTHOR', ''),
+        session_id=kv.get('SESSION', ''),
+        parent_sha=kv.get('PARENT') or None,
+        diff_summary=DiffSummary(added=added, modified=updated, removed=removed),
+    )
+    ck.sha = slug
+    return ck
+
+
+def _parse_mnemonic(
+    lines: list[str],
+    type_code: str,
+    slug: str,
+    timestamp: datetime,
+    priority: int,
+) -> Mnemonic:
+    tags: list[str] = []
+    rule = None
+    why = who = when = desc = where = dl = inc = cost = source = None
+    supersedes: list[str] = []
+    related: list[str] = []
+
+    for line in lines:
+        line = line.strip()
+        if not line:
+            continue
+
+        if line.startswith('#'):
+            for tag in line.split():
+                t = tag.lstrip('#').strip()
+                if t:
+                    tags.append(t)
+        elif line.startswith('~'):
+            rest = line[1:]
+            if ':' in rest:
+                k, v = rest.split(':', 1)
+                k = k.strip().upper()
+                v = v.strip()
+                if k == 'SUP':
+                    supersedes = [s.strip() for s in v.split(',') if s.strip()]
+                elif k == 'REL':
+                    related = [s.strip() for s in v.split(',') if s.strip()]
+                elif k == 'SRC':
+                    source = v
+        elif ':' in line:
+            k, v = line.split(':', 1)
+            k = k.strip().upper()
+            v = v.strip()
+            if k == 'RULE':
+                rule = v
+            elif k == 'WHY':
+                why = v
+            elif k == 'WHEN':
+                when = v
+            elif k == 'DESC':
+                desc = v
+            elif k == 'WHO':
+                who = v
+            elif k == 'WHERE':
+                where = v
+            elif k == 'DL':
+                dl = v
+            elif k == 'INC':
+                inc = v
+            elif k == 'COST':
+                cost = v
+
+    return Mnemonic(
+        type_code=type_code,
+        slug=slug,
+        timestamp=timestamp,
+        rule=rule or desc or '',
+        priority=priority,
+        tags=tags,
+        why=why,
+        when=when,
+        desc=desc,
+        who=who,
+        where=where,
+        dl=dl,
+        inc=inc,
+        cost=cost,
+        supersedes=supersedes,
+        related=related,
+        source=source,
+    )
+
+
+def serialize_mnemonic(m: Mnemonic, canonical: bool = False) -> str:
+    """Serialize Mnemonic to TOON.
+
+    canonical=True: sorted fields (used for SHA computation).
+    canonical=False: human-friendly output order.
+    """
+    flags = f'|!{m.priority}' if m.priority == 3 else ''
+    header = f'TOON1|{m.type_code}|{m.slug}|{format_ts(m.timestamp)}{flags}'
+    lines = [header]
+
+    if canonical:
+        # Deterministic field order for SHA: alphabetical by sigil
+        fields: list[tuple[str, str]] = []
+        if m.cost:
+            fields.append(('COST', m.cost))
+        if m.desc:
+            fields.append(('DESC', m.desc))
+        if m.dl:
+            fields.append(('DL', m.dl))
+        if m.inc:
+            fields.append(('INC', m.inc))
+        fields.append(('RULE', m.rule))
+        if m.tags:
+            fields.append(('TAGS', ' '.join(sorted(m.tags))))
+        if m.when:
+            fields.append(('WHEN', m.when))
+        if m.where:
+            fields.append(('WHERE', m.where))
+        if m.who:
+            fields.append(('WHO', m.who))
+        if m.why:
+            fields.append(('WHY', m.why))
+        if m.related:
+            fields.append(('~REL', ','.join(sorted(m.related))))
+        if m.source:
+            fields.append(('~SRC', m.source))
+        if m.supersedes:
+            fields.append(('~SUP', ','.join(sorted(m.supersedes))))
+
+        for k, v in fields:
+            if k == 'TAGS':
+                lines.append(f'#{v}')
+            elif k.startswith('~'):
+                lines.append(f'{k}:{v}')
+            else:
+                lines.append(f'{k}:{v}')
+    else:
+        if m.tags:
+            lines.append('#' + ' #'.join(m.tags))
+        lines.append(f'RULE:{m.rule}')
+        if m.why:
+            lines.append(f'WHY:{m.why}')
+        if m.when:
+            lines.append(f'WHEN:{m.when}')
+        if m.desc:
+            lines.append(f'DESC:{m.desc}')
+        if m.who:
+            lines.append(f'WHO:{m.who}')
+        if m.where:
+            lines.append(f'WHERE:{m.where}')
+        if m.dl:
+            lines.append(f'DL:{m.dl}')
+        if m.inc:
+            lines.append(f'INC:{m.inc}')
+        if m.cost:
+            lines.append(f'COST:{m.cost}')
+        if m.supersedes:
+            lines.append(f'~SUP:{",".join(m.supersedes)}')
+        if m.related:
+            lines.append(f'~REL:{",".join(m.related)}')
+        if m.source:
+            lines.append(f'~SRC:{m.source}')
+
+    return '\n'.join(lines)
+
+
+def serialize_mindstate(ms: MindState) -> str:
+    """Serialize MindState to TOON."""
+    slug_field = ms.sha[:16] if ms.sha else '0' * 16
+    lines = [
+        f'TOON1|ms|{slug_field}|{format_ts(ms.timestamp)}',
+        f'COUNT:{ms.count}',
+    ]
+    for e in sorted(ms.entries, key=lambda e: e.slug):
+        lines.append(f'ENTRY:{e.slug}:{e.mnem_sha}')
+    return '\n'.join(lines)
+
+
+def serialize_checkpoint(ck: Checkpoint) -> str:
+    """Serialize Checkpoint to TOON."""
+    slug_field = ck.sha[:16] if ck.sha else '0' * 16
+    lines = [f'TOON1|ck|{slug_field}|{format_ts(ck.timestamp)}']
+    if ck.parent_sha:
+        lines.append(f'PARENT:{ck.parent_sha}')
+    lines.append(f'MSTATE:{ck.mindstate_sha}')
+    lines.append(f'TRIGGER:{ck.trigger}')
+    lines.append(f'MSG:{ck.message}')
+    if ck.author:
+        lines.append(f'AUTHOR:{ck.author}')
+    if ck.session_id:
+        lines.append(f'SESSION:{ck.session_id}')
+    if ck.diff_summary:
+        d = ck.diff_summary
+        for s in d.added:
+            lines.append(f'+ADD:{s}')
+        for s in d.modified:
+            lines.append(f'+UPD:{s}')
+        for s in d.removed:
+            lines.append(f'+REM:{s}')
+    return '\n'.join(lines)
+
+
+def mnemonic_to_markdown(m: Mnemonic) -> str:
+    """Convert a Mnemonic back to Claude Code markdown format."""
+    type_map = {
+        'fb': 'feedback', 'us': 'user', 'pj': 'project',
+        'rf': 'reference', 'cn': 'convention', 'lx': 'lesson',
+    }
+    type_str = type_map.get(m.type_code, 'feedback')
+    desc = m.rule[:120]
+
+    lines = [
+        '---',
+        f'name: {m.slug}',
+        f'description: {desc}',
+        'metadata:',
+        f'  type: {type_str}',
+        '---',
+        '',
+        m.rule,
+        '',
+    ]
+    if m.why:
+        lines += [f'**Why:** {m.why}', '']
+    if m.when:
+        lines += [f'**How to apply:** {m.when}', '']
+    return '\n'.join(lines)