abstractnote 0.1.0__py3-none-any.whl

abstractnote/ids.py

@@ -0,0 +1,177 @@
+"""Deterministic identifiers and time normalisation (docs/data-model.md).
+
+Every id is derived from its documented inputs with sha256, so replays,
+retries and rebuilds produce the same ids. Prefixes: ``nt_`` notes, ``at_``
+attachments, ``cl_`` claims, ``rv_`` revisions, ``tk_`` tasks, ``ka_`` the
+non-claim AbstractMemory assertions (names, aliases, merges). Entity ids follow
+AbstractFramework ADR 0025: ``ex:<kind>-<slug>``.
+
+All stored times are ISO 8601 UTC with second precision and a ``Z`` suffix
+(``2026-10-19T09:12:00Z``) so that string comparison is time comparison, in
+the note index and in AbstractMemory alike.
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import re
+import unicodedata
+from datetime import date, datetime, timezone
+from typing import Any, Iterable, Optional
+
+_SEP = "\x1f"
+
+ENTITY_KINDS = ("person", "org", "project", "concept", "place", "event", "artifact", "software", "thing")
+
+
+def digest(*parts: Any) -> str:
+    """sha256 hex of the parts joined by an ASCII unit separator."""
+    h = hashlib.sha256(_SEP.join("" if p is None else str(p) for p in parts).encode("utf-8"))
+    return h.hexdigest()
+
+
+def _b32(*parts: Any, length: int = 26) -> str:
+    raw = hashlib.sha256(_SEP.join("" if p is None else str(p) for p in parts).encode("utf-8")).digest()
+    return base64.b32encode(raw).decode("ascii").rstrip("=").lower()[:length]
+
+
+def sha256_bytes(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def sha256_text(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+# --------------------------------------------------------------------------- time
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def normalize_time(value: Any, *, field: str = "time") -> str:
+    """Normalise an ISO 8601 value (or datetime/date) to ``YYYY-MM-DDTHH:MM:SSZ`` UTC.
+
+    A date alone means midnight UTC; a naive datetime is taken as UTC.
+    Raises ValueError with a plain sentence on anything else.
+    """
+    if isinstance(value, datetime):
+        dt = value
+    elif isinstance(value, date):
+        dt = datetime(value.year, value.month, value.day, tzinfo=timezone.utc)
+    elif isinstance(value, str) and value.strip():
+        raw = value.strip()
+        if raw.endswith(("Z", "z")):
+            raw = raw[:-1] + "+00:00"
+        try:
+            dt = datetime.fromisoformat(raw)
+        except ValueError:
+            try:
+                d = date.fromisoformat(raw)
+            except ValueError:
+                raise ValueError(f"The {field} {value!r} is not an ISO 8601 date or time.") from None
+            dt = datetime(d.year, d.month, d.day, tzinfo=timezone.utc)
+    else:
+        raise ValueError(f"The {field} must be an ISO 8601 date or time.")
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def optional_time(value: Any, *, field: str = "time") -> Optional[str]:
+    if value is None or (isinstance(value, str) and not value.strip()):
+        return None
+    return normalize_time(value, field=field)
+
+
+def parse_time(value: str) -> datetime:
+    return datetime.strptime(normalize_time(value), "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
+
+
+def to_minute(value: str) -> str:
+    """``occurred_at`` truncated to the minute (the capture fallback key)."""
+    return normalize_time(value)[:16]
+
+
+# --------------------------------------------------------------------------- notes
+
+def note_id_for_import(importer: str, source_key: str) -> str:
+    return "nt_" + _b32("import", importer, source_key)
+
+
+def note_id_for_key(idempotency_key: str) -> str:
+    return "nt_" + _b32("key", idempotency_key)
+
+
+def note_id_fallback(text_norm: str, attachment_hashes: Iterable[str], occurred_at: str, client: str) -> str:
+    return "nt_" + _b32("capture", text_norm, ",".join(sorted(attachment_hashes)), to_minute(occurred_at), client)
+
+
+def note_id_for_insight(kind: str, title: str, body: str, evidence_note_ids: Iterable[str]) -> str:
+    return "nt_" + _b32("insight", kind, title, body, ",".join(sorted(evidence_note_ids)))
+
+
+def attachment_id(note_id: str, sha256: str) -> str:
+    return "at_" + _b32(note_id, sha256)
+
+
+def claim_id(note_id: str, subject: str, predicate: str, obj: str) -> str:
+    return "cl_" + _b32(note_id, subject, predicate, obj)
+
+
+def revision_id_for_note(note_id: str, attempt: int) -> str:
+    return "rv_" + _b32("note", note_id, int(attempt))
+
+
+def revision_id_for_action(action: str, target: str, occurred_at: str) -> str:
+    return "rv_" + _b32("action", action, target, occurred_at)
+
+
+def task_id(note_id: str, text_norm: str) -> str:
+    return "tk_" + _b32(note_id, text_norm)
+
+
+def kg_assertion_id(subject: str, predicate: str, obj: str) -> str:
+    """Ids of the non-claim assertions (schema:name, skos:altLabel, schema:sameAs)."""
+    return "ka_" + _b32(subject, predicate, obj)
+
+
+def closure_id(revision_id: str, assertion_id: str, kind: str) -> str:
+    return "cz_" + _b32(revision_id, assertion_id, kind)
+
+
+# --------------------------------------------------------------------------- entities
+
+def slugify(name: str, *, max_len: int = 48) -> str:
+    """ASCII slug of a model-chosen name (formatting only; identity is decided by the model)."""
+    folded = unicodedata.normalize("NFKD", str(name or "")).encode("ascii", "ignore").decode("ascii")
+    slug = re.sub(r"[^a-z0-9]+", "-", folded.lower()).strip("-")
+    if len(slug) > max_len:
+        slug = slug[:max_len].rstrip("-")
+    return slug or "unnamed"
+
+
+def entity_id_base(kind: str, name: str) -> str:
+    if kind not in ENTITY_KINDS:
+        raise ValueError(f"The entity kind {kind!r} is not one of {', '.join(ENTITY_KINDS)}.")
+    return f"ex:{kind}-{slugify(name)}"
+
+
+def mint_entity_id(kind: str, name: str, taken: "set[str] | frozenset[str] | Any") -> str:
+    """``ex:<kind>-<slug>``, with ``-2``, ``-3`` ... when the id is already taken.
+
+    ``taken`` is any container answering ``in`` (ids are never reused).
+    """
+    base = entity_id_base(kind, name)
+    if base not in taken:
+        return base
+    n = 2
+    while f"{base}-{n}" in taken:
+        n += 1
+    return f"{base}-{n}"
+
+
+def fold(text: str) -> str:
+    """Case-folded, whitespace-collapsed form used for exact alias matching."""
+    return " ".join(str(text or "").casefold().split())

abstractnote/importers/__init__.py

@@ -0,0 +1,142 @@
+"""Importers: existing corpora into a notebook with faithful history time and provenance (docs/importers.md).
+
+Each importer yields ``NoteDraft`` records; ``write_drafts`` captures them through the core lane's
+``Notebook.capture`` at bulk priority (``0``) with ``importer`` + ``source_key`` (note id =
+``sha256(importer, source_key)``), so a re-run creates nothing new and live notes always go first.
+Processing happens later through the queue (the ``note-drain`` flow), never here.
+
+Noise filtering is structural: record types, flags and the harness's own envelope markers decide
+what is machine output (tool results, hook output, system reminders, alerts), never the meaning of
+the words (ADR 0004).
+"""
+
+from __future__ import annotations
+
+import datetime as _dt
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional
+
+BULK_PRIORITY = 0
+
+
+@dataclass
+class NoteDraft:
+    """One note an importer wants to write."""
+
+    importer: str
+    source_key: str
+    text: str
+    occurred_at: str
+    kind: str  # what it is in the source (prompt, summary, subagent, memory, decision, thread, ...), for counts
+    title: Optional[str] = None
+    source: Dict[str, Any] = field(default_factory=dict)
+    attachments: List[Dict[str, Any]] = field(default_factory=list)  # {"data": bytes, "mime"?, "filename"?}
+
+
+@dataclass
+class ImportReport:
+    importer: str
+    dry_run: bool = False
+    created: int = 0
+    duplicates: int = 0
+    skipped: Dict[str, int] = field(default_factory=dict)  # noise and filtered records, by reason
+    attachments: int = 0
+    by_kind: Dict[str, int] = field(default_factory=dict)
+    earliest: Optional[str] = None
+    latest: Optional[str] = None
+    errors: List[Dict[str, str]] = field(default_factory=list)
+
+    def skip(self, reason: str, n: int = 1) -> None:
+        self.skipped[reason] = self.skipped.get(reason, 0) + n
+
+    def seen(self, draft: NoteDraft) -> None:
+        self.by_kind[draft.kind] = self.by_kind.get(draft.kind, 0) + 1
+        if self.earliest is None or draft.occurred_at < self.earliest:
+            self.earliest = draft.occurred_at
+        if self.latest is None or draft.occurred_at > self.latest:
+            self.latest = draft.occurred_at
+
+    def as_dict(self) -> Dict[str, Any]:
+        return {
+            "importer": self.importer, "dry_run": self.dry_run, "created": self.created,
+            "duplicates": self.duplicates, "attachments": self.attachments, "by_kind": dict(sorted(self.by_kind.items())),
+            "skipped": dict(sorted(self.skipped.items())), "range": {"earliest": self.earliest, "latest": self.latest},
+            "errors": list(self.errors),
+        }
+
+
+def iso_from_epoch(value: Any) -> str:
+    return _dt.datetime.fromtimestamp(float(value), tz=_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def iso_utc(value: str) -> str:
+    """An ISO 8601 timestamp (any offset, ``Z`` accepted) as UTC ``YYYY-MM-DDTHH:MM:SSZ``."""
+    text = str(value).strip().replace("Z", "+00:00")
+    dt = _dt.datetime.fromisoformat(text)
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=_dt.timezone.utc)
+    return dt.astimezone(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def in_window(occurred_at: str, since: Optional[str], until: Optional[str]) -> bool:
+    if since and occurred_at < iso_utc(since):
+        return False
+    if until and occurred_at >= iso_utc(until):
+        return False
+    return True
+
+
+def write_drafts(nb: Any, drafts: Iterable[NoteDraft], report: ImportReport, *, dry_run: bool = False,
+                 exists: Optional[Callable[[str, str], bool]] = None) -> ImportReport:
+    """Capture every draft (bulk priority) or, with ``dry_run``, only count what would happen.
+
+    ``nb`` may be None for a dry run on a notebook that does not exist yet (everything counts as new).
+    """
+    from abstractnote import ids  # core lane
+
+    report.dry_run = dry_run
+    for draft in drafts:
+        report.seen(draft)
+        if dry_run:
+            note_id = ids.note_id_for_import(draft.importer, draft.source_key)
+            known = nb is not None and (exists(draft.importer, draft.source_key) if exists else nb.note(note_id) is not None)
+            if known:
+                report.duplicates += 1
+            else:
+                report.created += 1
+                report.attachments += len(draft.attachments)
+            continue
+        try:
+            result = nb.capture(
+                draft.text, title=draft.title, occurred_at=draft.occurred_at, source=draft.source,
+                priority=BULK_PRIORITY, attachments=draft.attachments, importer=draft.importer,
+                source_key=draft.source_key,
+            )
+        except ValueError as e:
+            report.errors.append({"source_key": draft.source_key, "error": str(e)})
+            continue
+        if getattr(result, "status", None) == "duplicate":
+            report.duplicates += 1
+        else:
+            report.created += 1
+            report.attachments += len(draft.attachments)
+    return report
+
+
+def run(importer: str, nb: Any, *, dry_run: bool = False, **options: Any) -> ImportReport:
+    """Run one importer by name (``claude-sessions``, ``agora``, ``markdown``, ``files``)."""
+    from . import agora, claude_sessions, files, markdown
+
+    makers: Dict[str, Callable[..., Iterator[NoteDraft]]] = {
+        claude_sessions.IMPORTER: claude_sessions.drafts,
+        agora.IMPORTER: agora.drafts,
+        markdown.IMPORTER: markdown.drafts,
+        files.IMPORTER: files.drafts,
+    }
+    if importer not in makers:
+        raise ValueError(f"Unknown importer {importer!r}; choose one of {', '.join(sorted(makers))}.")
+    report = ImportReport(importer=importer)
+    return write_drafts(nb, makers[importer](report=report, **options), report, dry_run=dry_run)
+
+
+__all__ = ["BULK_PRIORITY", "ImportReport", "NoteDraft", "in_window", "iso_from_epoch", "iso_utc", "run", "write_drafts"]

abstractnote/importers/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main(prog="python -m abstractnote.importers"))

abstractnote/importers/agora.py

@@ -0,0 +1,203 @@
+"""An agora hub database -> notes (docs/importers.md, recon data-sources.md section B).
+
+One note per recorded decision (``store`` key ``decision:*``), per claim and work entry
+(``claim:*``, ``work:*``), per resolved or answered thread (the root message plus its replies, in
+order), and per operator message that is not already part of an imported thread.
+``occurred_at`` is the message's ``created_at`` (a store entry takes its message's time when it
+names one, else its ``updated_at``). The database is opened read-only.
+
+Skipped as noise: system and file-change messages (``kind`` system/fs), the ``hub-alerts`` channel,
+silence alerts, retracted messages. ``--channels`` keeps only the named channels.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from pathlib import Path
+from typing import Any, Dict, Iterator, List, Optional, Sequence
+
+from . import ImportReport, NoteDraft, in_window, iso_from_epoch
+
+IMPORTER = "agora"
+DEFAULT_OPERATORS = ("laurent", "dl-owner")
+NOISE_CHANNELS = ("hub-alerts",)
+STORE_PREFIXES = {"decision:": "decision", "claim:": "claim", "work:": "work"}
+
+
+def _connect(db: Path) -> sqlite3.Connection:
+    if not db.is_file():
+        raise ValueError(f"There is no agora database at {db}.")
+    conn = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
+    conn.row_factory = sqlite3.Row
+    return conn
+
+
+def _data(raw: Any) -> Dict[str, Any]:
+    if not raw:
+        return {}
+    try:
+        value = json.loads(raw)
+    except (TypeError, json.JSONDecodeError):
+        return {}
+    return value if isinstance(value, dict) else {}
+
+
+def _source(row: sqlite3.Row, *, extra: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+    origin = {"message_id": row["id"], "channel": row["channel"], "seq": row["seq"], "sender": row["sender"],
+              "reply_to": row["reply_to"], "status": row["status"], "hash": row["hash"]}
+    origin.update(extra or {})
+    return {"client": "importer", "importer": IMPORTER, "author": row["sender"],
+            "origin": {k: v for k, v in origin.items() if v is not None}}
+
+
+def _message_text(row: sqlite3.Row) -> str:
+    title = str(row["title"] or "").strip()
+    body = str(row["body"] or "").strip()
+    return f"# {title}\n\n{body}" if title and body else (title or body)
+
+
+def _attachments(conn: sqlite3.Connection, row: sqlite3.Row) -> List[Dict[str, Any]]:
+    out = []
+    for att in _data(row["data"]).get("attachments") or []:
+        if not isinstance(att, dict) or not att.get("id"):
+            continue
+        blob = conn.execute("SELECT bytes, filename, content_type FROM blobs WHERE id=? LIMIT 1", (att["id"],)).fetchone()
+        if blob is None or blob["bytes"] is None:
+            continue
+        out.append({"data": bytes(blob["bytes"]), "mime": att.get("content_type") or blob["content_type"],
+                    "filename": att.get("filename") or blob["filename"]})
+    return out
+
+
+def _is_noise(row: sqlite3.Row, report: ImportReport, *, skip_system: bool) -> bool:
+    if row["retracted_at"] is not None:
+        report.skip("retracted")
+        return True
+    if skip_system and row["kind"] in ("system", "fs"):
+        report.skip(f"kind_{row['kind']}")
+        return True
+    if skip_system and row["channel"] in NOISE_CHANNELS:
+        report.skip("hub_alerts")
+        return True
+    if skip_system and _data(row["data"]).get("silence_alert"):
+        report.skip("silence_alert")
+        return True
+    return False
+
+
+def drafts(*, report: ImportReport, path: str | Path, channels: Optional[Sequence[str]] = None,
+           skip_system: bool = True, operators: Sequence[str] = DEFAULT_OPERATORS,
+           since: Optional[str] = None, until: Optional[str] = None) -> Iterator[NoteDraft]:
+    conn = _connect(Path(path).expanduser())
+    keep = {c.strip() for c in (channels or []) if c and c.strip()} or None
+    try:
+        yield from _store_drafts(conn, report, keep, since, until)
+        yield from _thread_and_operator_drafts(conn, report, keep, skip_system, set(operators), since, until)
+    finally:
+        conn.close()
+
+
+def _store_drafts(conn: sqlite3.Connection, report: ImportReport, keep: Optional[set], since: Optional[str],
+                  until: Optional[str]) -> Iterator[NoteDraft]:
+    rows = conn.execute(
+        "SELECT channel, key, value, version, updated_by, updated_at FROM store "
+        "WHERE key LIKE 'decision:%' OR key LIKE 'claim:%' OR key LIKE 'work:%' ORDER BY updated_at")
+    for row in rows:
+        if keep is not None and row["channel"] not in keep:
+            report.skip("channel_filtered")
+            continue
+        kind = next(v for p, v in STORE_PREFIXES.items() if row["key"].startswith(p))
+        try:
+            value = json.loads(row["value"])
+        except (TypeError, json.JSONDecodeError):
+            value = row["value"]
+        message = None
+        if isinstance(value, dict) and value.get("message_id"):
+            message = conn.execute("SELECT * FROM messages WHERE id=?", (value["message_id"],)).fetchone()
+        if isinstance(value, dict):
+            summary = str(value.get("summary") or value.get("text") or value.get("title") or "").strip()
+            details = json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True)
+        else:
+            summary, details = str(value or "").strip(), ""
+        parts = [summary] if summary else []
+        if message is not None:
+            parts.append("## From the message\n\n" + _message_text(message))
+        if details and not summary:
+            parts.append(details)
+        text = "\n\n".join(p for p in parts if p).strip()
+        if not text:
+            report.skip("empty")
+            continue
+        ts = iso_from_epoch(message["created_at"]) if message is not None else iso_from_epoch(row["updated_at"])
+        if not in_window(ts, since, until):
+            report.skip("outside_window")
+            continue
+        origin = {"channel": row["channel"], "store_key": row["key"], "version": row["version"],
+                  "updated_by": row["updated_by"], "message_id": message["id"] if message is not None else None}
+        source = {"client": "importer", "importer": IMPORTER, "author": row["updated_by"],
+                  "origin": {k: v for k, v in origin.items() if v is not None}}
+        yield NoteDraft(IMPORTER, f"store:{row['channel']}:{row['key']}:{row['version']}", text, ts, kind,
+                        row["key"].split(":", 1)[1], source)
+
+
+def _thread_and_operator_drafts(conn: sqlite3.Connection, report: ImportReport, keep: Optional[set],
+                                skip_system: bool, operators: set, since: Optional[str], until: Optional[str]
+                                ) -> Iterator[NoteDraft]:
+    rows = conn.execute("SELECT * FROM messages ORDER BY created_at, seq").fetchall()
+    by_id = {r["id"]: r for r in rows}
+
+    def root_of(r: sqlite3.Row) -> sqlite3.Row:
+        seen = set()
+        while r["reply_to"] and r["reply_to"] in by_id and r["id"] not in seen:
+            seen.add(r["id"])
+            r = by_id[r["reply_to"]]
+        return r
+
+    replies: Dict[str, List[sqlite3.Row]] = {}
+    kept: List[sqlite3.Row] = []
+    for r in rows:
+        if keep is not None and r["channel"] not in keep:
+            report.skip("channel_filtered")
+            continue
+        if _is_noise(r, report, skip_system=skip_system):
+            continue
+        kept.append(r)
+        if r["reply_to"]:
+            replies.setdefault(root_of(r)["id"], []).append(r)
+    in_thread: set = set()
+    for r in kept:
+        if r["reply_to"]:
+            continue
+        thread = replies.get(r["id"]) or []
+        if not thread and r["status"] != "resolved":
+            continue
+        ts = iso_from_epoch(r["created_at"])
+        in_thread.add(r["id"])
+        in_thread.update(x["id"] for x in thread)
+        if not in_window(ts, since, until):
+            report.skip("outside_window")
+            continue
+        lines = [_message_text(r)]
+        for x in thread:
+            lines.append(f"## {x['sender']} ({iso_from_epoch(x['created_at'])})\n\n{_message_text(x)}")
+        atts = _attachments(conn, r)
+        for x in thread:
+            atts.extend(_attachments(conn, x))
+        yield NoteDraft(IMPORTER, f"thread:{r['id']}", "\n\n".join(lines), ts,
+                        "resolved_thread" if r["status"] == "resolved" else "answered_thread",
+                        str(r["title"] or "").strip() or None,
+                        _source(r, extra={"replies": len(thread)}), atts)
+    for r in kept:
+        if r["sender"] not in operators or r["id"] in in_thread:
+            continue
+        text = _message_text(r)
+        if not text:
+            report.skip("empty")
+            continue
+        ts = iso_from_epoch(r["created_at"])
+        if not in_window(ts, since, until):
+            report.skip("outside_window")
+            continue
+        yield NoteDraft(IMPORTER, f"message:{r['id']}", text, ts, "operator_message",
+                        str(r["title"] or "").strip() or None, _source(r), _attachments(conn, r))