weft-memory 0.1.0a0__py3-none-any.whl

weft/ingest.py

@@ -0,0 +1,301 @@
+"""Phase 3 ingestion (spec §1, D3-3..D3-9).
+
+Every entry point returns either a record id or a Rejection with a named reason.
+Nothing is recorded as a success when extraction fails (D3-9).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import io
+import os
+import re
+import uuid
+import zipfile
+from dataclasses import dataclass
+
+from weft.substrate import Record, RecordConflict, ScopeViolation
+
+def _centroid(vectors: list) -> list:
+    """Normalised mean of the given vectors (note-level representation, no extra model calls)."""
+    dim = len(vectors[0])
+    mean = [sum(v[i] for v in vectors) / len(vectors) for i in range(dim)]
+    norm = sum(x * x for x in mean) ** 0.5 or 1.0
+    return [x / norm for x in mean]
+
+
+DOC_HEAD_CHARS = 2000  # opening text embedded per document when doc_vectors is on
+_TURN_ATTRIBUTION = {"user": "user", "assistant": "agent", "system": "system"}
+_FENCE = re.compile(r"<weft-recall\b[^>]*>.*?</weft-recall>", re.DOTALL | re.IGNORECASE)
+_STRAY = re.compile(r"</?weft-recall\b[^>]*>", re.IGNORECASE)
+_OPEN = re.compile(r"<weft-recall\b[^>]*>", re.IGNORECASE)
+_CLOSE = re.compile(r"</weft-recall>", re.IGNORECASE)
+
+
+@dataclass(frozen=True)
+class Rejection:
+    name: str
+    reason: str
+
+
+def fence(text: str) -> tuple[str, bool]:
+    """Strip recall blocks and stray recall tags (D3-5). Returns (text, changed).
+
+    Blocks are removed until none remain (review G-P3-8: nested or repeated blocks
+    must not leave recalled content behind), then any stray tag token is removed.
+    """
+    cleaned = text
+    while True:
+        opened = _OPEN.search(cleaned)
+        if opened is None:
+            break
+        closes = list(_CLOSE.finditer(cleaned, opened.end()))
+        if not closes:
+            break
+        # Conservative: everything from the first open tag through the last close tag goes.
+        cleaned = cleaned[:opened.start()] + cleaned[closes[-1].end():]
+    cleaned = _STRAY.sub("", cleaned)
+    return cleaned, cleaned != text
+
+
+def _extract_paragraphs(text: str) -> list[str]:
+    return [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
+
+
+def _digest(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def _record_digest(record: Record) -> str:
+    row = list(record.to_row())
+    del row[5]  # creation time is not part of the intended content
+    return _digest(json.dumps(row, separators=(',', ':')).encode())
+
+
+def _parse(name: str, data: bytes):
+    """Return extracted text, or a Rejection. Never returns partial text on failure."""
+    ext = os.path.splitext(name.lower())[1]
+    if ext in (".txt", ".md"):
+        try:
+            return data.decode("utf-8")
+        except UnicodeDecodeError:
+            return Rejection(name, "encoding_unsupported")
+    if ext == ".pdf":
+        if not data.startswith(b"%PDF"):
+            return Rejection(name, "parse_failed")
+        if b"/Encrypt" in data:
+            return Rejection(name, "encrypted_pdf")
+        try:
+            from pypdf import PdfReader
+            reader = PdfReader(io.BytesIO(data))
+            if reader.is_encrypted:
+                return Rejection(name, "encrypted_pdf")
+            text = "\n\n".join((page.extract_text() or "") for page in reader.pages).strip()
+        except Exception:
+            return Rejection(name, "parse_failed")
+        return text if text else Rejection(name, "no_text_layer")
+    if ext == ".docx":
+        try:
+            import docx
+            doc = docx.Document(io.BytesIO(data))
+            text = "\n\n".join(p.text for p in doc.paragraphs if p.text.strip())
+        except Exception:
+            return Rejection(name, "parse_failed")
+        return text if text else Rejection(name, "no_text_layer")
+    return Rejection(name, "unsupported_format")
+
+
+class Ingestor:
+    """Writes Sources and Observations through the substrate. Holds no state of its own."""
+
+    def __init__(self, substrate, embedder=None, doc_vectors=False, doc_centroid=True):
+        self.sub = substrate
+        self.embedder = embedder  # optional: anything with embed_document(text) -> vector
+        self.doc_vectors = doc_vectors  # also embed each document's opening text (note-level)
+        self.doc_centroid = doc_centroid  # note-level vector: normalised mean of paragraph vectors (default on)
+
+    def _put_matching(self, record, ctx):
+        """Recover only an identical, currently readable and writable incumbent."""
+        try:
+            self.sub.put(record, ctx)
+        except RecordConflict:
+            incumbent = self.sub._lookup_record(record.record_id)
+            engine = self.sub.scope_engine
+            if (incumbent is None
+                    or not engine.authorize_write(incumbent, ctx).allowed
+                    or not engine.authorize_read(incumbent.scope, ctx, engine._payload_text(incumbent)).allowed
+                    or any(getattr(incumbent, field) != getattr(record, field) for field in (
+                        'record_id', 'record_type', 'scope', 'payload', 'epistemic_status',
+                        'effective_at', 'supersedes', 'source_ids', 'provenance', 'attribution'))):
+                raise
+
+    def ingest_turn(self, text: str, speaker: str, ctx, scope: str, conversation_id: str) -> str:
+        """A conversation turn becomes a Source with the speaker's attribution (spec §7.1)."""
+        attribution = _TURN_ATTRIBUTION.get(speaker)
+        if attribution is None:
+            raise ValueError(f"unknown speaker: {speaker!r}")
+        clean, changed = fence(text)
+        provenance = {"conversation_id": conversation_id, "speaker": speaker}
+        if changed:
+            provenance["fenced"] = True
+        rid = "turn-" + uuid.uuid4().hex[:24]
+        record = Record(
+            record_id=rid, record_type="Source", scope=scope, payload={"text": clean},
+            attribution=attribution, source_ids=[], provenance=provenance,
+        )
+        if not self.sub.scope_engine.authorize_write(record, ctx).allowed:
+            raise ScopeViolation("write not permitted", audit_recorded=False)
+        vector = self.embedder.embed_document(clean) if self.embedder is not None else None
+        with self.sub.ingestion_transaction():
+            self.sub.put(record, ctx)
+            if self.embedder is not None:
+                self.sub.vector_put(rid, scope, vector, ctx=ctx, model_id=self.embedder.model_id)
+        return rid
+
+    def ingest_document(self, name: str, data: bytes, ctx, scope: str, author: str):
+        """Document text is evidence: Observations carry attribution `document` (D3-3)."""
+        parsed = _parse(name, data)
+        if isinstance(parsed, Rejection):
+            return parsed
+        parsed, _ = fence(parsed)  # review G-P3-4: recall blocks never persist from documents
+        if not parsed.strip():
+            return Rejection(name, "empty_document")
+        content_hash = _digest(data)
+        sid = "doc-" + hashlib.sha256(f"{scope}\0{content_hash}".encode()).hexdigest()[:24]
+        paragraphs = _extract_paragraphs(parsed)
+        source = Record(
+            record_id=sid, record_type="Source", scope=scope,
+            payload={"text": parsed, "name": name}, attribution="document", source_ids=[],
+            provenance={"author": author, "document": name, "content_hash": content_hash},
+        )
+        observations = [Record(
+            record_id=f"{sid}-p{i:04d}", record_type="Observation", scope=scope,
+            payload={"text": paragraph}, attribution="document", source_ids=[sid],
+            provenance={"location": f"paragraph {i + 1}", "document": name},
+        ) for i, paragraph in enumerate(paragraphs)]
+        if any(not self.sub.scope_engine.authorize_write(record, ctx).allowed
+               for record in [source, *observations]):
+            raise ScopeViolation("write not permitted", audit_recorded=False)
+        state = self.sub.job_state(sid, "ingest", content_hash, scope=scope, ctx=ctx)
+        if state == "done":
+            with self.sub.ingestion_transaction():
+                # A completed bundle with missing members may reflect a deletion.
+                # Never repair it implicitly by recreating those records on retry.
+                if any(self.sub._lookup_record(record.record_id) is None
+                       for record in [source, *observations]):
+                    raise RecordConflict('completed ingestion is incomplete')
+                self._put_matching(source, ctx)
+                for observation in observations:
+                    self._put_matching(observation, ctx)
+                if self.embedder is None:
+                    return sid
+                # Existing vector rows carry no generation-configuration evidence.
+                # Recompute rather than treating their presence as a matching mode.
+
+        # Compute every embedding before writing the job, Source, or Observations. Model
+        # load/inference failures therefore cannot leave an apparently successful partial import.
+        paragraph_vectors = []
+        source_vector = None
+        if self.embedder is not None:
+            paragraph_vectors = [self.embedder.embed_document(paragraph) for paragraph in paragraphs]
+            if self.doc_vectors:
+                source_vector = self.embedder.embed_document(parsed[:DOC_HEAD_CHARS])
+            if self.doc_centroid and paragraph_vectors:
+                source_vector = _centroid(paragraph_vectors)
+
+        with self.sub.ingestion_transaction():
+            # Read completion under the write transaction: another ingestion may
+            # have completed and a member been deleted while embeddings ran.
+            current_state = self.sub.job_state(sid, "ingest", content_hash, scope=scope, ctx=ctx)
+            if (state == "done" or current_state == "done") and any(
+                    self.sub._lookup_record(record.record_id) is None
+                    for record in [source, *observations]):
+                raise RecordConflict('completed ingestion is incomplete')
+            self.sub.set_job(sid, "ingest", content_hash, "running", scope=scope, ctx=ctx)
+            self._put_matching(source, ctx)
+            # A preexisting source vector would contradict this requested mode.
+            # Refuse rather than silently retaining it under the same model ID.
+            if (self.embedder is not None and not self.doc_vectors and not self.doc_centroid
+                    and self.sub.vector_has_record(sid, scope, ctx, model_id=self.embedder.model_id)):
+                raise ValueError('existing source vector conflicts with requested vector mode')
+            for i, observation in enumerate(observations):
+                oid = observation.record_id
+                self._put_matching(observation, ctx)
+                if self.embedder is not None:
+                    self.sub.vector_put(oid, scope, paragraph_vectors[i], ctx=ctx, model_id=self.embedder.model_id)
+            if self.embedder is not None and source_vector is not None:
+                self.sub.vector_put(sid, scope, source_vector, ctx=ctx, model_id=self.embedder.model_id)
+            self.sub.set_job(sid, "ingest", content_hash, "done", scope=scope, ctx=ctx)
+        return sid
+
+    def ingest_media(self, name: str, data: bytes, kind: str, ctx, scope: str,
+                     describer=None, transcriber=None):
+        """Image or audio becomes machine-derived text. Never presented as verbatim (D3-6)."""
+        if kind == "image":
+            adapter = describer
+        elif kind == "audio":
+            adapter = transcriber
+        else:
+            return Rejection(name, "unsupported_kind")
+        if adapter is None:
+            return Rejection(name, "no_adapter_configured")
+        content_hash = _digest(data)
+        sid = "media-" + hashlib.sha256(f"{scope}\0{content_hash}".encode()).hexdigest()[:24]
+        stub = Record(record_id=sid, record_type="Source", scope=scope, payload={}, attribution="document")
+        if ctx is None or not self.sub.scope_engine.authorize_write(stub, ctx).allowed:
+            raise ScopeViolation("write not permitted")
+        with self.sub.ingestion_transaction():
+            row = self.sub._conn.execute(
+                "SELECT state FROM jobs WHERE scope=? AND source_id=? AND stage='derive' AND content_hash=?",
+                (scope, sid, content_hash),
+            ).fetchone()
+            # The public job writer and an unkeyed digest cannot attest inference.
+            # Until trusted provenance exists, no completed media job is reusable.
+            if row and row[0] == 'done':
+                raise RecordConflict('completed media reuse is unavailable')
+        self.sub.set_job(sid, "derive", content_hash, "running", scope=scope, ctx=ctx)
+        try:
+            text = adapter(data)
+        except Exception:
+            self.sub.set_job(sid, "derive", content_hash, "failed", "adapter_failed", scope=scope, ctx=ctx)
+            return Rejection(name, "adapter_failed")
+        if isinstance(text, str):
+            text, _ = fence(text)  # review G-P3-4: derived text is fenced like any other input
+        if not isinstance(text, str) or not text.strip():
+            self.sub.set_job(sid, "derive", content_hash, "failed", "empty_output", scope=scope, ctx=ctx)
+            return Rejection(name, "empty_output")
+        vector = None
+        if self.embedder is not None:
+            try:
+                vector = self.embedder.embed_document(text.strip())
+            except Exception as exc:
+                try:
+                    self.sub.set_job(sid, "derive", content_hash, "failed", "embedding_failed", scope=scope, ctx=ctx)
+                except Exception as job_exc:
+                    exc.add_note(f"Could not record failed job: {job_exc}")
+                raise
+        failure_reason = "persistence_failed"
+        try:
+            with self.sub.ingestion_transaction():
+                model = getattr(adapter, "model_id", "unknown")
+                output = Record(
+                    record_id=sid, record_type="Source", scope=scope, payload={"text": text.strip()},
+                    attribution="media-derived", source_ids=[],
+                    provenance={"machine_derived": True, "model": model, "kind": kind,
+                                "name": name, "content_hash": content_hash},
+                )
+                # Reusing a preexisting output is not proof this adapter produced it.
+                self.sub.put(output, ctx)
+                if self.embedder is not None:
+                    failure_reason = "embedding_failed"
+                    self.sub.vector_put(sid, scope, vector, ctx=ctx, model_id=self.embedder.model_id)
+                    failure_reason = "persistence_failed"
+                self.sub.set_job(sid, "derive", content_hash, "done", 'sha256:' + _record_digest(output), scope=scope, ctx=ctx)
+        except Exception as exc:
+            try:
+                self.sub.set_job(sid, "derive", content_hash, "failed", failure_reason, scope=scope, ctx=ctx)
+            except Exception as job_exc:
+                exc.add_note(f"Could not record failed job: {job_exc}")
+            raise
+        return sid

weft/knowledge.py

@@ -0,0 +1,109 @@
+"""Phase 2 knowledge rules (spec §3, D2-2, D2-3, D2-9).
+
+A pure decision function over a record and the caller's context. Storage lookups
+are passed in, so the rules stay testable and cannot reach around the store.
+Returns a reason string when the write must be refused, or None when allowed.
+Callers never show the reason to the writer (generic denial, G4/G10).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Callable
+
+
+class KnowledgeRefusal(Exception):
+    """Raised inside a write transaction; the store turns it into a generic denial."""
+
+
+@dataclass(frozen=True)
+class Rule:
+    # attribution -> statuses that attribution may produce for this type
+    statuses: dict = field(default_factory=dict)
+    needs_source: bool = False
+
+
+def _s(*names: str) -> frozenset:
+    return frozenset(names)
+
+
+RULES: dict[str, Rule] = {
+    "Person": Rule({"user": _s("stated"), "document": _s("stated")}),
+    "Agent": Rule({"system": _s("stated")}),
+    "Project": Rule({"user": _s("stated")}),
+    # Phase 3 (spec §1): assistant turns are agent-attributed Sources; fenced
+    # or system-injected text is system-attributed. Neither can yield Decisions.
+    "Source": Rule({
+        "document": _s("stated"), "media-derived": _s("stated"), "user": _s("stated"),
+        "agent": _s("stated"), "system": _s("stated"),
+    }),
+    "Conversation": Rule({"system": _s("stated")}),
+    "Observation": Rule(
+        {"document": _s("stated"), "media-derived": _s("stated")}, needs_source=True
+    ),
+    "Claim": Rule({"agent": _s("derived"), "verifier": _s("verified")}),
+    "Hypothesis": Rule({"agent": _s("derived")}),
+    "Decision": Rule({"user": _s("stated")}),
+    "Commitment": Rule({"user": _s("stated")}),
+    "Correction": Rule({"user": _s("stated")}),
+    "Verification": Rule({"verifier": _s("stated")}),
+    "Deletion": Rule({"user": _s("stated")}),
+    "Summary": Rule({"system": _s("derived")}),
+    "RecallReceipt": Rule({"system": _s("stated")}),
+}
+
+
+def check_record(
+    record,
+    ctx,
+    *,
+    is_verifier: Callable[[str | None], bool],
+    lookup: Callable[[str], object | None],
+) -> str | None:
+    """Return a refusal reason, or None if the record may be written."""
+    rule = RULES.get(record.record_type)
+    if rule is None:
+        return "unknown record type"
+    attribution = record.attribution
+    if not attribution:
+        return "attribution required"
+    allowed = rule.statuses.get(attribution)
+    if allowed is None:
+        return "attribution not permitted for this record type"
+    if record.epistemic_status not in allowed:
+        return "status not permitted for this attribution"
+    if attribution == "user" and not ctx.user_authored:
+        # D2-2 / D2-7: "user" means the user's own turn. An agent cannot claim it.
+        return "user attribution requires a user-authored turn"
+    if rule.needs_source and not record.source_ids:
+        return "observation must cite a source"
+    if rule.needs_source:
+        for sid in record.source_ids:
+            src = lookup(sid)
+            # Same-scope only: a foreign or missing source is one generic refusal.
+            if src is None or src.scope != record.scope:
+                return "cited source not available"
+    if attribution == "verifier" and not is_verifier(ctx.actor):
+        return "actor is not a registered verifier"
+    if record.epistemic_status == "verified" and not _names_claim(record, lookup):
+        return "no verification names this claim"
+    return None
+
+
+def _names_claim(claim, lookup) -> bool:
+    """D2-3: a verified Claim must cite a Verification that names it.
+
+    The Verification must be in the same scope as the Claim, so the answer
+    reveals nothing about records the writer cannot already see.
+    """
+    for vid in claim.source_ids:
+        v = lookup(vid)
+        if (
+            v is not None
+            and v.record_type == "Verification"
+            and v.attribution == "verifier"
+            and v.scope == claim.scope
+            and v.payload.get("claim_id") == claim.record_id
+        ):
+            return True
+    return False

weft/logging_config.py

@@ -0,0 +1,54 @@
+"""Phase 6: structured logging with redaction."""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+import sys
+from datetime import datetime, timezone
+from typing import Any
+
+
+class RedactingFormatter(logging.Formatter):
+    """JSON formatter that redacts untrusted content fields."""
+
+    _PATTERNS = [
+        re.compile(r"<\s*UNTRUSTED_DOCUMENT\s*>", re.IGNORECASE),
+        re.compile(r"<\s*/\s*UNTRUSTED_DOCUMENT\s*>", re.IGNORECASE),
+        re.compile(r"\[\s*REDACTED\s+INJECTION\s*\]", re.IGNORECASE),
+    ]
+
+    def format(self, record: logging.LogRecord) -> str:
+        message = super().format(record)
+        for pat in self._PATTERNS:
+            message = pat.sub("[REDACTED]", message)
+        return message
+
+
+def setup_logging(level: str = "INFO") -> None:
+    """Configure structured JSON logging with redaction."""
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(RedactingFormatter("%(message)s"))
+    root = logging.getLogger()
+    root.handlers = [handler]
+    root.setLevel(getattr(logging, level, logging.INFO))
+
+
+def log_event(
+    operation: str,
+    record_id: str | None = None,
+    scope: str | None = None,
+    detail: str = "",
+    level: str = "INFO",
+) -> None:
+    """Emit a structured log event."""
+    entry = {
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "level": level,
+        "operation": operation,
+        "record_id": record_id,
+        "scope": scope,
+        "detail": detail,
+    }
+    logging.getLogger("weft").log(getattr(logging, level, logging.INFO), json.dumps(entry))

weft/models.py

@@ -0,0 +1,99 @@
+"""Model layer for Weft: local and external models behind one interface.
+
+Each task may use a different model. External calls require explicit opt-in
+and are recorded in the receipt/audit trail.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Callable, List, Optional
+
+
+@dataclass
+class ModelCall:
+    model_id: str
+    provider: str
+    task: str
+    input_chars: int = 0
+    output_chars: int = 0
+    egress: bool = False
+    error: Optional[str] = None
+
+
+@dataclass
+class ModelResult:
+    text: str
+    model: ModelCall
+    raw: Any = None
+
+
+class ModelLayerError(Exception):
+    """Base error for model layer failures."""
+
+
+class EgressPolicyError(ModelLayerError):
+    """External inference was attempted without opt-in."""
+
+
+class ModelUnavailable(ModelLayerError):
+    """The requested model cannot be loaded or reached."""
+
+
+class ModelLayer:
+    """One interface for local and external inference."""
+
+    def __init__(self, *, local_factory: Optional[Callable[[str], Any]] = None,
+                 external_factory: Optional[Callable[[str, str], Any]] = None,
+                 egress_allowed: bool = False) -> None:
+        self.local_factory = local_factory
+        self.external_factory = external_factory
+        self.egress_allowed = egress_allowed
+        self.calls: List[ModelCall] = []
+
+    def _record(self, call: ModelCall, text: str) -> ModelResult:
+        self.calls.append(call)
+        return ModelResult(text=text, model=call)
+
+    def _run_local(self, model_id: str, prompt: str, task: str) -> ModelResult:
+        if self.local_factory is None:
+            raise ModelUnavailable(f"no local model configured for {task}")
+        model = self.local_factory(model_id)
+        call = ModelCall(model_id=model_id, provider="local", task=task,
+                         input_chars=len(prompt))
+        try:
+            text = model(prompt)
+        except Exception as exc:
+            call.error = str(exc)
+            self.calls.append(call)
+            raise ModelLayerError(f"local model failed: {exc}") from exc
+        call.output_chars = len(text)
+        return self._record(call, text)
+
+    def _run_external(self, model_id: str, prompt: str, task: str, base_url: str, api_key: str) -> ModelResult:
+        if not self.egress_allowed:
+            raise EgressPolicyError(f"external inference requires opt-in for {task}")
+        if self.external_factory is None:
+            raise ModelUnavailable(f"no external factory configured for {task}")
+        model = self.external_factory(model_id, base_url)
+        call = ModelCall(model_id=model_id, provider="external", task=task,
+                         input_chars=len(prompt), egress=True)
+        try:
+            text = model(prompt, api_key=api_key)
+        except Exception as exc:
+            call.error = str(exc)
+            self.calls.append(call)
+            raise ModelLayerError(f"external model failed: {exc}") from exc
+        call.output_chars = len(text)
+        return self._record(call, text)
+
+    def run(self, model_id: str, prompt: str, task: str, *,
+            base_url: str = "", api_key: str = "", force_local: bool = False) -> ModelResult:
+        """Run a prompt through the appropriate model.
+
+        External calls require `egress_allowed=True` unless `force_local=True`.
+        Returns the result and records the call for audit.
+        """
+        if force_local or not base_url or not api_key:
+            return self._run_local(model_id, prompt, task)
+        return self._run_external(model_id, prompt, task, base_url, api_key)