windcode 0.1.0__py3-none-any.whl

windcode/memory/store.py

@@ -0,0 +1,362 @@
+from __future__ import annotations
+
+import hashlib
+import os
+import re
+import sqlite3
+from dataclasses import replace
+from pathlib import Path
+from typing import Any, cast
+from uuid import uuid4
+
+import yaml
+
+from windcode.memory.models import (
+    MemoryActivation,
+    MemoryKind,
+    MemoryRecord,
+    MemoryScope,
+    MemorySearchResult,
+    MemoryStatus,
+    utc_now,
+)
+from windcode.memory.security import validate_memory_text
+
+SCHEMA_VERSION = 2
+_FRONTMATTER = re.compile(r"\A---\n(.*?)\n---\n", re.DOTALL)
+
+
+def project_identifier(workspace: Path) -> str:
+    normalized = str(workspace.expanduser().resolve())
+    return hashlib.sha256(normalized.encode()).hexdigest()[:24]
+
+
+class MemoryStore:
+    """Markdown source of truth with a disposable SQLite FTS index."""
+
+    def __init__(self, root: Path) -> None:
+        self.root = root.expanduser().resolve()
+        self.records_dir = self.root / "records"
+        self.index_path = self.root / "index.sqlite3"
+        self.records_dir.mkdir(parents=True, exist_ok=True)
+        self._initialize()
+
+    def _connect(self) -> sqlite3.Connection:
+        connection = sqlite3.connect(self.index_path, timeout=10)
+        connection.row_factory = sqlite3.Row
+        connection.execute("PRAGMA journal_mode=WAL")
+        connection.execute("PRAGMA foreign_keys=ON")
+        return connection
+
+    def _initialize(self) -> None:
+        try:
+            with self._connect() as connection:
+                version = int(connection.execute("PRAGMA user_version").fetchone()[0])
+            if version not in (0, SCHEMA_VERSION):
+                self.index_path.unlink(missing_ok=True)
+                for suffix in ("-wal", "-shm"):
+                    Path(f"{self.index_path}{suffix}").unlink(missing_ok=True)
+                self._create_schema()
+                self.rebuild()
+                return
+            self._create_schema()
+        except sqlite3.DatabaseError:
+            corrupt = self.index_path.with_suffix(f".corrupt-{uuid4().hex}")
+            self.index_path.replace(corrupt)
+            for suffix in ("-wal", "-shm"):
+                Path(f"{self.index_path}{suffix}").unlink(missing_ok=True)
+            self._create_schema()
+            self.rebuild()
+
+    def _create_schema(self) -> None:
+        with self._connect() as connection:
+            connection.executescript(
+                """
+                CREATE TABLE IF NOT EXISTS memories (
+                    memory_id TEXT PRIMARY KEY, path TEXT NOT NULL UNIQUE,
+                    kind TEXT NOT NULL, scope TEXT NOT NULL, project_id TEXT,
+                    status TEXT NOT NULL, activation TEXT NOT NULL, priority INTEGER NOT NULL,
+                    title TEXT NOT NULL, summary TEXT NOT NULL,
+                    confidence REAL NOT NULL, version INTEGER NOT NULL,
+                    updated_at TEXT NOT NULL
+                );
+                CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
+                    memory_id UNINDEXED, title, summary, body, tags, evidence,
+                    tokenize='unicode61'
+                );
+                PRAGMA user_version=2;
+                """
+            )
+
+    def _path(self, record: MemoryRecord) -> Path:
+        return self.records_dir / record.scope.value / record.kind.value / f"{record.memory_id}.md"
+
+    @staticmethod
+    def _serialize(record: MemoryRecord) -> str:
+        metadata = record.to_dict()
+        body = str(metadata.pop("body"))
+        frontmatter = yaml.safe_dump(metadata, allow_unicode=True, sort_keys=True).strip()
+        return f"---\n{frontmatter}\n---\n\n{body}\n"
+
+    @staticmethod
+    def _parse(content: str) -> MemoryRecord:
+        match = _FRONTMATTER.match(content)
+        if match is None:
+            raise ValueError("memory file is missing YAML frontmatter")
+        raw = yaml.safe_load(match.group(1))
+        if not isinstance(raw, dict):
+            raise ValueError("memory frontmatter must be an object")
+        value = {str(key): item for key, item in cast(dict[object, object], raw).items()}
+        value["body"] = content[match.end() :].strip()
+        return MemoryRecord.from_dict(value)
+
+    def _write(self, record: MemoryRecord) -> Path:
+        path = self._path(record)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        temporary = path.with_suffix(f".tmp-{uuid4().hex}")
+        data = self._serialize(record)
+        try:
+            with temporary.open("w", encoding="utf-8") as stream:
+                stream.write(data)
+                stream.flush()
+                os.fsync(stream.fileno())
+            temporary.replace(path)
+        finally:
+            temporary.unlink(missing_ok=True)
+        return path
+
+    def _index(self, connection: sqlite3.Connection, record: MemoryRecord, path: Path) -> None:
+        connection.execute("DELETE FROM memories_fts WHERE memory_id = ?", (record.memory_id,))
+        connection.execute(
+            """INSERT OR REPLACE INTO memories
+               (memory_id,path,kind,scope,project_id,status,activation,priority,title,summary,
+                confidence,version,updated_at)
+               VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)""",
+            (
+                record.memory_id,
+                str(path.relative_to(self.root)),
+                record.kind.value,
+                record.scope.value,
+                record.project_id,
+                record.status.value,
+                record.activation.value,
+                record.priority,
+                record.title,
+                record.summary,
+                record.confidence,
+                record.version,
+                record.updated_at.isoformat(),
+            ),
+        )
+        connection.execute(
+            "INSERT INTO memories_fts VALUES (?,?,?,?,?,?)",
+            (
+                record.memory_id,
+                record.title,
+                record.summary,
+                record.body,
+                " ".join(record.tags),
+                " ".join(record.evidence),
+            ),
+        )
+
+    def save(self, record: MemoryRecord) -> MemoryRecord:
+        if not record.title or not record.summary or not record.body:
+            raise ValueError("memory title, summary, and body are required")
+        if record.scope is MemoryScope.PROJECT and not record.project_id:
+            raise ValueError("project-scoped memory requires project_id")
+        if not 0 <= record.priority <= 100:
+            raise ValueError("memory priority must be between 0 and 100")
+        validate_memory_text(record.title, record.summary, record.body, *record.evidence)
+        path = self._write(record)
+        with self._connect() as connection:
+            self._index(connection, record, path)
+        return record
+
+    def get(self, memory_id: str) -> MemoryRecord:
+        with self._connect() as connection:
+            row = connection.execute(
+                "SELECT path FROM memories WHERE memory_id = ?", (memory_id,)
+            ).fetchone()
+        if row is None:
+            raise KeyError(memory_id)
+        path = (self.root / str(row["path"])).resolve()
+        if not path.is_relative_to(self.records_dir):
+            raise ValueError("memory index contains an unsafe path")
+        return self._parse(path.read_text(encoding="utf-8"))
+
+    def list(
+        self,
+        *,
+        status: MemoryStatus | None = None,
+        project_id: str | None = None,
+        activation: MemoryActivation | None = None,
+    ) -> tuple[MemoryRecord, ...]:
+        clauses: list[str] = []
+        values: list[str] = []
+        if status is not None:
+            clauses.append("status = ?")
+            values.append(status.value)
+        if project_id is not None:
+            clauses.append("(scope = 'user' OR project_id = ?)")
+            values.append(project_id)
+        if activation is not None:
+            clauses.append("activation = ?")
+            values.append(activation.value)
+        where = " WHERE " + " AND ".join(clauses) if clauses else ""
+        with self._connect() as connection:
+            rows = connection.execute(
+                f"SELECT memory_id FROM memories{where} ORDER BY updated_at DESC", values
+            ).fetchall()
+        return tuple(self.get(str(row["memory_id"])) for row in rows)
+
+    def search(
+        self,
+        query: str,
+        *,
+        project_id: str | None = None,
+        limit: int = 5,
+        statuses: tuple[MemoryStatus, ...] = (MemoryStatus.ACTIVE,),
+        kind: MemoryKind | None = None,
+        scope: MemoryScope | None = None,
+        activation: MemoryActivation | None = None,
+    ) -> tuple[MemorySearchResult, ...]:
+        if not query.strip() or limit <= 0:
+            return ()
+        tokens = tuple(re.findall(r"[\w-]+", query.casefold(), flags=re.UNICODE))[:32]
+        if not tokens:
+            return ()
+        fts_query = " OR ".join(f'"{token.replace(chr(34), chr(34) * 2)}"' for token in tokens)
+        status_values = tuple(status.value for status in statuses)
+        placeholders = ",".join("?" for _ in status_values)
+        filters = [
+            f"m.status IN ({placeholders})",
+            "(m.scope = 'user' OR m.project_id = ?)",
+        ]
+        filter_values: list[str | int] = [*status_values, project_id or ""]
+        if kind is not None:
+            filters.append("m.kind = ?")
+            filter_values.append(kind.value)
+        if scope is not None:
+            filters.append("m.scope = ?")
+            filter_values.append(scope.value)
+        if activation is not None:
+            filters.append("m.activation = ?")
+            filter_values.append(activation.value)
+        sql = f"""
+            SELECT m.memory_id, bm25(memories_fts) AS rank, m.confidence
+            FROM memories_fts JOIN memories m USING(memory_id)
+            WHERE memories_fts MATCH ? AND {" AND ".join(filters)}
+            ORDER BY rank ASC, m.confidence DESC, m.updated_at DESC LIMIT ?
+        """
+        try:
+            with self._connect() as connection:
+                rows = connection.execute(sql, (fts_query, *filter_values, limit)).fetchall()
+        except sqlite3.OperationalError:
+            rows = ()
+        indexed = tuple(
+            MemorySearchResult(self.get(str(row["memory_id"])), float(-row["rank"])) for row in rows
+        )
+        if len(indexed) >= limit:
+            return indexed
+
+        # unicode61 does not segment unspaced CJK sentences reliably. A bounded
+        # lexical supplement lets “我喜欢什么” recall “我喜欢 Python”.
+        selected = {item.record.memory_id for item in indexed}
+        query_terms = self._lexical_terms(query)
+        supplemental: list[MemorySearchResult] = []
+        for record in self.list(project_id=project_id):
+            if (
+                record.memory_id in selected
+                or record.status not in statuses
+                or (kind is not None and record.kind is not kind)
+                or (scope is not None and record.scope is not scope)
+                or (activation is not None and record.activation is not activation)
+            ):
+                continue
+            content_terms = self._lexical_terms(
+                " ".join((record.title, record.summary, record.body, *record.tags))
+            )
+            overlap = query_terms & content_terms
+            if not overlap:
+                continue
+            score = len(overlap) / max(len(query_terms), 1) + record.confidence * 0.1
+            supplemental.append(MemorySearchResult(record, score))
+        supplemental.sort(key=lambda item: (item.score, item.record.updated_at), reverse=True)
+        return (*indexed, *supplemental[: limit - len(indexed)])
+
+    @staticmethod
+    def _lexical_terms(text: str) -> set[str]:
+        normalized = text.casefold()
+        terms = set(re.findall(r"[a-z0-9_-]{2,}", normalized))
+        for run in re.findall(r"[\u3400-\u9fff]+", normalized):
+            terms.update(run[index : index + 2] for index in range(max(0, len(run) - 1)))
+        return terms
+
+    def transition(self, memory_id: str, status: MemoryStatus) -> MemoryRecord:
+        record = self.get(memory_id).transition(status)
+        return self.save(record)
+
+    def update(self, memory_id: str, **changes: Any) -> MemoryRecord:
+        current = self.get(memory_id)
+        allowed = {
+            "title",
+            "summary",
+            "body",
+            "tags",
+            "evidence",
+            "confidence",
+            "activation",
+            "priority",
+        }
+        if unknown := set(changes) - allowed:
+            raise ValueError(f"unsupported memory fields: {', '.join(sorted(unknown))}")
+        updated = replace(current, **changes, version=current.version + 1, updated_at=utc_now())
+        return self.save(updated)
+
+    def delete(self, memory_id: str) -> None:
+        record = self.get(memory_id)
+        path = self._path(record)
+        with self._connect() as connection:
+            connection.execute("DELETE FROM memories_fts WHERE memory_id = ?", (memory_id,))
+            connection.execute("DELETE FROM memories WHERE memory_id = ?", (memory_id,))
+        path.unlink(missing_ok=True)
+
+    def rebuild(self) -> int:
+        records: list[tuple[MemoryRecord, Path]] = []
+        for path in sorted(self.records_dir.rglob("*.md")):
+            if path.is_symlink() or not path.is_file():
+                continue
+            if not path.resolve().is_relative_to(self.records_dir):
+                continue
+            records.append((self._parse(path.read_text(encoding="utf-8")), path))
+        with self._connect() as connection:
+            connection.execute("DELETE FROM memories_fts")
+            connection.execute("DELETE FROM memories")
+            for record, path in records:
+                self._index(connection, record, path)
+        return len(records)
+
+    def export_project(self, project_id: str, destination: Path) -> tuple[Path, ...]:
+        destination = destination.expanduser().resolve()
+        destination.mkdir(parents=True, exist_ok=True)
+        exported: list[Path] = []
+        for record in self.list(status=MemoryStatus.ACTIVE, project_id=project_id):
+            if record.scope is not MemoryScope.PROJECT:
+                continue
+            target = destination / f"{record.memory_id}.md"
+            target.write_text(self._serialize(record), encoding="utf-8")
+            exported.append(target)
+        return tuple(exported)
+
+    def record_outcome(self, memory_id: str, *, success: bool) -> MemoryRecord:
+        current = self.get(memory_id)
+        updated = replace(
+            current,
+            success_count=current.success_count + int(success),
+            failure_count=current.failure_count + int(not success),
+            last_verified_at=utc_now(),
+            version=current.version + 1,
+            updated_at=utc_now(),
+        )
+        return self.save(updated)

windcode/observability/__init__.py

@@ -0,0 +1,4 @@
+from windcode.observability.redaction import REDACTED, DynamicRedactor, redact
+from windcode.observability.trace import TraceStore
+
+__all__ = ["REDACTED", "DynamicRedactor", "TraceStore", "redact"]

windcode/observability/redaction.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import re
+import threading
+from collections.abc import Iterable, Mapping
+from typing import Any, cast
+
+REDACTED = "[REDACTED]"
+_SENSITIVE_KEYS = {
+    "authorization",
+    "proxy_authorization",
+    "api_key",
+    "apikey",
+    "access_token",
+    "refresh_token",
+    "password",
+    "secret",
+    "credential",
+    "credentials",
+}
+
+
+def _normalize_key(key: object) -> str:
+    return re.sub(r"[^a-z0-9]+", "_", str(key).casefold()).strip("_")
+
+
+def _is_sensitive_key(key: object) -> bool:
+    normalized = _normalize_key(key)
+    return normalized in _SENSITIVE_KEYS or normalized.endswith(
+        ("_api_key", "_access_token", "_refresh_token", "_password", "_secret")
+    )
+
+
+def _redact_string(value: str, secrets: tuple[str, ...]) -> str:
+    redacted = value
+    for secret in secrets:
+        if secret:
+            redacted = redacted.replace(secret, REDACTED)
+    return redacted
+
+
+def redact(value: Any, *, secrets: Iterable[str] = ()) -> Any:
+    """Return a recursively redacted copy without mutating the input."""
+    secret_values = tuple(secret for secret in secrets if secret)
+
+    def visit(item: object) -> object:
+        if isinstance(item, Mapping):
+            mapping = cast(Mapping[object, object], item)
+            return {
+                str(key): REDACTED if _is_sensitive_key(key) else visit(child)
+                for key, child in mapping.items()
+            }
+        if isinstance(item, tuple):
+            sequence = cast(tuple[object, ...], item)
+            return tuple(visit(child) for child in sequence)
+        if isinstance(item, list):
+            sequence = cast(list[object], item)
+            return [visit(child) for child in sequence]
+        if isinstance(item, set):
+            sequence = cast(set[object], item)
+            return {visit(child) for child in sequence}
+        if isinstance(item, str):
+            return _redact_string(item, secret_values)
+        return item
+
+    return visit(cast(object, value))
+
+
+class DynamicRedactor:
+    """Run-scoped sensitive value registry with explicit cleanup."""
+
+    def __init__(self) -> None:
+        self._secrets: set[str] = set()
+        self._lock = threading.Lock()
+
+    def register(self, secret: str) -> None:
+        if not secret:
+            return
+        with self._lock:
+            self._secrets.add(secret)
+
+    def redact(self, value: Any) -> Any:
+        with self._lock:
+            secrets = tuple(sorted(self._secrets, key=len, reverse=True))
+        return redact(value, secrets=secrets)
+
+    def clear(self) -> None:
+        with self._lock:
+            self._secrets.clear()
+
+    def __enter__(self) -> DynamicRedactor:
+        return self
+
+    def __exit__(self, *_: object) -> None:
+        self.clear()

windcode/observability/trace.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+import json
+import os
+from collections.abc import Iterable, Mapping
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any, cast
+
+from platformdirs import user_state_path
+
+from windcode.domain.events import AgentEvent, AgentEventType, event_to_dict
+from windcode.observability.redaction import redact
+
+_TRANSIENT_EVENT_KINDS = frozenset(
+    {"text_delta", "reasoning_status", "tool_progress", "subagent_progress"}
+)
+
+
+class TraceStore:
+    def __init__(
+        self,
+        run_id: str,
+        *,
+        root: Path | None = None,
+        secrets: Iterable[str] = (),
+        enabled: bool = True,
+        include_tool_arguments: bool = False,
+        include_transient_events: bool = False,
+        retention_days: int = 14,
+        max_total_mb: int = 100,
+    ) -> None:
+        self.run_id = run_id
+        self.root = (root or user_state_path("windcode") / "traces").expanduser().resolve()
+        self.path = self.root / f"{run_id}.jsonl"
+        self.secrets = tuple(secrets)
+        self.enabled = enabled
+        self.include_tool_arguments = include_tool_arguments
+        self.include_transient_events = include_transient_events
+        self.retention_days = retention_days
+        self.max_total_bytes = max_total_mb * 1024 * 1024
+        if self.enabled:
+            self.root.mkdir(parents=True, exist_ok=True)
+            self._prune()
+
+    def _prune(self) -> None:
+        traces = [path for path in self.root.glob("*.jsonl") if path.is_file()]
+        cutoff = datetime.now(UTC).timestamp() - timedelta(days=self.retention_days).total_seconds()
+        for path in traces:
+            if path.stat().st_mtime < cutoff:
+                path.unlink(missing_ok=True)
+
+        remaining = sorted(
+            (path for path in traces if path.exists()), key=lambda path: path.stat().st_mtime
+        )
+        total_bytes = sum(path.stat().st_size for path in remaining)
+        for path in remaining:
+            if total_bytes <= self.max_total_bytes:
+                break
+            size = path.stat().st_size
+            path.unlink(missing_ok=True)
+            total_bytes -= size
+
+    def _prepare_payload(self, payload: Mapping[str, Any]) -> dict[str, Any]:
+        prepared = dict(payload)
+        if not self.include_tool_arguments:
+            prepared.pop("arguments", None)
+            nested = prepared.get("payload")
+            if isinstance(nested, dict):
+                nested_copy = cast(dict[str, Any], nested).copy()
+                nested_copy.pop("arguments", None)
+                prepared["payload"] = nested_copy
+        return cast(dict[str, Any], redact(prepared, secrets=self.secrets))
+
+    def write(
+        self,
+        event: AgentEventType | Mapping[str, Any],
+        *,
+        elapsed_seconds: float | None = None,
+        error_category: str | None = None,
+        durable: bool = False,
+    ) -> dict[str, Any]:
+        if isinstance(event, AgentEvent):
+            payload = event_to_dict(event)
+            session_id = event.session_id
+            run_id = event.run_id
+        else:
+            payload = dict(event)
+            session_id = str(payload.get("session_id", ""))
+            run_id = str(payload.get("run_id", self.run_id))
+
+        record: dict[str, Any] = {
+            "timestamp": datetime.now(UTC).isoformat(),
+            "run_id": run_id,
+            "session_id": session_id,
+            "event": self._prepare_payload(payload),
+        }
+        if elapsed_seconds is not None:
+            record["elapsed_seconds"] = elapsed_seconds
+        if error_category is not None:
+            record["error_category"] = error_category
+
+        kind = str(payload.get("kind", ""))
+        if not self.enabled or (
+            not self.include_transient_events and kind in _TRANSIENT_EVENT_KINDS
+        ):
+            return record
+
+        line = json.dumps(record, ensure_ascii=True, sort_keys=True) + "\n"
+        with self.path.open("a", encoding="utf-8") as stream:
+            stream.write(line)
+            stream.flush()
+            if durable:
+                os.fsync(stream.fileno())
+        return record

windcode/policy/__init__.py

@@ -0,0 +1,22 @@
+from windcode.policy.engine import PolicyEngine, assess_risk
+from windcode.policy.models import (
+    ApprovalChoice,
+    PermissionMode,
+    PolicyAction,
+    PolicyDecision,
+    PolicyRequest,
+    RiskLevel,
+    summarize_policy_arguments,
+)
+
+__all__ = [
+    "ApprovalChoice",
+    "PermissionMode",
+    "PolicyAction",
+    "PolicyDecision",
+    "PolicyEngine",
+    "PolicyRequest",
+    "RiskLevel",
+    "assess_risk",
+    "summarize_policy_arguments",
+]