superlocalmemory 4.1.13 → 4.1.14

package/src/superlocalmemory/infra/local_diagnostics.py

@@ -1,327 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-
-"""Bounded, content-free operational diagnostics stored on this machine.
-
-The store accepts only enumerated operations and normalized client families.
-It persists daily counters, latency buckets, coarse error classes, and client
-family transitions. Export is a deliberate local file operation; this module
-has no automatic reporting mechanism.
-"""
-
-from __future__ import annotations
-
-import json
-import math
-import os
-import re
-import sqlite3
-import tempfile
-from collections.abc import Callable, Iterable
-from datetime import UTC, datetime, timedelta
-from pathlib import Path
-from typing import Any
-
-from superlocalmemory.infra.data_root import state_path
-
-SCHEMA_VERSION = 1
-RETENTION_DAYS = 31
-MAX_COUNTER = 2**63 - 1
-
-_OPERATIONS = frozenset(
-    {
-        "activation",
-        "remember",
-        "recall",
-        "observe",
-        "session_open",
-        "session_close",
-    }
-)
-_CLIENT_FAMILIES = (
-    "claude",
-    "codex",
-    "cursor",
-    "windsurf",
-    "copilot",
-    "vscode",
-    "jetbrains",
-    "dashboard",
-    "hook",
-    "mcp",
-    "api",
-    "cli",
-)
-_LATENCY_BOUNDS_MS = (5, 25, 100, 500, 2_000, 10_000, 60_000)
-_ERROR_CLASSES = frozenset(
-    {"timeout", "authorization", "validation", "unavailable", "internal"}
-)
-
-
-def normalize_client(value: str | None) -> str:
-    """Collapse an arbitrary actor label to a fixed non-identifying family."""
-    tokens = frozenset(re.findall(r"[a-z0-9]+", str(value or "").lower()))
-    for family in _CLIENT_FAMILIES:
-        if family in tokens:
-            return family
-    return "other"
-
-
-def classify_error(error: BaseException | None) -> str | None:
-    """Map an exception type to a fixed class without inspecting its message."""
-    if error is None:
-        return None
-    if isinstance(error, TimeoutError):
-        return "timeout"
-    if isinstance(error, PermissionError):
-        return "authorization"
-    if isinstance(error, (TypeError, ValueError)):
-        return "validation"
-    if isinstance(error, (ConnectionError, FileNotFoundError)):
-        return "unavailable"
-    return "internal"
-
-
-def _latency_bucket(duration_ms: float | int | None) -> str | None:
-    if duration_ms is None:
-        return None
-    try:
-        value = float(duration_ms)
-    except (TypeError, ValueError, OverflowError):
-        return "invalid"
-    if not math.isfinite(value) or value < 0:
-        return "invalid"
-    for bound in _LATENCY_BOUNDS_MS:
-        if value <= bound:
-            return f"le_{bound}"
-    return "gt_60000"
-
-
-class LocalDiagnostics:
-    """SQLite-backed bounded aggregate diagnostics.
-
-    The schema intentionally has no free-text value column. Every persisted
-    dimension comes from a finite allow-list, so caller payloads cannot enter
-    the database or an export by accident.
-    """
-
-    def __init__(
-        self,
-        db_file: str | Path,
-        *,
-        clock: Callable[[], datetime] | None = None,
-        retention_days: int = RETENTION_DAYS,
-    ) -> None:
-        if retention_days < 1 or retention_days > RETENTION_DAYS:
-            raise ValueError(f"retention_days must be between 1 and {RETENTION_DAYS}")
-        self._db_file = Path(db_file)
-        self._clock = clock or (lambda: datetime.now(UTC))
-        self._retention_days = retention_days
-
-    def _connect(self) -> sqlite3.Connection:
-        self._db_file.parent.mkdir(parents=True, exist_ok=True)
-        if os.name == "posix":
-            os.chmod(self._db_file.parent, 0o700)
-        conn = sqlite3.connect(str(self._db_file), timeout=5.0)
-        conn.execute(
-            "CREATE TABLE IF NOT EXISTS diagnostic_aggregates ("
-            "day TEXT NOT NULL, group_name TEXT NOT NULL, "
-            "dimension TEXT NOT NULL, bucket TEXT NOT NULL, "
-            "value INTEGER NOT NULL CHECK(value >= 0), "
-            "PRIMARY KEY(day, group_name, dimension, bucket))"
-        )
-        conn.commit()
-        if os.name == "posix":
-            os.chmod(self._db_file, 0o600)
-        return conn
-
-    def _today(self) -> datetime:
-        current = self._clock()
-        if current.tzinfo is None:
-            current = current.replace(tzinfo=UTC)
-        return current.astimezone(UTC)
-
-    @staticmethod
-    def _increment(
-        conn: sqlite3.Connection,
-        day: str,
-        group_name: str,
-        dimension: str,
-        bucket: str,
-    ) -> None:
-        conn.execute(
-            "INSERT INTO diagnostic_aggregates "
-            "(day, group_name, dimension, bucket, value) VALUES (?,?,?,?,1) "
-            "ON CONFLICT(day, group_name, dimension, bucket) DO UPDATE SET "
-            "value=MIN(diagnostic_aggregates.value + 1, ?)",
-            (day, group_name, dimension, bucket, MAX_COUNTER),
-        )
-
-    def record(
-        self,
-        operation: str,
-        *,
-        client: str | None = None,
-        duration_ms: float | int | None = None,
-        error: BaseException | None = None,
-        source_clients: Iterable[str] = (),
-    ) -> None:
-        """Record one operation using only fixed aggregate dimensions."""
-        if operation not in _OPERATIONS:
-            raise ValueError(f"unsupported diagnostic operation: {operation}")
-        current = self._today()
-        day = current.date().isoformat()
-        cutoff = (current.date() - timedelta(days=self._retention_days - 1)).isoformat()
-        client_family = normalize_client(client)
-        status = "error" if error is not None else "success"
-        duration_bucket = _latency_bucket(duration_ms)
-        error_class = classify_error(error)
-        transitions = {
-            f"{normalize_client(source)}_to_{client_family}"
-            for source in source_clients
-            if normalize_client(source) != client_family
-        }
-
-        conn = self._connect()
-        try:
-            conn.execute("BEGIN IMMEDIATE")
-            conn.execute("DELETE FROM diagnostic_aggregates WHERE day < ?", (cutoff,))
-            self._increment(conn, day, "operations", operation, status)
-            self._increment(conn, day, "clients", client_family, operation)
-            if duration_bucket is not None:
-                self._increment(conn, day, "latency_ms", operation, duration_bucket)
-            if error_class is not None:
-                self._increment(conn, day, "errors", operation, error_class)
-            if operation == "activation":
-                self._increment(conn, day, "activations", client_family, "count")
-            for transition in sorted(transitions):
-                self._increment(conn, day, "handoffs", transition, "count")
-            conn.commit()
-        except BaseException:
-            conn.rollback()
-            raise
-        finally:
-            conn.close()
-
-    def export_payload(self) -> dict[str, Any]:
-        """Return a deterministic, sanitized aggregate document."""
-        days: dict[str, dict[str, Any]] = {}
-        if self._db_file.is_file():
-            conn = self._connect()
-            try:
-                rows = conn.execute(
-                    "SELECT day, group_name, dimension, bucket, value "
-                    "FROM diagnostic_aggregates "
-                    "ORDER BY day, group_name, dimension, bucket"
-                ).fetchall()
-            finally:
-                conn.close()
-            for day, group_name, dimension, bucket, value in rows:
-                entry = days.setdefault(str(day), {"day": str(day)})
-                group = entry.setdefault(str(group_name), {})
-                dimension_values = group.setdefault(str(dimension), {})
-                dimension_values[str(bucket)] = int(value)
-        return {
-            "schema_version": SCHEMA_VERSION,
-            "retention_days": self._retention_days,
-            "privacy": {
-                "local_only": True,
-                "reporting": "manual_export_only",
-                "content_values": False,
-                "query_values": False,
-                "fact_identifiers": False,
-                "filesystem_locations": False,
-                "identity_values": False,
-            },
-            "days": [days[key] for key in sorted(days)],
-        }
-
-    def export_json(self, destination: str | Path) -> dict[str, Any]:
-        """Atomically write a deterministic manual export with mode 0600."""
-        payload = self.export_payload()
-        output = Path(destination)
-        output.parent.mkdir(parents=True, exist_ok=True)
-        data = (json.dumps(payload, indent=2, sort_keys=True) + "\n").encode("utf-8")
-        temporary_name = ""
-        try:
-            with tempfile.NamedTemporaryFile(
-                mode="wb", dir=output.parent, prefix=f".{output.name}.", delete=False,
-            ) as temporary:
-                temporary_name = temporary.name
-                os.chmod(temporary_name, 0o600)
-                temporary.write(data)
-                temporary.flush()
-                os.fsync(temporary.fileno())
-            os.replace(temporary_name, output)
-            if os.name == "posix":
-                os.chmod(output, 0o600)
-        finally:
-            if temporary_name:
-                Path(temporary_name).unlink(missing_ok=True)
-        return payload
-
-
-def default_diagnostics() -> LocalDiagnostics:
-    """Return the diagnostics store for the active local state root."""
-    return LocalDiagnostics(state_path("diagnostics.db"))
-
-
-def record_operation(
-    operation: str,
-    *,
-    client: str | None = None,
-    duration_ms: float | int | None = None,
-    error: BaseException | None = None,
-    source_clients: Iterable[str] = (),
-) -> None:
-    """Best-effort hot-path adapter; diagnostics never affect correctness."""
-    try:
-        default_diagnostics().record(
-            operation,
-            client=client,
-            duration_ms=duration_ms,
-            error=error,
-            source_clients=source_clients,
-        )
-    except Exception:
-        return
-
-
-def record_recall(db: Any, response: Any, *, client: str | None) -> None:
-    """Record recall and aggregate cross-client provenance transitions."""
-    source_clients: list[str] = []
-    fact_ids = [
-        str(result.fact.fact_id)
-        for result in getattr(response, "results", ())
-        if getattr(getattr(result, "fact", None), "fact_id", None)
-    ]
-    profile_id = ""
-    if getattr(response, "results", None):
-        profile_id = str(getattr(response.results[0].fact, "profile_id", ""))
-    if fact_ids and profile_id:
-        try:
-            marks = ",".join("?" for _ in fact_ids)
-            rows = db.execute(
-                f"SELECT DISTINCT created_by FROM provenance "
-                f"WHERE profile_id=? AND fact_id IN ({marks})",
-                (profile_id, *fact_ids),
-            )
-            source_clients = [str(dict(row).get("created_by", "")) for row in rows]
-        except Exception:
-            source_clients = []
-    record_operation(
-        "recall",
-        client=client,
-        duration_ms=getattr(response, "retrieval_time_ms", None),
-        source_clients=source_clients,
-    )
-
-
-__all__ = [
-    "LocalDiagnostics",
-    "classify_error",
-    "default_diagnostics",
-    "normalize_client",
-    "record_operation",
-    "record_recall",
-]

package/src/superlocalmemory/infra/pid_manager.py

@@ -1,193 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""SuperLocalMemory V3 -- PID File Manager.
-
-Atomic JSON-based PID tracking. Records which SLM processes are running
-and their parent PIDs for orphan detection.
-
-File format: {"pids": [{"pid": 1234, "ppid": 5678, "started_at": "2026-03-30T14:25:01"}]}
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import os
-import tempfile
-from dataclasses import dataclass
-from datetime import UTC, datetime
-from pathlib import Path
-
-logger = logging.getLogger(__name__)
-
-
-# ---------------------------------------------------------------------------
-# PidRecord -- frozen dataclass for a single process entry
-# ---------------------------------------------------------------------------
-
-@dataclass(frozen=True)
-class PidRecord:
-    """A single PID record stored in the PID file."""
-
-    pid: int
-    ppid: int
-    started_at: str
-
-    def to_dict(self) -> dict[str, object]:
-        """Serialize to a JSON-compatible dictionary."""
-        return {
-            "pid": self.pid,
-            "ppid": self.ppid,
-            "started_at": self.started_at,
-        }
-
-    @classmethod
-    def from_dict(cls, d: dict) -> PidRecord:
-        """Deserialize from a dictionary (as read from JSON)."""
-        return PidRecord(
-            pid=int(d["pid"]),
-            ppid=int(d["ppid"]),
-            started_at=str(d["started_at"]),
-        )
-
-
-# ---------------------------------------------------------------------------
-# PidManager -- atomic JSON PID file management
-# ---------------------------------------------------------------------------
-
-class PidManager:
-    """Manage a JSON file tracking running SLM process PIDs.
-
-    Uses atomic temp-file-then-rename writes so the file is never
-    corrupted by a crash mid-write.
-    """
-
-    def __init__(self, pid_file_path: Path) -> None:
-        self._path = pid_file_path
-        self._path.parent.mkdir(parents=True, exist_ok=True)
-
-    # -- Read ---------------------------------------------------------------
-
-    def read_all(self) -> list[PidRecord]:
-        """Read all PID records from the PID file.
-
-        Returns empty list if the file is missing, corrupt, or malformed.
-        Corrupt files are deleted so the next write starts clean.
-        """
-        if not self._path.exists():
-            return []
-
-        try:
-            raw = self._path.read_text(encoding="utf-8")
-            data = json.loads(raw)
-
-            if not isinstance(data, dict) or "pids" not in data:
-                logger.warning("Malformed PID file %s, resetting", self._path)
-                return []
-
-            return [
-                PidRecord.from_dict(entry)
-                for entry in data["pids"]
-                if isinstance(entry, dict)
-            ]
-
-        except json.JSONDecodeError:
-            logger.warning("Corrupt PID file %s, deleting", self._path)
-            try:
-                self._path.unlink(missing_ok=True)
-            except OSError:
-                pass
-            return []
-
-        except OSError as exc:
-            logger.warning("Cannot read PID file %s: %s", self._path, exc)
-            return []
-
-    # -- Write (atomic) -----------------------------------------------------
-
-    def _write_all(self, records: list[PidRecord]) -> None:
-        """Atomically write all PID records to the PID file.
-
-        Uses temp-file-then-rename pattern:
-        - Write to a temp file in the same directory
-        - os.replace() is atomic on POSIX (single inode swap)
-        - On crash: either old file or new file, never corruption
-        """
-        data = {"pids": [r.to_dict() for r in records]}
-        content = json.dumps(data, indent=2)
-
-        tmp_path: str | None = None
-        try:
-            fd, tmp_path = tempfile.mkstemp(
-                dir=str(self._path.parent),
-                suffix=".tmp",
-                prefix="slm-pids-",
-            )
-            os.write(fd, content.encode("utf-8"))
-            os.fsync(fd)
-            os.close(fd)
-            os.replace(tmp_path, str(self._path))
-            tmp_path = None  # Consumed by replace, no cleanup needed
-
-        except OSError as exc:
-            logger.warning("Cannot write PID file %s: %s", self._path, exc)
-
-        finally:
-            if tmp_path is not None:
-                try:
-                    Path(tmp_path).unlink(missing_ok=True)
-                except OSError:
-                    pass
-
-    # -- Register / Remove --------------------------------------------------
-
-    def register(self, pid: int, ppid: int) -> None:
-        """Add current process to PID file (replaces stale entry for same PID)."""
-        records = self.read_all()
-        records = [r for r in records if r.pid != pid]
-
-        new_record = PidRecord(
-            pid=pid,
-            ppid=ppid,
-            started_at=datetime.now(UTC).isoformat(),
-        )
-        records.append(new_record)
-        self._write_all(records)
-
-    def remove(self, pid: int) -> bool:
-        """Remove a PID from the file. Returns True if found and removed."""
-        records = self.read_all()
-        new_records = [r for r in records if r.pid != pid]
-
-        if len(new_records) == len(records):
-            return False
-
-        self._write_all(new_records)
-        return True
-
-    # -- Housekeeping -------------------------------------------------------
-
-    def cleanup_dead(self) -> int:
-        """Remove all PIDs that are no longer running.
-
-        Returns number of dead PIDs removed.
-        """
-        records = self.read_all()
-        alive: list[PidRecord] = []
-        removed = 0
-
-        for r in records:
-            try:
-                os.kill(r.pid, 0)
-                alive.append(r)
-            except ProcessLookupError:
-                removed += 1
-            except PermissionError:
-                alive.append(r)  # Exists but owned by another user
-
-        if removed > 0:
-            self._write_all(alive)
-
-        return removed

package/src/superlocalmemory/infra/process_identity.py

@@ -1,180 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""Clock-independent identity for one running local process.
-
-Why this module exists
-----------------------
-A process's *wall-clock* creation time is not a stable identifier on every
-platform.  On Linux -- and therefore inside WSL2 -- psutil derives it as::
-
-    create_time = /proc/<pid>/stat:starttime / CLOCK_TICKS + /proc/stat:btime
-
-``starttime`` is boot-relative and never changes for the life of the process.
-``btime`` is the kernel's *estimate* of the boot instant, recomputed from the
-current wall clock, so any clock step moves ``btime`` and moves every process's
-computed ``create_time`` with it -- retroactively.  WSL2 periodically
-resynchronises its VM clock against the Windows host, so a ``create_time``
-recorded when the daemon started stops matching the ``create_time`` computed
-for that very same process minutes later.  Issue #104 measured a ~35 second
-divergence after roughly four minutes of uptime.
-
-Any *constant* tolerance on that comparison is a delay, not a fix: the
-divergence is unbounded and keeps growing.  The correct identifier is one the
-wall clock cannot move at all, which is what this module produces.
-
-What a start token is
----------------------
-``process_start_token_for(pid)`` returns an opaque string identifying one
-process *instance*, derived without reference to wall-clock time, or ``None``
-when the platform cannot supply one.  Tokens are only comparable when they use
-the same scheme, so :func:`compare_start_tokens` is deliberately tri-state --
-callers fall back to a weaker signal instead of guessing.
-
-Schemes
--------
-``lx1``
-    Linux/WSL2: ``lx1:<boot_id>:<starttime_ticks>``.  Both halves come straight
-    from procfs and are an opaque UUID and an integer tick count rather than
-    timestamps, so a clock adjustment cannot rewrite either.  ``boot_id``
-    differs across reboots, so a post-reboot PID collision can never look like
-    a match.
-``mn1``
-    Platforms where psutil exposes a monotonic creation time (macOS, NetBSD,
-    and Linux if procfs is unreadable): ``mn1:<value>``.  psutil builds its own
-    PID-reuse identity from exactly this value for exactly this reason.
-
-Windows has no monotonic variant, and needs none: its creation time comes from
-``GetProcessTimes``, a kernel timestamp that a clock adjustment does not
-rewrite.  ``process_start_token_for`` returns ``None`` there and the caller's
-creation-time comparison stays correct.
-"""
-
-from __future__ import annotations
-
-import logging
-import sys
-from pathlib import Path
-
-logger = logging.getLogger(__name__)
-
-LINUX_SCHEME = "lx1"
-MONOTONIC_SCHEME = "mn1"
-
-_PROCFS = Path("/proc")
-# "man proc" numbers /proc/<pid>/stat fields from 1 and starttime is field 22.
-# The comm field can contain spaces and parentheses, so parsing starts after
-# the last ')', which drops fields 1 and 2 -- hence 22 - 3 == 19.
-_STARTTIME_INDEX = 19
-
-
-def _boot_id() -> str | None:
-    """Return this boot's opaque kernel identifier, or None when unavailable.
-
-    Deliberately strict: without a boot id, two processes from different boots
-    could share a PID *and* a tick count, so the token would be unsound. A
-    missing boot id therefore means "no token" rather than a weaker token.
-    """
-    try:
-        value = (_PROCFS / "sys" / "kernel" / "random" / "boot_id").read_text(
-            encoding="utf-8",
-        ).strip()
-    except OSError:
-        return None
-    return value or None
-
-
-def _linux_start_ticks(pid: int) -> int | None:
-    """Return boot-relative start ticks for a PID from procfs, or None."""
-    try:
-        data = (_PROCFS / str(pid) / "stat").read_bytes()
-    except OSError:
-        return None
-    closing = data.rfind(b")")
-    if closing < 0:
-        return None
-    fields = data[closing + 2:].split()
-    if len(fields) <= _STARTTIME_INDEX:
-        return None
-    try:
-        return int(fields[_STARTTIME_INDEX])
-    except ValueError:
-        return None
-
-
-def _linux_start_token(pid: int) -> str | None:
-    boot_id = _boot_id()
-    if boot_id is None:
-        return None
-    ticks = _linux_start_ticks(pid)
-    if ticks is None:
-        return None
-    return f"{LINUX_SCHEME}:{boot_id}:{ticks}"
-
-
-def _monotonic_start_token(pid: int) -> str | None:
-    """Return psutil's monotonic creation time as a token, or None.
-
-    ``Process._proc.create_time(monotonic=True)`` is the same private accessor
-    psutil uses internally to build ``Process._ident``.  It is guarded on every
-    axis -- missing psutil, missing attribute, platforms whose implementation
-    takes no ``monotonic`` keyword (Windows) -- and degrades to ``None``.
-    """
-    try:
-        import psutil
-
-        platform_process = getattr(psutil.Process(pid), "_proc", None)
-    except Exception:  # noqa: BLE001 - identity probing must never raise
-        return None
-    create_time = getattr(platform_process, "create_time", None)
-    if create_time is None:
-        return None
-    try:
-        raw = create_time(monotonic=True)
-    except TypeError:
-        # No monotonic variant on this platform (Windows).
-        return None
-    except Exception:  # noqa: BLE001
-        return None
-    try:
-        return f"{MONOTONIC_SCHEME}:{float(raw)!r}"
-    except (TypeError, ValueError):
-        return None
-
-
-def process_start_token_for(pid: int) -> str | None:
-    """Return a clock-independent identity token for ``pid``, or None.
-
-    ``None`` is a normal answer, not an error: it means "this platform cannot
-    prove process identity without the wall clock", and the caller should fall
-    back to comparing creation times.
-    """
-    try:
-        pid = int(pid)
-    except (TypeError, ValueError):
-        return None
-    if pid <= 0:
-        return None
-    if sys.platform.startswith("linux"):
-        token = _linux_start_token(pid)
-        if token is not None:
-            return token
-    return _monotonic_start_token(pid)
-
-
-def compare_start_tokens(recorded: str | None, observed: str | None) -> bool | None:
-    """Tri-state comparison of two start tokens.
-
-    ``True``  -- same scheme, identical value: proven the same process instance.
-    ``False`` -- same scheme, different value: proven a different instance.
-    ``None``  -- not comparable (either side missing, or different schemes);
-                 the caller must fall back rather than assume either way.
-    """
-    if not recorded or not observed:
-        return None
-    if not isinstance(recorded, str) or not isinstance(observed, str):
-        return None
-    if recorded.split(":", 1)[0] != observed.split(":", 1)[0]:
-        return None
-    return recorded == observed