contextos-dd 0.1.0__py3-none-any.whl

contextos/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

contextos/archive/__init__.py

@@ -0,0 +1,3 @@
+from contextos.archive.store import ArchiveStore
+
+__all__ = ["ArchiveStore"]

contextos/archive/store.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+import logging
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+
+import lancedb
+import pyarrow as pa
+from fastapi.concurrency import run_in_threadpool
+
+log = logging.getLogger("contextos.archive")
+
+_TABLE = "turns"
+
+
+@dataclass(slots=True)
+class ArchivedRow:
+    archive_id: str
+    session_id: str
+    content: str
+    summary: str
+    distance: float = 0.0
+
+
+class ArchiveStore:
+    """LanceDB-backed archive for compacted turns. Single table, vector column
+    sized to settings.embedding_dim. All writes/reads off the event loop."""
+
+    def __init__(self, archive_dir: Path, embedding_dim: int) -> None:
+        archive_dir.mkdir(parents=True, exist_ok=True)
+        self._dim = embedding_dim
+        self._db = lancedb.connect(str(archive_dir))
+        self._ensure_table()
+
+    def _ensure_table(self) -> None:
+        if _TABLE in self._db.list_tables():
+            self._tbl = self._db.open_table(_TABLE)
+            return
+        schema = pa.schema([
+            pa.field("archive_id", pa.string()),
+            pa.field("session_id", pa.string()),
+            pa.field("content", pa.string()),
+            pa.field("summary", pa.string()),
+            pa.field("vector", pa.list_(pa.float32(), self._dim)),
+        ])
+        self._tbl = self._db.create_table(_TABLE, schema=schema)
+
+    async def add(self, session_id: str, content: str, summary: str,
+                  vector: list[float]) -> str:
+        archive_id = uuid.uuid4().hex
+        row = [{
+            "archive_id": archive_id,
+            "session_id": session_id,
+            "content": content,
+            "summary": summary,
+            "vector": vector,
+        }]
+        await run_in_threadpool(self._tbl.add, row)
+        return archive_id
+
+    async def search(self, session_id: str, query_vector: list[float],
+                     top_k: int = 3) -> list[ArchivedRow]:
+        def _q() -> list[ArchivedRow]:
+            df = (
+                self._tbl.search(query_vector)
+                .where(f"session_id = '{session_id}'")
+                .limit(top_k)
+                .to_list()
+            )
+            return [
+                ArchivedRow(
+                    archive_id=r["archive_id"],
+                    session_id=r["session_id"],
+                    content=r["content"],
+                    summary=r["summary"],
+                    distance=float(r.get("_distance", 0.0)),
+                )
+                for r in df
+            ]
+
+        return await run_in_threadpool(_q)

contextos/classifier/__init__.py

@@ -0,0 +1,3 @@
+from contextos.classifier.rules import Heat, Tagged, classify
+
+__all__ = ["Heat", "Tagged", "classify"]

contextos/classifier/rules.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import Any
+
+from contextos.proxy.payload import stringify_content
+
+HOT_WINDOW = 5     # last N turns always HOT
+WARM_WINDOW = 15   # turns within this distance from end stay WARM (else COLD)
+
+
+class Heat(StrEnum):
+    HOT = "HOT"
+    WARM = "WARM"
+    COLD = "COLD"
+    DEAD = "DEAD"
+
+
+@dataclass(slots=True)
+class Tagged:
+    index: int
+    message: dict[str, Any]
+    heat: Heat
+
+
+def _fingerprint(msg: dict[str, Any]) -> str:
+    role = str(msg.get("role", ""))
+    content = stringify_content(msg.get("content", "")).strip()
+    return hashlib.sha256(f"{role}\x1f{content}".encode()).hexdigest()
+
+
+def classify(messages: list[dict[str, Any]]) -> list[Tagged]:
+    """Tag each message in-order. Pure function — no I/O, no side effects."""
+    n = len(messages)
+    tagged: list[Tagged] = []
+    seen: set[str] = set()
+
+    for i, m in enumerate(messages):
+        from_end = n - 1 - i
+        fp = _fingerprint(m)
+        content = stringify_content(m.get("content", "")).strip()
+
+        # DEAD: byte-identical earlier message, or empty content.
+        if not content or fp in seen:
+            tagged.append(Tagged(i, m, Heat.DEAD))
+            continue
+        seen.add(fp)
+
+        if from_end < HOT_WINDOW:
+            heat = Heat.HOT
+        elif from_end < WARM_WINDOW:
+            heat = Heat.WARM
+        else:
+            heat = Heat.COLD
+        tagged.append(Tagged(i, m, heat))
+
+    return tagged

contextos/cli.py

@@ -0,0 +1,241 @@
+from __future__ import annotations
+
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+import httpx
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from contextos import __version__
+from contextos.daemon import read_pid
+from contextos.daemon import stop as stop_daemon
+from contextos.installer import detect, install_all, uninstall_all
+from contextos.settings import get_settings
+
+app = typer.Typer(add_completion=False, help="ContextOS - agentic memory lifecycle manager")
+console = Console()
+
+
+def _health_url() -> str:
+    s = get_settings()
+    return f"http://{s.proxy_host}:{s.proxy_port}/_contextos/health"
+
+
+def _is_running() -> bool:
+    try:
+        r = httpx.get(_health_url(), timeout=1.0)
+        return r.status_code == 200
+    except httpx.HTTPError:
+        return False
+
+
+@app.command()
+def version() -> None:
+    """Print version."""
+    console.print(f"context-os {__version__}")
+
+
+@app.command()
+def status() -> None:
+    """Show daemon and IDE status."""
+    s = get_settings()
+    running = _is_running()
+    pid = read_pid(s.pid_file)
+    table = Table(title="ContextOS Status")
+    table.add_column("Field")
+    table.add_column("Value")
+    table.add_row("daemon", "[green]running[/]" if running else "[red]stopped[/]")
+    table.add_row("pid", str(pid) if pid else "-")
+    table.add_row("proxy", f"http://{s.proxy_host}:{s.proxy_port}")
+    table.add_row("data dir", str(s.data_dir))
+    console.print(table)
+
+    targets = detect()
+    if targets:
+        t = Table(title="Detected IDEs")
+        t.add_column("IDE")
+        t.add_column("Config")
+        for x in targets:
+            t.add_row(x.name, str(x.config_path))
+        console.print(t)
+    else:
+        console.print("[yellow]No supported IDE configs found.[/]")
+
+
+@app.command()
+def start(foreground: bool = typer.Option(False, "--foreground", "-f")) -> None:
+    """Start the proxy daemon."""
+    if _is_running():
+        console.print("[yellow]Already running.[/]")
+        return
+    if foreground:
+        from contextos.daemon import run
+        run()
+        return
+    # Background spawn (detached). Logs go to ~/.contextos/logs/.
+    proc = subprocess.Popen(
+        [sys.executable, "-m", "contextos.daemon"],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+        close_fds=True,
+    )
+    for _ in range(50):
+        if _is_running():
+            console.print(f"[green]Daemon started[/] (pid {proc.pid})")
+            return
+        time.sleep(0.1)
+    console.print("[red]Daemon failed to become healthy in time.[/]")
+    raise typer.Exit(1)
+
+
+@app.command()
+def stop() -> None:
+    """Stop the proxy daemon."""
+    if stop_daemon():
+        console.print("[green]Stop signal sent.[/]")
+    else:
+        console.print("[yellow]Daemon not running.[/]")
+
+
+@app.command()
+def install(
+    skip_models: bool = typer.Option(
+        False, "--skip-models",
+        help="Don't download the local LLM + embedder during install "
+             "(use for air-gapped installs; first compaction will pull them).",
+    ),
+) -> None:
+    """Detect supported IDEs, back up their configs, patch base URLs, "
+    download local models, start daemon."""
+    patched = install_all()
+    if not patched:
+        console.print("[yellow]No IDE configs were patched (none detected).[/]")
+    else:
+        for t in patched:
+            console.print(f"[green]patched[/] {t.name}  ({t.config_path})")
+
+    if not skip_models:
+        console.print("\n[bold]Downloading local models[/] (one time, ~430 MB)")
+        try:
+            from contextos.llm import warmup as do_warmup
+            do_warmup(get_settings(), on_progress=lambda m: console.print(f"  {m}"))
+            console.print("[green]models ready[/]")
+        except Exception as e:
+            console.print(f"[yellow]warmup skipped: {e}[/]")
+            console.print("  models will be downloaded on first compaction.")
+    else:
+        console.print("[yellow]--skip-models set; models will download on first use.[/]")
+
+    if not _is_running():
+        start(foreground=False)
+
+
+@app.command()
+def warmup() -> None:
+    """Download the local LLM and embedder into the model cache."""
+    from contextos.llm import warmup as do_warmup
+    console.print("[bold]Downloading local models[/]")
+    try:
+        out = do_warmup(get_settings(), on_progress=lambda m: console.print(f"  {m}"))
+        for label, path in out.items():
+            console.print(f"[green]{label}[/]: {path}")
+    except Exception as e:
+        console.print(f"[red]warmup failed:[/] {e}")
+        raise typer.Exit(1) from None
+
+
+@app.command()
+def uninstall() -> None:
+    """Restore IDE configs and stop the daemon."""
+    restored = uninstall_all()
+    for t in restored:
+        console.print(f"[green]restored[/] {t.name}")
+    stop_daemon()
+    console.print("Done.")
+
+
+@app.command()
+def dashboard(
+    port: int = typer.Option(None, help="Override dashboard port (default from settings)"),
+) -> None:
+    """Launch the Streamlit dashboard."""
+    s = get_settings()
+    bind_port = port or s.dashboard_port
+    script = Path(__file__).parent / "dashboard" / "app.py"
+    cmd = [
+        sys.executable, "-m", "streamlit", "run", str(script),
+        "--server.port", str(bind_port),
+        "--server.address", s.proxy_host,
+        "--server.headless", "true",
+        "--browser.gatherUsageStats", "false",
+    ]
+    console.print(f"[green]Launching dashboard[/] at http://{s.proxy_host}:{bind_port}")
+    try:
+        subprocess.run(cmd, check=False)
+    except FileNotFoundError:
+        console.print("[red]Streamlit not installed.[/] Run: pip install 'context-os[dashboard]'")
+        raise typer.Exit(1) from None
+
+
+def _check(label: str, ok: bool, detail: str = "") -> None:
+    mark = "[green]OK[/]" if ok else "[red]!![/]"
+    console.print(f"  {mark} {label}" + (f"  [dim]{detail}[/]" if detail else ""))
+
+
+def _module_present(name: str) -> bool:
+    import importlib.util
+    return importlib.util.find_spec(name) is not None
+
+
+def _port_free(host: str, port: int) -> bool:
+    import socket
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.settimeout(0.5)
+        return sock.connect_ex((host, port)) != 0
+
+
+@app.command()
+def doctor() -> None:
+    """Run preflight checks: paths, deps, ports, IDE configs, daemon health."""
+    s = get_settings()
+    console.print("[bold]ContextOS doctor[/]")
+
+    console.print("\n[bold]Paths[/]")
+    _check(f"data dir   {s.data_dir}", s.data_dir.exists())
+    _check(f"log dir    {s.log_dir}", s.log_dir.exists())
+    _check(f"db file    {s.db_path}", s.db_path.exists(), "created on first request")
+    _check(f"model dir  {s.model_dir}", s.model_dir.exists())
+
+    console.print("\n[bold]Dependencies[/]")
+    _check("llama-cpp-python", _module_present("llama_cpp"),
+           "required for compaction summaries")
+    _check("fastembed",        _module_present("fastembed"),
+           "required for retrieval embeddings")
+    _check("lancedb",          _module_present("lancedb"))
+    _check("streamlit",        _module_present("streamlit"),
+           "install with: pip install 'context-os[dashboard]'")
+
+    console.print("\n[bold]Network[/]")
+    running = _is_running()
+    if running:
+        _check(f"proxy http://{s.proxy_host}:{s.proxy_port}", True, "daemon running")
+    else:
+        _check(f"port {s.proxy_port} free", _port_free(s.proxy_host, s.proxy_port),
+               "daemon stopped - port should be free")
+    _check(f"dashboard port {s.dashboard_port} free",
+           _port_free(s.proxy_host, s.dashboard_port))
+
+    console.print("\n[bold]IDEs[/]")
+    targets = detect()
+    if not targets:
+        console.print("  [yellow]no supported IDE configs found[/]")
+    for t in targets:
+        _check(f"{t.name}  {t.config_path}", True)
+
+
+if __name__ == "__main__":
+    app()

contextos/compactor/__init__.py

@@ -0,0 +1,3 @@
+from contextos.compactor.service import CompactionJob, CompactorService
+
+__all__ = ["CompactionJob", "CompactorService"]

contextos/compactor/prompt.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+SUMMARY_INSTRUCTIONS = """You are a context compactor for a long coding session. Summarise
+the conversation transcript below into a dense, third-person narrative paragraph
+of AT MOST 100 words.
+
+You MUST preserve:
+- Entity names (files, functions, classes, variables)
+- Decisions made and the reasoning given for them
+- Breaking changes or API surface modifications
+- File paths that were modified
+- Open questions or unresolved items
+
+You MUST omit:
+- Resolved errors and their fixes (unless the fix is itself a load-bearing decision)
+- Reasoning chains that led to a final answer (keep the answer, drop the chain)
+- Polite filler, acknowledgements, status updates
+- Duplicate file contents
+
+Output the summary only — no preamble, no markdown headings, no commentary on the task.
+"""
+
+
+def build_summary_prompt(turns_text: str) -> str:
+    return f"{SUMMARY_INSTRUCTIONS}\n\n--- TRANSCRIPT ---\n{turns_text}\n--- END ---\n\nSummary:"

contextos/compactor/service.py

@@ -0,0 +1,124 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import logging
+from dataclasses import dataclass
+
+from contextos.archive import ArchiveStore
+from contextos.compactor.prompt import build_summary_prompt
+from contextos.llm import Embedder, Generator, LLMUnavailable
+from contextos.proxy.payload import stringify_content
+from contextos.session_memory import SessionMemory
+from contextos.settings import Settings
+
+log = logging.getLogger("contextos.compactor")
+
+
+@dataclass(slots=True)
+class CompactionJob:
+    session_id: str
+    run_id: str
+    messages: list[dict]
+
+
+def _render_transcript(messages: list[dict]) -> str:
+    parts: list[str] = []
+    for m in messages:
+        role = str(m.get("role", "user"))
+        content = stringify_content(m.get("content", "")).strip()
+        if content:
+            parts.append(f"{role.upper()}: {content}")
+    return "\n\n".join(parts)
+
+
+class CompactorService:
+    """Async, fire-and-forget compaction worker. Never blocks the proxy hot path."""
+
+    def __init__(
+        self,
+        settings: Settings,
+        generator: Generator,
+        embedder: Embedder,
+        archive: ArchiveStore,
+        memory: SessionMemory,
+    ) -> None:
+        self._settings = settings
+        self._generator = generator
+        self._embedder = embedder
+        self._archive = archive
+        self._memory = memory
+        self._queue: asyncio.Queue[CompactionJob] = asyncio.Queue(maxsize=128)
+        self._task: asyncio.Task | None = None
+        self._seen_runs: set[tuple[str, str]] = set()
+
+    def start(self) -> None:
+        if self._task is None or self._task.done():
+            self._task = asyncio.create_task(self._worker(), name="contextos-compactor")
+
+    async def stop(self) -> None:
+        if self._task is None:
+            return
+        self._task.cancel()
+        with contextlib.suppress(asyncio.CancelledError):
+            await self._task
+        self._task = None
+
+    def submit(self, job: CompactionJob) -> bool:
+        key = (job.session_id, job.run_id)
+        if key in self._seen_runs:
+            return False
+        if job.run_id in self._memory.get(job.session_id).summaries:
+            return False
+        try:
+            self._queue.put_nowait(job)
+            self._seen_runs.add(key)
+            return True
+        except asyncio.QueueFull:
+            log.warning("compactor queue full; dropping job for %s", job.session_id)
+            return False
+
+    async def _worker(self) -> None:
+        while True:
+            job = await self._queue.get()
+            try:
+                await self._process(job)
+            except Exception as e:
+                log.warning("compaction failed for %s/%s: %s",
+                            job.session_id, job.run_id, e)
+            finally:
+                self._queue.task_done()
+
+    async def _process(self, job: CompactionJob) -> None:
+        transcript = _render_transcript(job.messages)
+        if not transcript:
+            return
+        try:
+            summary = await self._generator.generate(
+                build_summary_prompt(transcript),
+                max_tokens=self._settings.llm_max_tokens,
+            )
+        except LLMUnavailable as e:
+            log.info("generator unavailable, skipping compaction: %s", e)
+            return
+        if not summary:
+            return
+
+        self._memory.put_summary(job.session_id, job.run_id, summary)
+
+        for m in job.messages:
+            content = stringify_content(m.get("content", "")).strip()
+            if not content:
+                continue
+            try:
+                vec = await self._embedder.embed(content)
+            except LLMUnavailable:
+                continue
+            if len(vec) != self._settings.embedding_dim:
+                log.warning("embedding dim %d != expected %d; skipping archive",
+                            len(vec), self._settings.embedding_dim)
+                continue
+            try:
+                await self._archive.add(job.session_id, content, summary, vec)
+            except Exception as e:
+                log.warning("archive add failed: %s", e)

contextos/daemon.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import contextlib
+import os
+import signal
+from pathlib import Path
+
+import uvicorn
+
+from contextos.proxy import create_app
+from contextos.settings import configure_logging, get_settings
+
+
+def run() -> None:
+    s = get_settings()
+    configure_logging(s)
+    s.pid_file.write_text(str(os.getpid()), encoding="utf-8")
+    try:
+        uvicorn.run(
+            create_app(s),
+            host=s.proxy_host,
+            port=s.proxy_port,
+            log_config=None,
+            access_log=False,
+        )
+    finally:
+        with contextlib.suppress(FileNotFoundError):
+            s.pid_file.unlink()
+
+
+def read_pid(pid_file: Path) -> int | None:
+    try:
+        return int(pid_file.read_text(encoding="utf-8").strip())
+    except (FileNotFoundError, ValueError):
+        return None
+
+
+def stop() -> bool:
+    s = get_settings()
+    pid = read_pid(s.pid_file)
+    if pid is None:
+        return False
+    try:
+        os.kill(pid, signal.SIGTERM)
+        return True
+    except (OSError, ProcessLookupError):
+        return False
+
+
+if __name__ == "__main__":
+    run()