codecortex-context-engine 0.1.0a1__py3-none-any.whl

codecortex/memory/knowledge.py

@@ -0,0 +1,193 @@
+"""Automatic project knowledge extraction and persistence."""
+
+from __future__ import annotations
+
+import json
+from collections import Counter
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+from codecortex.git_intelligence import GitIntelligence
+from codecortex.memory.json_store import JsonMemoryStore
+
+_LANGUAGE_BY_SUFFIX = {
+    ".py": "Python",
+    ".js": "JavaScript",
+    ".jsx": "JavaScript",
+    ".ts": "TypeScript",
+    ".tsx": "TypeScript",
+    ".go": "Go",
+    ".rs": "Rust",
+    ".java": "Java",
+    ".c": "C",
+    ".h": "C/C++",
+    ".cc": "C++",
+    ".cpp": "C++",
+    ".cxx": "C++",
+    ".hpp": "C++",
+    ".cs": "C#",
+    ".php": "PHP",
+    ".rb": "Ruby",
+}
+
+_PACKAGE_FILES = {
+    "pyproject.toml": "Python/pip",
+    "requirements.txt": "Python/pip",
+    "package.json": "Node.js",
+    "pnpm-lock.yaml": "pnpm",
+    "yarn.lock": "Yarn",
+    "go.mod": "Go modules",
+    "Cargo.toml": "Cargo",
+    "pom.xml": "Maven",
+    "build.gradle": "Gradle",
+    "composer.json": "Composer",
+    "Gemfile": "Bundler",
+}
+
+_TEST_MARKERS = {
+    "pytest": ("pytest.ini", "conftest.py", "pytest"),
+    "vitest": ("vitest.config", "vitest"),
+    "jest": ("jest.config", "jest"),
+    "go test": ("_test.go",),
+    "cargo test": ("#[test]",),
+    "JUnit": ("junit", "@test"),
+    "RSpec": ("spec_helper", "rspec"),
+}
+
+_ARCHITECTURE_DIRS = {
+    "controllers": "controller layer",
+    "controller": "controller layer",
+    "services": "service layer",
+    "service": "service layer",
+    "repositories": "repository layer",
+    "repository": "repository layer",
+    "models": "model layer",
+    "domain": "domain layer",
+    "api": "API layer",
+    "routes": "routing layer",
+    "adapters": "adapter layer",
+    "ports": "ports layer",
+    "core": "core layer",
+    "infra": "infrastructure layer",
+    "infrastructure": "infrastructure layer",
+}
+
+_ENTRY_NAMES = {
+    "main.py",
+    "app.py",
+    "manage.py",
+    "index.js",
+    "index.ts",
+    "server.js",
+    "server.ts",
+    "main.go",
+    "main.rs",
+    "Program.cs",
+    "Main.java",
+}
+
+_EXCLUDED = {".git", ".codecortex", ".venv", "venv", "node_modules", "dist", "build"}
+
+
+@dataclass(frozen=True, slots=True)
+class ProjectKnowledge:
+    languages: tuple[tuple[str, int], ...]
+    package_systems: tuple[str, ...]
+    entry_points: tuple[str, ...]
+    test_frameworks: tuple[str, ...]
+    architecture: tuple[str, ...]
+    hot_files: tuple[str, ...]
+
+    def facts(self) -> dict[str, str]:
+        return {
+            "languages": ", ".join(f"{name} ({count})" for name, count in self.languages),
+            "package_systems": ", ".join(self.package_systems) or "unknown",
+            "entry_points": ", ".join(self.entry_points) or "not detected",
+            "test_frameworks": ", ".join(self.test_frameworks) or "not detected",
+            "architecture": ", ".join(self.architecture) or "no strong convention detected",
+            "hot_files": ", ".join(self.hot_files) or "not available",
+        }
+
+
+class ProjectKnowledgeExtractor:
+    def __init__(self, root: Path) -> None:
+        self.root = root.resolve()
+
+    def _files(self) -> list[Path]:
+        result: list[Path] = []
+        for path in self.root.rglob("*"):
+            if not path.is_file():
+                continue
+            relative = path.relative_to(self.root)
+            if any(part in _EXCLUDED for part in relative.parts):
+                continue
+            result.append(path)
+        return result
+
+    def extract(self) -> ProjectKnowledge:
+        files = self._files()
+        languages: Counter[str] = Counter()
+        package_systems: set[str] = set()
+        entry_points: list[str] = []
+        architecture: set[str] = set()
+        searchable: list[str] = []
+
+        for path in files:
+            relative = path.relative_to(self.root)
+            language = _LANGUAGE_BY_SUFFIX.get(path.suffix.lower())
+            if language:
+                languages[language] += 1
+            package = _PACKAGE_FILES.get(path.name)
+            if package:
+                package_systems.add(package)
+            if path.name in _ENTRY_NAMES:
+                entry_points.append(relative.as_posix())
+            for part in relative.parts[:-1]:
+                marker = _ARCHITECTURE_DIRS.get(part.lower())
+                if marker:
+                    architecture.add(marker)
+            if path.stat().st_size <= 512_000:
+                try:
+                    searchable.append(path.read_text(encoding="utf-8").lower())
+                except (OSError, UnicodeDecodeError):
+                    pass
+
+        joined = "\n".join(searchable)
+        test_frameworks = {
+            name
+            for name, markers in _TEST_MARKERS.items()
+            if any(marker.lower() in joined or any(marker.lower() in p.name.lower() for p in files)
+                   for marker in markers)
+        }
+        git = GitIntelligence(self.root).analyze(limit=300)
+        return ProjectKnowledge(
+            languages=tuple(languages.most_common()),
+            package_systems=tuple(sorted(package_systems)),
+            entry_points=tuple(sorted(set(entry_points))),
+            test_frameworks=tuple(sorted(test_frameworks)),
+            architecture=tuple(sorted(architecture)),
+            hot_files=tuple(item.path for item in git.hot_files[:10]),
+        )
+
+    def save(self, knowledge: ProjectKnowledge | None = None) -> Path:
+        knowledge = knowledge or self.extract()
+        path = self.root / ".codecortex" / "knowledge" / "project.json"
+        path.parent.mkdir(parents=True, exist_ok=True)
+        temp = path.with_suffix(".tmp")
+        temp.write_text(
+            json.dumps(asdict(knowledge), ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+        temp.replace(path)
+        return path
+
+    async def remember(
+        self,
+        store: JsonMemoryStore,
+        namespace: str = "project_knowledge",
+    ) -> ProjectKnowledge:
+        knowledge = self.extract()
+        self.save(knowledge)
+        for key, value in knowledge.facts().items():
+            await store.put(namespace, key, value)
+        return knowledge

codecortex/memory/team_store.py

@@ -0,0 +1,193 @@
+"""Concurrent shared team memory backed by SQLite."""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+
+from codecortex.core.contracts import MemoryStore
+
+
+@dataclass(frozen=True, slots=True)
+class TeamMemoryEntry:
+    namespace: str
+    key: str
+    value: str
+    revision: int
+    actor: str
+    source: str
+    tags: tuple[str, ...]
+    updated_at: str
+
+
+class RevisionConflict(RuntimeError):
+    pass
+
+
+class TeamMemoryStore(MemoryStore):
+    """Shared memory with revisions, audit history, and optimistic concurrency."""
+
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._initialize()
+
+    def _connect(self) -> sqlite3.Connection:
+        connection = sqlite3.connect(self.path, timeout=5.0)
+        connection.row_factory = sqlite3.Row
+        connection.execute("PRAGMA journal_mode=WAL")
+        connection.execute("PRAGMA synchronous=NORMAL")
+        connection.execute("PRAGMA busy_timeout=5000")
+        return connection
+
+    def _initialize(self) -> None:
+        with self._connect() as connection:
+            connection.executescript(
+                """
+                CREATE TABLE IF NOT EXISTS memories (
+                    namespace TEXT NOT NULL,
+                    key TEXT NOT NULL,
+                    value TEXT NOT NULL,
+                    revision INTEGER NOT NULL,
+                    actor TEXT NOT NULL,
+                    source TEXT NOT NULL,
+                    tags TEXT NOT NULL,
+                    updated_at TEXT NOT NULL,
+                    PRIMARY KEY(namespace, key)
+                );
+                CREATE TABLE IF NOT EXISTS memory_history (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    namespace TEXT NOT NULL,
+                    key TEXT NOT NULL,
+                    value TEXT NOT NULL,
+                    revision INTEGER NOT NULL,
+                    actor TEXT NOT NULL,
+                    source TEXT NOT NULL,
+                    tags TEXT NOT NULL,
+                    updated_at TEXT NOT NULL
+                );
+                CREATE INDEX IF NOT EXISTS idx_memory_namespace
+                    ON memories(namespace);
+                CREATE INDEX IF NOT EXISTS idx_history_key
+                    ON memory_history(namespace, key, revision);
+                """
+            )
+
+    async def put(self, namespace: str, key: str, value: str) -> None:
+        self.put_entry(namespace, key, value)
+
+    def put_entry(
+        self,
+        namespace: str,
+        key: str,
+        value: str,
+        *,
+        actor: str = "system",
+        source: str = "manual",
+        tags: tuple[str, ...] = (),
+        expected_revision: int | None = None,
+    ) -> TeamMemoryEntry:
+        if not namespace.strip() or not key.strip():
+            raise ValueError("namespace and key are required")
+        now = datetime.now(UTC).isoformat()
+        encoded_tags = json.dumps(sorted(set(tags)), ensure_ascii=False)
+        with self._connect() as connection:
+            connection.execute("BEGIN IMMEDIATE")
+            row = connection.execute(
+                "SELECT revision FROM memories WHERE namespace = ? AND key = ?",
+                (namespace, key),
+            ).fetchone()
+            current_revision = int(row["revision"]) if row else 0
+            if expected_revision is not None and expected_revision != current_revision:
+                raise RevisionConflict(
+                    f"expected revision {expected_revision}, found {current_revision}"
+                )
+            revision = current_revision + 1
+            connection.execute(
+                """
+                INSERT INTO memories(namespace, key, value, revision, actor, source, tags, updated_at)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(namespace, key) DO UPDATE SET
+                    value = excluded.value,
+                    revision = excluded.revision,
+                    actor = excluded.actor,
+                    source = excluded.source,
+                    tags = excluded.tags,
+                    updated_at = excluded.updated_at
+                """,
+                (namespace, key, value, revision, actor, source, encoded_tags, now),
+            )
+            connection.execute(
+                """
+                INSERT INTO memory_history(namespace, key, value, revision, actor, source, tags, updated_at)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (namespace, key, value, revision, actor, source, encoded_tags, now),
+            )
+        return TeamMemoryEntry(namespace, key, value, revision, actor, source, tags, now)
+
+    async def get(self, namespace: str, key: str) -> str | None:
+        entry = self.get_entry(namespace, key)
+        return entry.value if entry else None
+
+    def get_entry(self, namespace: str, key: str) -> TeamMemoryEntry | None:
+        with self._connect() as connection:
+            row = connection.execute(
+                "SELECT * FROM memories WHERE namespace = ? AND key = ?",
+                (namespace, key),
+            ).fetchone()
+        return self._row_to_entry(row) if row else None
+
+    async def search(self, namespace: str, query: str, limit: int = 10) -> list[str]:
+        return [entry.value for entry in self.search_entries(namespace, query, limit)]
+
+    def search_entries(self, namespace: str, query: str, limit: int = 10) -> list[TeamMemoryEntry]:
+        terms = [term.lower() for term in query.split() if term.strip()]
+        with self._connect() as connection:
+            rows = connection.execute(
+                "SELECT * FROM memories WHERE namespace = ? ORDER BY updated_at DESC",
+                (namespace,),
+            ).fetchall()
+        scored: list[tuple[int, TeamMemoryEntry]] = []
+        for row in rows:
+            entry = self._row_to_entry(row)
+            haystack = f"{entry.key} {entry.value} {' '.join(entry.tags)}".lower()
+            score = sum(3 if term in entry.key.lower() else 1 for term in terms if term in haystack)
+            if score or not terms:
+                scored.append((score, entry))
+        scored.sort(key=lambda item: (-item[0], item[1].updated_at), reverse=False)
+        return [entry for _, entry in scored[: max(1, limit)]]
+
+    def history(self, namespace: str, key: str, limit: int = 50) -> list[TeamMemoryEntry]:
+        with self._connect() as connection:
+            rows = connection.execute(
+                """
+                SELECT namespace, key, value, revision, actor, source, tags, updated_at
+                FROM memory_history
+                WHERE namespace = ? AND key = ?
+                ORDER BY revision DESC
+                LIMIT ?
+                """,
+                (namespace, key, max(1, limit)),
+            ).fetchall()
+        return [self._row_to_entry(row) for row in rows]
+
+    @staticmethod
+    def _row_to_entry(row: sqlite3.Row) -> TeamMemoryEntry:
+        try:
+            tags = tuple(str(item) for item in json.loads(row["tags"]))
+        except (json.JSONDecodeError, TypeError):
+            tags = ()
+        return TeamMemoryEntry(
+            namespace=str(row["namespace"]),
+            key=str(row["key"]),
+            value=str(row["value"]),
+            revision=int(row["revision"]),
+            actor=str(row["actor"]),
+            source=str(row["source"]),
+            tags=tags,
+            updated_at=str(row["updated_at"]),
+        )

codecortex/orchestrator.py

@@ -0,0 +1,153 @@
+"""Core request orchestration."""
+
+from __future__ import annotations
+
+import asyncio
+from time import perf_counter
+
+from codecortex.context import BudgetContextProcessor
+from codecortex.core.models import AgentRequest, Capability, EngineResult, ExecutionResult
+from codecortex.engines import EngineRegistry
+from codecortex.router import AdaptiveRouter
+from codecortex.telemetry import TelemetryCollector
+from codecortex.tracing import TaskTraceRecorder
+
+
+class Orchestrator:
+    """Route requests, execute independent engines concurrently, and fit context."""
+
+    def __init__(
+        self,
+        registry: EngineRegistry,
+        router: AdaptiveRouter,
+        context_processor: BudgetContextProcessor | None = None,
+        telemetry: TelemetryCollector | None = None,
+        tracer: TaskTraceRecorder | None = None,
+    ) -> None:
+        self.registry = registry
+        self.router = router
+        self.context_processor = context_processor or BudgetContextProcessor()
+        self.telemetry = telemetry or TelemetryCollector()
+        self.tracer = tracer
+
+    async def execute(self, request: AgentRequest) -> ExecutionResult:
+        if self.tracer is None:
+            return await self._execute(request, None, None)
+        trace_id = str(request.metadata.get("trace_id") or self.tracer.new_trace_id())
+        attributes: dict[str, object] = {"query_chars": len(request.query)}
+        async with self.tracer.async_span(
+            "request.execute",
+            trace_id=trace_id,
+            attributes=attributes,
+        ) as root_span:
+            result = await self._execute(request, trace_id, root_span)
+            attributes["context_tokens"] = result.context_tokens
+            attributes["capabilities"] = [item.value for item in result.plan.selected]
+            return result.model_copy(
+                update={"metadata": {**result.metadata, "trace_id": trace_id}}
+            )
+
+    async def _run_engine(
+        self,
+        capability: Capability,
+        request: AgentRequest,
+        trace_id: str | None,
+        parent_span: str | None,
+    ) -> EngineResult | None:
+        engine = self.registry.get(capability)
+        if engine is None or not await engine.health():
+            self.telemetry.emit("engine.skipped", capability=capability.value)
+            return None
+        started = perf_counter()
+        try:
+            if self.tracer and trace_id:
+                attrs: dict[str, object] = {"capability": capability.value}
+                async with self.tracer.async_span(
+                    "engine.execute",
+                    trace_id=trace_id,
+                    parent_id=parent_span,
+                    attributes=attrs,
+                ):
+                    result = await engine.execute(request)
+                    attrs["chunks"] = len(result.chunks)
+                    attrs["context_tokens"] = sum(chunk.tokens for chunk in result.chunks)
+            else:
+                result = await engine.execute(request)
+        except Exception as exc:
+            self.telemetry.emit(
+                "engine.failed",
+                capability=capability.value,
+                error_type=type(exc).__name__,
+            )
+            if request.metadata.get("strict_engines"):
+                raise
+            return None
+        self.telemetry.emit(
+            "engine.executed",
+            capability=capability.value,
+            duration_ms=(perf_counter() - started) * 1000,
+        )
+        return result
+
+    async def _execute(
+        self,
+        request: AgentRequest,
+        trace_id: str | None,
+        parent_span: str | None,
+    ) -> ExecutionResult:
+        plan = self.router.route(request)
+        self.telemetry.emit(
+            "route.created",
+            kind=plan.request_kind.value,
+            capabilities=[capability.value for capability in plan.selected],
+        )
+        if self.tracer and trace_id:
+            self.tracer.record(
+                "route.created",
+                trace_id=trace_id,
+                parent_id=parent_span,
+                attributes={
+                    "kind": plan.request_kind.value,
+                    "capabilities": [item.value for item in plan.selected],
+                },
+            )
+
+        capabilities = [item for item in plan.selected if item != Capability.CONTEXT]
+        outcomes = await asyncio.gather(
+            *(
+                self._run_engine(capability, request, trace_id, parent_span)
+                for capability in capabilities
+            )
+        )
+        results = [result for result in outcomes if result is not None]
+        all_chunks = [chunk for result in results for chunk in result.chunks]
+
+        original_tokens = sum(chunk.tokens for chunk in all_chunks)
+        fitted = await self.context_processor.fit(all_chunks, plan.context_budget)
+        fitted_sources = {(chunk.source, chunk.content) for chunk in fitted}
+        normalized_results: list[EngineResult] = []
+        for result in results:
+            kept = [
+                chunk for chunk in result.chunks if (chunk.source, chunk.content) in fitted_sources
+            ]
+            normalized_results.append(result.model_copy(update={"chunks": kept}))
+
+        context_tokens = sum(chunk.tokens for chunk in fitted)
+        self.telemetry.emit(
+            "context.fitted",
+            budget=plan.context_budget,
+            original=original_tokens,
+            used=context_tokens,
+            saved=max(0, original_tokens - context_tokens),
+            chunks=len(fitted),
+        )
+        return ExecutionResult(
+            request=request,
+            plan=plan,
+            results=normalized_results,
+            context_tokens=context_tokens,
+            metadata={
+                "original_context_tokens": original_tokens,
+                "context_tokens_saved": max(0, original_tokens - context_tokens),
+            },
+        )