core-runtime 0.1.1__py3-none-any.whl

will/workers/repo_crawler.py

@@ -0,0 +1,171 @@
+# src/will/workers/repo_crawler.py
+"""
+Repo Crawler Worker — structural self-model builder.
+
+- Declaration:  .intent/workers/repo_crawler.yaml
+- Class:        sensing
+- Phase:        audit
+- Schedule:     max_interval=86400s, glide_off=8640s (10% default)
+
+Responsibilities (one per run):
+  1. Delegate full crawl orchestration to CrawlService.run_crawl().
+  2. Post blackboard report with summary stats.
+
+Crawl logic (scope config, call-graph extraction, artifact registration)
+now lives in body.services.crawl_service and is imported here for
+backward-compatibility access via this module's namespace.
+"""
+
+from __future__ import annotations
+
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+# Re-exported from crawl_service so any code that previously imported these
+# from this module continues to resolve them without changes.
+from shared.logger import getLogger
+from shared.workers.base import Worker
+
+
+logger = getLogger(__name__)
+
+
+# ID: f1a2b3c4-d5e6-7890-abcd-ef1234567891
+class RepoCrawlerWorker(Worker):
+    """
+    Sensing worker. Delegates crawl orchestration to CrawlService.run_crawl()
+    and posts the result to the blackboard.
+    """
+
+    declaration_name = "repo_crawler"
+
+    def __init__(self, cognitive_service: Any = None) -> None:
+        from shared.infrastructure.bootstrap_registry import BootstrapRegistry
+
+        super().__init__()
+        self._cognitive_service = cognitive_service
+        self._repo_root: Path = BootstrapRegistry.get_repo_path()
+        schedule = self._declaration.get("mandate", {}).get("schedule", {})
+        self._max_interval: int = schedule.get("max_interval", 86400)
+        self._glide_off: int = schedule.get(
+            "glide_off", max(int(self._max_interval * 0.10), 10)
+        )
+
+    # ID: ec3e046d-7df4-4e47-b34b-973f8bbe617f
+    async def run_loop(self) -> None:
+        """
+        Continuous self-scheduling loop. Runs one crawl pass per
+        max_interval seconds.
+        """
+        logger.info(
+            "RepoCrawlerWorker: starting loop (max_interval=%ds, glide_off=%ds)",
+            self._max_interval,
+            self._glide_off,
+        )
+
+        await self._register()
+
+        while True:
+            cycle_start = time.monotonic()
+            try:
+                await self.run()
+            except Exception as exc:
+                logger.error("RepoCrawlerWorker: cycle failed: %s", exc, exc_info=True)
+                try:
+                    await self._post_entry(
+                        entry_type="report",
+                        subject="repo_crawler.cycle_error",
+                        payload={"error": str(exc)},
+                        status="abandoned",
+                    )
+                except Exception:
+                    logger.exception("RepoCrawlerWorker: failed to post error report")
+
+            elapsed = time.monotonic() - cycle_start
+            await __import__("asyncio").sleep(max(self._max_interval - elapsed, 0))
+
+    # ID: b2c3d4e5-f6a7-8901-bcde-f12345678903
+    async def run(self) -> None:
+        """Crawl repository — delegate to CrawlService.run_crawl."""
+        await self.post_heartbeat()
+        from body.services.service_registry import service_registry
+
+        logger.info("RepoCrawlerWorker: starting crawl pass")
+        svc = await service_registry.get_crawl_service()
+        stats = await svc.run_crawl(self._repo_root, self._cognitive_service)
+
+        completed_at = datetime.now(UTC).isoformat()
+
+        # ADR-070 D8 writer-as-sensor: emit a coherence.repo_artifacts.drift
+        # finding when this cycle either reaped orphans (status=resolved) or
+        # skipped the reap because a safety guard tripped (status=open). The
+        # two cases are mutually exclusive: the orchestrator either runs the
+        # DELETE inline OR records the guard state and leaves orphans_reaped
+        # at 0. The worker reads stats and posts the appropriate shape.
+        orphans_reaped = stats.get("orphans_reaped", 0)
+        guard = stats.get("coherence_guard") or {}
+
+        if guard.get("triggered"):
+            # Safety rail tripped — reap was SKIPPED. Post an OPEN finding
+            # so a governor inspects the candidate list before any rows
+            # are removed. This is the "system noticed and stopped" signal.
+            await self._post_entry(
+                entry_type="finding",
+                subject="coherence.repo_artifacts.drift",
+                payload={
+                    "rule_id": "coherence.repo_artifacts.drift",
+                    "severity": "high",
+                    "drift_class": "excessive_or_partial_walk",
+                    "trigger": guard.get("trigger"),
+                    "proposed_reaps": guard.get("proposed_reaps"),
+                    "total_known": guard.get("total_known"),
+                    "total_walked": guard.get("total_walked"),
+                    "walk_floor_required": guard.get("walk_floor_required"),
+                    "reap_hard_cap": guard.get("reap_hard_cap"),
+                    "reap_fraction_cap": guard.get("reap_fraction_cap"),
+                    "sample_paths": guard.get("sample_paths", []),
+                    "remediation": "skipped",
+                    "remediation_hint": (
+                        "Reap exceeded the ADR-070 D8 safety bound. "
+                        "Inspect the candidate list before any rows "
+                        "are removed. If the candidates are legitimate "
+                        "(e.g. a large refactor deleting many files), "
+                        "delete them manually via SQL. If a config "
+                        "issue (narrowed crawl scope, missing mount), "
+                        "restore the source-of-truth state before the "
+                        "next cycle."
+                    ),
+                    "detected_at": completed_at,
+                    "pair_id": "repo_artifacts ↔ filesystem",
+                },
+                status="open",
+            )
+        elif orphans_reaped > 0:
+            # Normal inline reap completed — record audit-trail attribution
+            # with status=resolved (the writer handled remediation in-cycle,
+            # ADR-070 D4 inline-remediation pattern).
+            await self._post_entry(
+                entry_type="finding",
+                subject="coherence.repo_artifacts.drift",
+                payload={
+                    "rule_id": "coherence.repo_artifacts.drift",
+                    "severity": "medium",
+                    "drift_class": "reaped_inline",
+                    "orphan_count": orphans_reaped,
+                    "remediation": "inline-reap",
+                    "remediated_at": completed_at,
+                    "pair_id": "repo_artifacts ↔ filesystem",
+                },
+                status="resolved",
+            )
+
+        await self.post_report(
+            subject="repo.crawl.complete",
+            payload={
+                **stats,
+                "completed_at": completed_at,
+            },
+        )
+        logger.info("RepoCrawlerWorker: crawl complete — %s", stats)

will/workers/repo_embedding/__init__.py

@@ -0,0 +1,18 @@
+# src/will/workers/repo_embedding/__init__.py
+# repo_embedding/__init__.py
+"""
+Package split from repo_embedder.py.
+"""
+
+from __future__ import annotations
+
+from .helpers import (
+    _chunk_by_function,
+    _chunk_by_heading,
+    _chunk_by_symbol,
+    _chunk_file,
+    _chunk_whole,
+    _embed_and_upsert,
+    _split_large,
+)
+from .repo_embedder_workers import RepoEmbedderWorker, logger

will/workers/repo_embedding/helpers.py

@@ -0,0 +1,245 @@
+# src/will/workers/repo_embedding/helpers.py
+# helpers.py
+"""Contains helper functions for chunking files and upserting them to Qdrant."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from shared.infrastructure.intent.operational_config import load_operational_config
+
+
+_CFG_CHK = load_operational_config().chunking
+
+
+def _chunk_file(file_path: Path, artifact_type: str) -> list[dict[str, Any]]:
+    """
+    Chunk a file into semantic units for embedding.
+    Returns list of chunk dicts: {text, metadata}.
+    """
+    content = file_path.read_text(encoding="utf-8", errors="replace")
+    rel_path = str(file_path)
+
+    if artifact_type == "python":
+        return _chunk_by_symbol(content, rel_path)
+    elif artifact_type in ("doc", "report", "infra"):
+        return _chunk_by_heading(content, rel_path)
+    elif artifact_type == "test":
+        return _chunk_by_function(content, rel_path)
+    elif artifact_type == "prompt":
+        return _chunk_whole(content, rel_path)
+    elif artifact_type == "intent":
+        return _chunk_by_heading(content, rel_path)
+    else:
+        return _chunk_by_heading(content, rel_path)
+
+
+def _chunk_by_symbol(content: str, source: str) -> list[dict[str, Any]]:
+    """
+    Chunk Python source by class and function boundaries using AST.
+
+    Each top-level class and function becomes one chunk (with its full body).
+    Nested functions inside classes are included in the class chunk rather
+    than split out — this preserves semantic coherence.
+
+    Falls back to _chunk_by_heading on syntax errors.
+    """
+    import ast as _ast
+
+    chunks = []
+    lines = content.splitlines()
+
+    try:
+        tree = _ast.parse(content)
+    except SyntaxError:
+        return _chunk_by_heading(content, source)
+
+    # Only visit top-level nodes (module body)
+    for node in tree.body:
+        if isinstance(node, (_ast.FunctionDef, _ast.AsyncFunctionDef)):
+            start = node.lineno - 1
+            end = node.end_lineno or (start + 30)
+            text = "\n".join(lines[start:end]).strip()
+            if text:
+                chunks.extend(
+                    _split_large(
+                        text,
+                        source,
+                        node.name,
+                        chunk_type="function",
+                    )
+                )
+
+        elif isinstance(node, _ast.ClassDef):
+            start = node.lineno - 1
+            end = node.end_lineno or (start + 50)
+            text = "\n".join(lines[start:end]).strip()
+            if text:
+                chunks.extend(
+                    _split_large(
+                        text,
+                        source,
+                        node.name,
+                        chunk_type="class",
+                    )
+                )
+
+    if not chunks:
+        # Module has no top-level classes/functions — embed whole file
+        return _chunk_whole(content, source)
+
+    return chunks
+
+
+def _chunk_by_heading(content: str, source: str) -> list[dict[str, Any]]:
+    """Split markdown/YAML by headings or top-level keys."""
+    chunks = []
+    current_heading = "intro"
+    current_text: list[str] = []
+
+    for line in content.splitlines():
+        if line.startswith("#"):
+            if current_text:
+                text = "\n".join(current_text).strip()
+                if text:
+                    chunks.extend(_split_large(text, source, current_heading))
+            current_heading = line.lstrip("#").strip()
+            current_text = [line]
+        else:
+            current_text.append(line)
+
+    if current_text:
+        text = "\n".join(current_text).strip()
+        if text:
+            chunks.extend(_split_large(text, source, current_heading))
+
+    return chunks
+
+
+def _chunk_by_function(content: str, source: str) -> list[dict[str, Any]]:
+    """Split Python test files by test function boundaries."""
+    import ast as _ast
+
+    chunks = []
+    lines = content.splitlines()
+    try:
+        tree = _ast.parse(content)
+    except SyntaxError:
+        return _chunk_by_heading(content, source)
+
+    for node in _ast.walk(tree):
+        if isinstance(node, (_ast.FunctionDef, _ast.AsyncFunctionDef)):
+            if node.name.startswith("test_"):
+                start = node.lineno - 1
+                end = node.end_lineno or (start + 20)
+                text = "\n".join(lines[start:end]).strip()
+                if text:
+                    chunks.append(
+                        {
+                            "text": text,
+                            "metadata": {
+                                "source": source,
+                                "section": node.name,
+                                "chunk_type": "test_function",
+                            },
+                        }
+                    )
+
+    if not chunks:
+        return _chunk_by_heading(content, source)
+    return chunks
+
+
+def _chunk_whole(content: str, source: str) -> list[dict[str, Any]]:
+    """Treat small files as a single chunk."""
+    return _split_large(content.strip(), source, "full")
+
+
+def _split_large(
+    text: str,
+    source: str,
+    section: str,
+    chunk_type: str = "section",
+) -> list[dict[str, Any]]:
+    """Split text that exceeds _CFG_CHK.max_chunk_chars into overlapping sub-chunks."""
+    if len(text) <= _CFG_CHK.max_chunk_chars:
+        return [
+            {
+                "text": text,
+                "metadata": {
+                    "source": source,
+                    "section": section,
+                    "chunk_type": chunk_type,
+                },
+            }
+        ]
+
+    chunks = []
+    step = _CFG_CHK.max_chunk_chars - 200  # 200-char overlap
+    for i, start in enumerate(range(0, len(text), step)):
+        chunk_text = text[start : start + _CFG_CHK.max_chunk_chars].strip()
+        if chunk_text:
+            chunks.append(
+                {
+                    "text": chunk_text,
+                    "metadata": {
+                        "source": source,
+                        "section": f"{section}_part{i}",
+                        "chunk_type": chunk_type,
+                    },
+                }
+            )
+    return chunks
+
+
+async def _embed_and_upsert(
+    chunks: list[dict[str, Any]],
+    collection: str,
+    file_path: str,
+    artifact_type: str,
+    qdrant: Any,
+    cognitive: Any,
+) -> int:
+    """Embed chunks and upsert to Qdrant. Returns number of chunks upserted."""
+    from qdrant_client import models as qm
+
+    from shared.universal import get_deterministic_id
+
+    await qdrant.ensure_collection(collection_name=collection)
+
+    points = []
+    for i, chunk in enumerate(chunks):
+        text = chunk["text"]
+        embedding = await cognitive.get_embedding_for_code(text)
+        if embedding is None:
+            continue
+
+        item_id = f"{file_path}::chunk::{i}"
+        point_id = get_deterministic_id(item_id)
+        payload = {
+            **chunk["metadata"],
+            "item_id": item_id,
+            "artifact_type": artifact_type,
+            "file_path": file_path,
+        }
+        points.append(
+            qm.PointStruct(
+                id=point_id,
+                vector=(
+                    embedding.tolist()
+                    if hasattr(embedding, "tolist")
+                    else list(embedding)
+                ),
+                payload=payload,
+            )
+        )
+
+    if points:
+        await qdrant.upsert_points(
+            collection_name=collection,
+            points=points,
+            wait=True,
+        )
+
+    return len(points)

will/workers/repo_embedding/repo_embedder_workers.py

@@ -0,0 +1,163 @@
+# src/will/workers/repo_embedding/repo_embedder_workers.py
+# repo_embedder_workers.py
+"""Contains the main worker class responsible for embedding repo artifacts."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from shared.logger import getLogger
+from shared.workers.base import Worker
+
+
+logger = getLogger(__name__)
+
+from .helpers import _chunk_file, _embed_and_upsert
+
+
+# ID: 1f09a2d8-0307-4172-b1b0-e3f14a918e00
+class RepoEmbedderWorker(Worker):
+    """
+    Sensing worker. Consumes unembedded repo_artifacts and upserts
+    semantic chunks into the appropriate Qdrant collections.
+    """
+
+    declaration_name = "repo_embedder"
+
+    def __init__(self, cognitive_service: Any = None) -> None:
+        from shared.infrastructure.bootstrap_registry import BootstrapRegistry
+
+        super().__init__()
+        self._cognitive_service = cognitive_service
+        self._repo_root: Path = BootstrapRegistry.get_repo_path()
+        schedule = self._declaration.get("mandate", {}).get("schedule", {})
+        self._max_interval: int = schedule.get("max_interval", 43200)
+        self._glide_off: int = schedule.get(
+            "glide_off", max(int(self._max_interval * 0.10), 10)
+        )
+        self._batch_size: int = schedule.get("batch_size", 10)
+
+    # ID: c4d5e6f7-a8b9-0c1d-3456-789012abcdef
+    async def run_loop(self) -> None:
+        """
+        Continuous self-scheduling loop. Runs one embedding pass per
+        max_interval seconds. Sanctuary calls this once on bootstrap.
+
+        In daemon context, self-initializes CognitiveService since no
+        CLI runner is present to inject it.
+
+        Never raises — exceptions are caught, logged, and posted to Blackboard.
+        """
+        from body.services.service_registry import service_registry
+
+        logger.info(
+            "RepoEmbedderWorker: starting loop (max_interval=%ds, glide_off=%ds)",
+            self._max_interval,
+            self._glide_off,
+        )
+
+        # Self-initialize CognitiveService if not injected (daemon context)
+        if self._cognitive_service is None:
+            self._cognitive_service = await service_registry.get_cognitive_service()
+
+        await self._register()
+
+        while True:
+            cycle_start = time.monotonic()
+            try:
+                await self.run()
+            except Exception as exc:
+                logger.error("RepoEmbedderWorker: cycle failed: %s", exc, exc_info=True)
+                try:
+                    await self._post_entry(
+                        entry_type="report",
+                        subject="repo_embedder.cycle_error",
+                        payload={"error": str(exc)},
+                        status="abandoned",
+                    )
+                except Exception:
+                    logger.exception("RepoEmbedderWorker: failed to post error report")
+
+            elapsed = time.monotonic() - cycle_start
+            await asyncio.sleep(max(self._max_interval - elapsed, 0))
+
+    # ID: b3c4d5e6-f7a8-9b0c-2345-678901abcdef
+    async def run(self) -> None:
+        """Embed a batch of unprocessed repo artifacts."""
+        await self.post_heartbeat()
+        from body.services.service_registry import service_registry
+
+        logger.info("RepoEmbedderWorker: starting embedding pass")
+
+        qdrant = await service_registry.get_qdrant_service()
+        cognitive = self._cognitive_service
+
+        stats = {"processed": 0, "chunks_total": 0, "errors": 0}
+
+        svc = await service_registry.get_artifact_service()
+        artifacts = await svc.fetch_unembedded_artifacts(self._batch_size)
+
+        if not artifacts:
+            logger.info("RepoEmbedderWorker: nothing to embed, all artifacts current")
+            return
+
+        for artifact in artifacts:
+            artifact_id = artifact["id"]
+            file_path = artifact["file_path"]
+            artifact_type = artifact["artifact_type"]
+            collection = artifact["qdrant_collection"]
+
+            full_path = self._repo_root / file_path
+            if not full_path.exists():
+                logger.warning("RepoEmbedderWorker: file missing: %s", file_path)
+                continue
+
+            try:
+                chunks = _chunk_file(full_path, artifact_type)
+                if not chunks:
+                    # Empty file — mark permanently skipped
+                    await svc.mark_artifact_empty(artifact_id)
+                    logger.info(
+                        "RepoEmbedderWorker: empty file skipped permanently: %s",
+                        file_path,
+                    )
+                    continue
+
+                chunk_count = await _embed_and_upsert(
+                    chunks=chunks,
+                    collection=collection,
+                    file_path=file_path,
+                    artifact_type=artifact_type,
+                    qdrant=qdrant,
+                    cognitive=cognitive,
+                )
+
+                await svc.update_artifact_chunk_count(artifact_id, chunk_count)
+
+                stats["processed"] += 1
+                stats["chunks_total"] += chunk_count
+                logger.info(
+                    "RepoEmbedderWorker: embedded %s → %s chunks → %s",
+                    file_path,
+                    chunk_count,
+                    collection,
+                )
+
+            except Exception as exc:
+                stats["errors"] += 1
+                logger.warning(
+                    "RepoEmbedderWorker: failed to embed %s: %s", file_path, exc
+                )
+
+        await self.post_report(
+            subject="repo.embed.complete",
+            payload={
+                **stats,
+                "completed_at": datetime.now(UTC).isoformat(),
+            },
+        )
+        logger.info("RepoEmbedderWorker: pass complete — %s", stats)