attocode-code-intel 0.1.0__py3-none-any.whl

attocode_intel/api/utils.py

@@ -0,0 +1,96 @@
+"""Shared utilities for the HTTP API."""
+
+from __future__ import annotations
+
+import os
+from typing import TYPE_CHECKING
+
+from fastapi import HTTPException
+from sqlalchemy import select
+
+if TYPE_CHECKING:
+    import uuid
+
+    from sqlalchemy.ext.asyncio import AsyncSession
+
+    from attocode_intel.api.auth.context import AuthContext
+    from attocode_intel.db.models import Repository
+
+# Language detection by extension
+LANG_MAP: dict[str, str] = {
+    ".py": "python", ".js": "javascript", ".ts": "typescript",
+    ".tsx": "typescriptreact", ".jsx": "javascriptreact",
+    ".rs": "rust", ".go": "go", ".java": "java", ".rb": "ruby",
+    ".c": "c", ".cpp": "cpp", ".h": "c", ".hpp": "cpp", ".metal": "cpp",
+    ".cs": "csharp", ".swift": "swift", ".kt": "kotlin",
+    ".sh": "shell", ".bash": "shell", ".zsh": "shell",
+    ".json": "json", ".yaml": "yaml", ".yml": "yaml",
+    ".toml": "toml", ".xml": "xml", ".html": "html", ".css": "css",
+    ".sql": "sql", ".md": "markdown", ".txt": "plaintext",
+    ".dockerfile": "dockerfile",
+}
+
+# Max file size for content retrieval (5 MB)
+MAX_FILE_SIZE = 5 * 1024 * 1024
+
+
+def detect_language(path: str) -> str:
+    """Detect language from file extension."""
+    _, ext = os.path.splitext(path)
+    ext = ext.lower()
+    if ext in LANG_MAP:
+        return LANG_MAP[ext]
+    basename = os.path.basename(path).lower()
+    if basename == "dockerfile":
+        return "dockerfile"
+    if basename == "makefile":
+        return "makefile"
+    return ""
+
+
+def is_binary(data: bytes) -> bool:
+    """Heuristic binary detection: check for null bytes in first 8KB."""
+    return b"\x00" in data[:8192]
+
+
+async def get_repo_by_id(
+    project_id: uuid.UUID,
+    session: AsyncSession,
+    auth: AuthContext | None = None,
+) -> Repository:
+    """Fetch a repository by ID, enforcing optional org isolation.
+
+    Returns 404 when the repo doesn't exist, or when ``auth.org_id`` is set
+    and does not match the repo's org.
+    """
+    from attocode_intel.db.models import Repository as _Repository
+
+    result = await session.execute(select(_Repository).where(_Repository.id == project_id))
+    repo = result.scalar_one_or_none()
+    if repo is None:
+        raise HTTPException(status_code=404, detail="Repository not found")
+    if auth and auth.org_id and repo.org_id != auth.org_id:
+        raise HTTPException(status_code=404, detail="Repository not found")
+    return repo
+
+
+async def get_repo_by_org(
+    org_id: uuid.UUID,
+    repo_id: uuid.UUID,
+    session: AsyncSession,
+) -> Repository:
+    """Fetch a repository scoped to a specific org.
+
+    Returns 404 when the (org_id, repo_id) tuple doesn't match a repo.
+    """
+    from attocode_intel.db.models import Repository as _Repository
+
+    result = await session.execute(
+        select(_Repository).where(
+            _Repository.id == repo_id, _Repository.org_id == org_id,
+        )
+    )
+    repo = result.scalar_one_or_none()
+    if repo is None:
+        raise HTTPException(status_code=404, detail="Repository not found")
+    return repo

attocode_intel/artifacts/__init__.py

@@ -0,0 +1,45 @@
+"""Shared code-intel reproducibility primitives.
+
+This package holds substrate shared by both the local stdio MCP side and the
+server HTTP side:
+
+- ``hashing``: canonical ``action_hash`` computation for derived artifacts.
+- ``provenance``: ``Provenance`` dataclass recording how an artifact was produced.
+- ``retrieval_pin``: ``RetrievalPin`` format + per-store manifest-hash helpers
+  used to make ranked-result tools deterministic.
+
+Everything here is pure Python with stdlib-only imports so it can be imported
+from either the local ``attocode.integrations.context.*`` modules or the
+server ``attocode_intel.storage.*`` modules without pulling in SQL,
+FastAPI, or FastMCP dependencies.
+"""
+
+from __future__ import annotations
+
+from .hashing import (
+    ARTIFACT_TYPES,
+    canonical_json,
+    compute_action_hash,
+    compute_config_digest,
+    sha256_file,
+)
+from .provenance import LEGACY_INDEXER_NAME, Provenance, legacy_provenance
+from .retrieval_pin import (
+    RetrievalPin,
+    compute_store_hash,
+    make_pin_id,
+)
+
+__all__ = [
+    "ARTIFACT_TYPES",
+    "LEGACY_INDEXER_NAME",
+    "Provenance",
+    "RetrievalPin",
+    "canonical_json",
+    "compute_action_hash",
+    "compute_config_digest",
+    "compute_store_hash",
+    "legacy_provenance",
+    "make_pin_id",
+    "sha256_file",
+]

attocode_intel/artifacts/hashing.py

@@ -0,0 +1,230 @@
+"""Canonical action-hash computation for derived code-intel artifacts.
+
+Given (artifact_type, indexer identity, input blob OID, config), produce a
+deterministic SHA-256 that is byte-identical across machines, processes, and
+the stdio-vs-HTTP split.
+
+Any tool that caches or ships a derived artifact should key it by this hash.
+The formula is frozen — changing it silently would invalidate every cached
+artifact across the fleet, so callers should bump the ``v1`` suffix and write
+a migration.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    import os
+    from collections.abc import Mapping
+
+# Version tag baked into every action_hash. Bump only in a coordinated
+# migration that also rebuilds the caches that depend on it.
+ACTION_HASH_SCHEMA = "atto.artifact.v1"
+
+# Allowed artifact types. Keeping this closed lets us catch typos at the
+# boundary and gives us a reason to fail loudly on unknown derived artifacts
+# rather than silently producing "works but not shared" hashes.
+ARTIFACT_TYPES: frozenset[str] = frozenset(
+    {
+        "symbols",
+        "references",
+        "embedding",
+        "ast",
+        "deps",
+        "repo_map_entry",
+        "stack_graph",
+        "conventions",
+        "learning_ref",
+        # Aliases for server-side content_type values:
+        "symbol_set_v1",
+        "dep_graph_v1",
+        "embedding_blob_v1",
+        "stack_graph_partial_v1",
+        "repo_map_slice_v1",
+        "conventions_v1",
+    }
+)
+
+# Whitelist of config keys that contribute to `config_digest` per artifact
+# type. Anything not in the whitelist is intentionally *excluded* from the
+# hash — it's meta (e.g. user preferences, display options) that doesn't
+# affect the derived output.
+CONFIG_SUBSET: Mapping[str, tuple[str, ...]] = {
+    "symbols": (
+        "language",
+        "parser_kind",
+        "parser_version",
+        "extract_signatures",
+        "extract_docstrings",
+    ),
+    "symbol_set_v1": (
+        "language",
+        "parser_kind",
+        "parser_version",
+        "extract_signatures",
+        "extract_docstrings",
+    ),
+    "references": (
+        "language",
+        "parser_kind",
+        "parser_version",
+    ),
+    "embedding": (
+        "model_name",
+        "model_version",
+        "dimension",
+        "normalize",
+        "chunker",
+        "chunk_size",
+        "chunk_overlap",
+        "preprocessor_version",
+    ),
+    "embedding_blob_v1": (
+        "model_name",
+        "model_version",
+        "dimension",
+        "normalize",
+        "chunker",
+        "chunk_size",
+        "chunk_overlap",
+        "preprocessor_version",
+    ),
+    "ast": (
+        "language",
+        "parser_kind",
+        "parser_version",
+    ),
+    "deps": (
+        "language",
+        "resolver_strategy",
+        "resolver_version",
+        "follow_dynamic_imports",
+    ),
+    "dep_graph_v1": (
+        "language",
+        "resolver_strategy",
+        "resolver_version",
+        "follow_dynamic_imports",
+    ),
+    "repo_map_entry": (
+        "indexer_version",
+        "slice_root",
+        "max_depth",
+        "ranker_version",
+        "max_tokens",
+    ),
+    "repo_map_slice_v1": (
+        "indexer_version",
+        "slice_root",
+        "max_depth",
+        "ranker_version",
+        "max_tokens",
+    ),
+    "stack_graph": (
+        "language",
+        "sg_grammar_version",
+        "rules_version",
+    ),
+    "stack_graph_partial_v1": (
+        "language",
+        "sg_grammar_version",
+        "rules_version",
+    ),
+    "conventions": (
+        "rule_pack_version",
+    ),
+    "conventions_v1": (
+        "rule_pack_version",
+    ),
+    "learning_ref": ("schema_version",),
+}
+
+
+def sha256_file(path: str | os.PathLike[str]) -> str:
+    """Compute SHA-256 hex digest of a file, reading in 64 KiB chunks."""
+    digest = hashlib.sha256()
+    with open(path, "rb") as fh:
+        for chunk in iter(lambda: fh.read(65536), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def canonical_json(obj: Any) -> bytes:
+    """Stable JSON encoding for hashing.
+
+    Sorts dict keys, disables non-ASCII escaping (so all bytes are stable
+    regardless of locale), no whitespace, no NaN/Infinity. The resulting
+    bytes are a canonical form — two processes on different machines
+    always produce the same output for equivalent Python objects.
+    """
+    return json.dumps(
+        obj,
+        sort_keys=True,
+        ensure_ascii=True,
+        separators=(",", ":"),
+        allow_nan=False,
+    ).encode("ascii")
+
+
+def compute_config_digest(artifact_type: str, config: Mapping[str, Any]) -> str:
+    """Hash a config dict down to a hex digest, using only the whitelisted keys.
+
+    Unknown keys are *silently ignored* (not an error) because callers legitimately
+    pass richer config objects than the hash cares about. Keys in the whitelist
+    that are missing from ``config`` default to ``None`` — so adding a new
+    config knob is a non-breaking change only when it starts as ``None``.
+    """
+    if artifact_type not in ARTIFACT_TYPES:
+        raise ValueError(
+            f"unknown artifact_type {artifact_type!r}; allowed: {sorted(ARTIFACT_TYPES)}"
+        )
+    allowed = CONFIG_SUBSET.get(artifact_type, ())
+    subset = {key: config.get(key) for key in allowed}
+    return hashlib.sha256(canonical_json(subset)).hexdigest()
+
+
+def compute_action_hash(
+    artifact_type: str,
+    *,
+    indexer_name: str,
+    indexer_version: str,
+    input_blob_oid: str,
+    config: Mapping[str, Any],
+    input_tree_oid: str | None = None,
+) -> str:
+    """Compute the canonical action hash for a derived artifact.
+
+    Formula::
+
+        sha256(
+            "atto.artifact.v1" || "\\n" ||
+            "type="     || artifact_type    || "\\n" ||
+            "indexer="  || indexer_name     || "\\n" ||
+            "iversion=" || indexer_version  || "\\n" ||
+            "input="    || input_blob_oid   || "\\n" ||
+            "tree="     || input_tree_oid   || "\\n" ||
+            "config="   || config_digest    || "\\n"
+        )
+
+    The delimiter is ``\\n`` and values are passed through unchanged — that's
+    fine because every field is either a short identifier or an already-hex
+    digest, so there is no delimiter ambiguity in practice.
+    """
+    if artifact_type not in ARTIFACT_TYPES:
+        raise ValueError(
+            f"unknown artifact_type {artifact_type!r}; allowed: {sorted(ARTIFACT_TYPES)}"
+        )
+    config_digest = compute_config_digest(artifact_type, config)
+    payload = (
+        f"{ACTION_HASH_SCHEMA}\n"
+        f"type={artifact_type}\n"
+        f"indexer={indexer_name}\n"
+        f"iversion={indexer_version}\n"
+        f"input={input_blob_oid}\n"
+        f"tree={input_tree_oid or ''}\n"
+        f"config={config_digest}\n"
+    )
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()

attocode_intel/artifacts/provenance.py

@@ -0,0 +1,190 @@
+"""Provenance records describe how a derived code-intel artifact was produced.
+
+Every symbol set, embedding, repo-map slice, etc. should carry a ``Provenance``
+record so we can answer:
+
+- Which model + version produced this embedding?
+- Which indexer + version extracted these symbols?
+- What blob (and optional tree) was the input?
+- When, and on what host, was it produced?
+
+This lets us detect staleness (indexer bumped, model swapped) and provides
+the audit trail for retrieval determinism.
+"""
+
+from __future__ import annotations
+
+import json
+import socket
+import time
+from dataclasses import asdict, dataclass
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+# Sentinel indexer_name used by the one-shot migration that backfills
+# provenance onto pre-existing rows. Searching for this string is an easy
+# way to find everything that still needs a real indexer pass.
+LEGACY_INDEXER_NAME = "legacy-pre-v2"
+
+PROVENANCE_SCHEMA_VERSION = 1
+
+
+@dataclass(frozen=True, slots=True)
+class Provenance:
+    """A record of how one derived artifact was produced.
+
+    Frozen + slotted so it's cheap to build many of these during an index
+    run and so accidental mutation is a TypeError.
+    """
+
+    schema_version: int
+    artifact_type: str
+    action_hash: str
+    input_blob_oid: str
+    indexer_name: str
+    indexer_version: str
+    producer_host: str
+    produced_at: float
+    # Embeddings-specific; None for other artifact types.
+    model_name: str | None = None
+    model_version: str | None = None
+    dimension: int | None = None
+    preprocessing_digest: str | None = None
+    # Optional contextual IDs — server-side sets these, local side leaves None.
+    producer_service: str = "attocode-local"
+    producer_user_id: str | None = None
+    producer_job_id: str | None = None
+    input_tree_oid: str | None = None
+    # Original config subset that went into the action_hash, as canonical JSON
+    # text — kept as a string so the dataclass stays hashable/frozen.
+    config_json: str = "{}"
+
+    # ------------------------------------------------------------------
+    # Constructors
+    # ------------------------------------------------------------------
+
+    @classmethod
+    def create(
+        cls,
+        *,
+        artifact_type: str,
+        action_hash: str,
+        input_blob_oid: str,
+        indexer_name: str,
+        indexer_version: str,
+        config: Mapping[str, Any] | None = None,
+        model_name: str | None = None,
+        model_version: str | None = None,
+        dimension: int | None = None,
+        preprocessing_digest: str | None = None,
+        producer_service: str = "attocode-local",
+        producer_user_id: str | None = None,
+        producer_job_id: str | None = None,
+        input_tree_oid: str | None = None,
+    ) -> Provenance:
+        """Build a fresh provenance record with ``produced_at`` set to now."""
+        config_json = json.dumps(
+            dict(config or {}),
+            sort_keys=True,
+            ensure_ascii=True,
+            separators=(",", ":"),
+        )
+        return cls(
+            schema_version=PROVENANCE_SCHEMA_VERSION,
+            artifact_type=artifact_type,
+            action_hash=action_hash,
+            input_blob_oid=input_blob_oid,
+            indexer_name=indexer_name,
+            indexer_version=indexer_version,
+            producer_host=socket.gethostname(),
+            produced_at=time.time(),
+            model_name=model_name,
+            model_version=model_version,
+            dimension=dimension,
+            preprocessing_digest=preprocessing_digest,
+            producer_service=producer_service,
+            producer_user_id=producer_user_id,
+            producer_job_id=producer_job_id,
+            input_tree_oid=input_tree_oid,
+            config_json=config_json,
+        )
+
+    # ------------------------------------------------------------------
+    # Serialization
+    # ------------------------------------------------------------------
+
+    def to_dict(self) -> dict[str, Any]:
+        """Return a plain dict suitable for JSON / JSONB storage."""
+        return asdict(self)
+
+    def to_json(self) -> str:
+        """Canonical JSON text — stable for hashing."""
+        return json.dumps(
+            self.to_dict(),
+            sort_keys=True,
+            ensure_ascii=True,
+            separators=(",", ":"),
+        )
+
+    @classmethod
+    def from_dict(cls, data: Mapping[str, Any]) -> Provenance:
+        """Rebuild a Provenance from its dict form (tolerant of missing fields)."""
+        # Copy known fields; unknown keys are silently dropped. Missing
+        # required fields get sensible defaults matching a "legacy" marker
+        # so we can still roundtrip rows that were inserted with a very old
+        # schema.
+        return cls(
+            schema_version=int(data.get("schema_version", PROVENANCE_SCHEMA_VERSION)),
+            artifact_type=str(data.get("artifact_type", "unknown")),
+            action_hash=str(data.get("action_hash", "")),
+            input_blob_oid=str(data.get("input_blob_oid", "")),
+            indexer_name=str(data.get("indexer_name", LEGACY_INDEXER_NAME)),
+            indexer_version=str(data.get("indexer_version", "0")),
+            producer_host=str(data.get("producer_host", "unknown")),
+            produced_at=float(data.get("produced_at", 0.0)),
+            model_name=data.get("model_name"),
+            model_version=data.get("model_version"),
+            dimension=data.get("dimension"),
+            preprocessing_digest=data.get("preprocessing_digest"),
+            producer_service=str(data.get("producer_service", "attocode-local")),
+            producer_user_id=data.get("producer_user_id"),
+            producer_job_id=data.get("producer_job_id"),
+            input_tree_oid=data.get("input_tree_oid"),
+            config_json=str(data.get("config_json", "{}")),
+        )
+
+
+def legacy_provenance(
+    artifact_type: str,
+    *,
+    input_blob_oid: str = "",
+    produced_at: float | None = None,
+    action_hash: str = "",
+) -> Provenance:
+    """Build a sentinel provenance record for the one-shot backfill.
+
+    Use this when you have existing rows in a store that pre-date provenance
+    tracking. Downstream tools can filter on ``indexer_name == LEGACY_INDEXER_NAME``
+    to find rows that still need a real re-index pass.
+    """
+    return Provenance(
+        schema_version=PROVENANCE_SCHEMA_VERSION,
+        artifact_type=artifact_type,
+        action_hash=action_hash,
+        input_blob_oid=input_blob_oid,
+        indexer_name=LEGACY_INDEXER_NAME,
+        indexer_version="0",
+        producer_host=socket.gethostname(),
+        produced_at=produced_at if produced_at is not None else time.time(),
+        model_name=None,
+        model_version=None,
+        dimension=None,
+        preprocessing_digest=None,
+        producer_service="attocode-backfill",
+        producer_user_id=None,
+        producer_job_id=None,
+        input_tree_oid=None,
+        config_json="{}",
+    )