devmem-agents 0.1.0__py3-none-any.whl

devmem/__init__.py

@@ -0,0 +1,5 @@
+"""devmem package."""
+
+from devmem.main import app
+
+__all__ = ["app"]

devmem/api.py

@@ -0,0 +1,257 @@
+"""HTTP API for devmem."""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, HTTPException, Query, Response
+
+from devmem.models import (
+    ApiResponse,
+    ArtifactUpsertRequest,
+    ContextPullRequest,
+    DecisionUpsertRequest,
+    FactUpsertRequest,
+    FeedbackRecordRequest,
+    HandoffCreateRequest,
+    HybridSearchRequest,
+    SessionCommitRequest,
+    SessionStartRequest,
+    TaskSimilarRequest,
+    TaskUpdateRequest,
+)
+from devmem.service import NamespaceError, service
+
+router = APIRouter(prefix="/v1")
+
+# Endpoints superseded by POST /v1/sessions/commit. They still work but clients
+# should migrate. Responses include a `Deprecation: true` header.
+_DEPRECATED_SUPERSEDED_BY = "/v1/sessions/commit"
+
+
+def _handle_namespace(exc: NamespaceError) -> None:
+    raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+
+def _mark_deprecated(response: Response) -> None:
+    response.headers["Deprecation"] = "true"
+    response.headers["Link"] = f'<{_DEPRECATED_SUPERSEDED_BY}>; rel="successor-version"'
+
+
+@router.get("/health", response_model=ApiResponse)
+def health() -> ApiResponse:
+    return ApiResponse(data=service.health())
+
+
+@router.post("/sessions/start", response_model=ApiResponse)
+def sessions_start(req: SessionStartRequest) -> ApiResponse:
+    try:
+        session = service.start_session(req)
+    except NamespaceError as exc:
+        _handle_namespace(exc)
+    return ApiResponse(data=session.model_dump())
+
+
+@router.post("/sessions/commit", response_model=ApiResponse)
+def sessions_commit(req: SessionCommitRequest) -> ApiResponse:
+    """Atomically finalize a session.
+
+    Replaces the 3-call ritual (`/artifacts/upsert` + `/decisions/upsert` +
+    `/tasks/update`) with one transactional write. Optional `client_commit_id`
+    gives idempotent retries — submitting the same id twice returns the
+    original result without re-inserting any rows.
+    """
+    try:
+        result = service.commit_session(
+            namespace=req.namespace,
+            session_id=req.session_id,
+            artifacts=[a.model_dump() for a in req.artifacts],
+            decisions=[d.model_dump() for d in req.decisions],
+            handoff=req.handoff.model_dump() if req.handoff else None,
+            task_update=req.task_update.model_dump() if req.task_update else None,
+            client_commit_id=req.client_commit_id,
+        )
+    except NamespaceError as exc:
+        _handle_namespace(exc)
+    return ApiResponse(data=result)
+
+
+@router.post("/context/pull", response_model=ApiResponse)
+def context_pull(req: ContextPullRequest) -> ApiResponse:
+    try:
+        payload = service.pull_context(
+            namespace=req.namespace,
+            session_id=req.session_id,
+            project=req.project,
+            repo=req.repo,
+            task=req.task,
+            top_k=req.top_k,
+        )
+    except NamespaceError as exc:
+        _handle_namespace(exc)
+    return ApiResponse(data=payload)
+
+
+@router.post("/search/hybrid", response_model=ApiResponse)
+def search_hybrid(req: HybridSearchRequest) -> ApiResponse:
+    try:
+        results = service.hybrid_search(
+            namespace=req.namespace,
+            q=req.q,
+            project=req.project,
+            repo=req.repo,
+            top_k=req.top_k,
+        )
+    except NamespaceError as exc:
+        _handle_namespace(exc)
+    return ApiResponse(data={"results": results})
+
+
+@router.post("/tasks/similar", response_model=ApiResponse)
+def tasks_similar(req: TaskSimilarRequest) -> ApiResponse:
+    try:
+        results = service.similar_tasks(
+            namespace=req.namespace,
+            q=req.q,
+            project=req.project,
+            repo=req.repo,
+            top_k=req.top_k,
+        )
+    except NamespaceError as exc:
+        _handle_namespace(exc)
+    return ApiResponse(data={"results": results})
+
+
+@router.post("/artifacts/upsert", response_model=ApiResponse)
+def artifacts_upsert(req: ArtifactUpsertRequest, response: Response) -> ApiResponse:
+    """Deprecated. Prefer POST /v1/sessions/commit for atomic multi-write."""
+    _mark_deprecated(response)
+    try:
+        result = service.upsert_artifact(req.namespace, req.model_dump())
+    except NamespaceError as exc:
+        _handle_namespace(exc)
+    return ApiResponse(data=result)
+
+
+@router.get("/artifacts", response_model=ApiResponse)
+def artifacts_list(
+    namespace: str = Query(...),
+    project: str | None = Query(default=None),
+    repo: str | None = Query(default=None),
+    session_id: str | None = Query(default=None),
+    artifact_type: str | None = Query(default=None),
+    since: str | None = Query(default=None, description="ISO-8601 timestamp; rows created_at >= since"),
+    limit: int = Query(default=50, ge=1, le=500),
+    offset: int = Query(default=0, ge=0),
+) -> ApiResponse:
+    try:
+        rows = service.list_artifacts(
+            namespace=namespace,
+            project=project,
+            repo=repo,
+            session_id=session_id,
+            artifact_type=artifact_type,
+            since=since,
+            limit=limit,
+            offset=offset,
+        )
+    except NamespaceError as exc:
+        _handle_namespace(exc)
+    return ApiResponse(data={"results": rows, "limit": limit, "offset": offset})
+
+
+@router.post("/facts/upsert", response_model=ApiResponse)
+def facts_upsert(req: FactUpsertRequest) -> ApiResponse:
+    try:
+        result = service.upsert_fact(req.namespace, req.model_dump())
+    except NamespaceError as exc:
+        _handle_namespace(exc)
+    return ApiResponse(data=result)
+
+
+@router.post("/decisions/upsert", response_model=ApiResponse)
+def decisions_upsert(req: DecisionUpsertRequest, response: Response) -> ApiResponse:
+    """Deprecated. Prefer POST /v1/sessions/commit for atomic multi-write."""
+    _mark_deprecated(response)
+    try:
+        result = service.upsert_decision(req.namespace, req.model_dump())
+    except NamespaceError as exc:
+        _handle_namespace(exc)
+    return ApiResponse(data=result)
+
+
+@router.get("/decisions", response_model=ApiResponse)
+def decisions_list(
+    namespace: str = Query(...),
+    project: str | None = Query(default=None),
+    repo: str | None = Query(default=None),
+    session_id: str | None = Query(default=None),
+    since: str | None = Query(default=None),
+    limit: int = Query(default=50, ge=1, le=500),
+    offset: int = Query(default=0, ge=0),
+) -> ApiResponse:
+    try:
+        rows = service.list_decisions(
+            namespace=namespace,
+            project=project,
+            repo=repo,
+            session_id=session_id,
+            since=since,
+            limit=limit,
+            offset=offset,
+        )
+    except NamespaceError as exc:
+        _handle_namespace(exc)
+    return ApiResponse(data={"results": rows, "limit": limit, "offset": offset})
+
+
+@router.post("/handoffs/create", response_model=ApiResponse)
+def handoffs_create(req: HandoffCreateRequest) -> ApiResponse:
+    try:
+        result = service.create_handoff(req.namespace, req.model_dump())
+    except NamespaceError as exc:
+        _handle_namespace(exc)
+    return ApiResponse(data=result)
+
+
+@router.get("/handoffs", response_model=ApiResponse)
+def handoffs_list(
+    namespace: str = Query(...),
+    project: str | None = Query(default=None),
+    repo: str | None = Query(default=None),
+    session_id: str | None = Query(default=None),
+    since: str | None = Query(default=None),
+    limit: int = Query(default=50, ge=1, le=500),
+    offset: int = Query(default=0, ge=0),
+) -> ApiResponse:
+    try:
+        rows = service.list_handoffs(
+            namespace=namespace,
+            project=project,
+            repo=repo,
+            session_id=session_id,
+            since=since,
+            limit=limit,
+            offset=offset,
+        )
+    except NamespaceError as exc:
+        _handle_namespace(exc)
+    return ApiResponse(data={"results": rows, "limit": limit, "offset": offset})
+
+
+@router.post("/tasks/update", response_model=ApiResponse)
+def tasks_update(req: TaskUpdateRequest, response: Response) -> ApiResponse:
+    """Deprecated. Prefer POST /v1/sessions/commit (pass task_update in body)."""
+    _mark_deprecated(response)
+    try:
+        result = service.update_task(req.namespace, req.model_dump())
+    except NamespaceError as exc:
+        _handle_namespace(exc)
+    return ApiResponse(data=result)
+
+
+@router.post("/feedback/record", response_model=ApiResponse)
+def feedback_record(req: FeedbackRecordRequest) -> ApiResponse:
+    try:
+        result = service.record_feedback(req.namespace, req.model_dump())
+    except NamespaceError as exc:
+        _handle_namespace(exc)
+    return ApiResponse(data=result)

devmem/config.py

@@ -0,0 +1,34 @@
+"""Configuration for devmem."""
+
+from __future__ import annotations
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    """Runtime settings loaded from environment."""
+
+    model_config = SettingsConfigDict(env_prefix="DEVMEM_", env_file=".env", extra="ignore")
+
+    namespace: str = Field(default="devlib_v1")
+    service_name: str = Field(default="devmem-gateway")
+    service_version: str = Field(default="0.1.0")
+
+    # Backing services (placeholders for real integration).
+    milvus_uri: str | None = Field(default=None)
+    neptune_endpoint: str | None = Field(default=None)
+    aurora_dsn: str | None = Field(default=None)
+
+    # Embedding model (sentence-transformers). Falls back to SHA-256
+    # pseudo-embeddings if the model cannot be loaded. `embedding_dim` must
+    # match the model's output dimensionality and the Milvus collection schema.
+    embedding_model: str = Field(default="sentence-transformers/all-MiniLM-L6-v2")
+    embedding_dim: int = Field(default=384)
+
+    # Record store DSN (SQLAlchemy-compatible, e.g. postgresql+psycopg://user:pw@host/db).
+    # When unset, devmem uses an in-memory store (fine for tests, not for production).
+    record_store_dsn: str | None = Field(default=None)
+
+
+settings = Settings()

devmem/embeddings.py

@@ -0,0 +1,119 @@
+"""Embedding provider for devmem.
+
+Uses `sentence-transformers/all-MiniLM-L6-v2` (384-dim) by default. The model
+is lazy-loaded on first use and cached for the process lifetime. If the
+`sentence-transformers` package is not installed (or model download fails),
+the provider falls back to a deterministic SHA-256 pseudo-embedding and logs
+a loud warning so operators know vector search quality is degraded.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import threading
+from typing import Iterable
+
+import numpy as np
+
+from devmem.config import settings
+
+logger = logging.getLogger(__name__)
+
+_MODEL_LOCK = threading.Lock()
+_MODEL: object | None = None
+_USING_FALLBACK: bool = False
+
+
+def embedding_dim() -> int:
+    """Configured embedding dimension (must match Milvus collection schema)."""
+    return int(settings.embedding_dim)
+
+
+def is_fallback_active() -> bool:
+    """True if real model failed to load and pseudo-embeddings are in use."""
+    return _USING_FALLBACK
+
+
+def _load_model() -> object | None:
+    """Try to load the sentence-transformers model; return None on failure."""
+    try:
+        from sentence_transformers import SentenceTransformer  # type: ignore
+    except Exception as exc:
+        logger.warning(
+            "sentence-transformers not available (%s); devmem will use SHA-256 "
+            "pseudo-embeddings. Install `sentence-transformers` to enable semantic search.",
+            exc,
+        )
+        return None
+    try:
+        model = SentenceTransformer(settings.embedding_model)
+        logger.info("Loaded embedding model=%s dim=%s", settings.embedding_model, embedding_dim())
+        return model
+    except Exception as exc:
+        logger.error(
+            "Failed to load embedding model %s (%s); falling back to SHA-256 pseudo-embeddings.",
+            settings.embedding_model,
+            exc,
+            exc_info=True,
+        )
+        return None
+
+
+def _get_model() -> object | None:
+    global _MODEL, _USING_FALLBACK
+    if _MODEL is not None or _USING_FALLBACK:
+        return _MODEL
+    with _MODEL_LOCK:
+        if _MODEL is None and not _USING_FALLBACK:
+            _MODEL = _load_model()
+            if _MODEL is None:
+                _USING_FALLBACK = True
+    return _MODEL
+
+
+def _pseudo_embed(text: str) -> list[float]:
+    """Deterministic hash-based fallback vector (not semantic)."""
+    dim = embedding_dim()
+    digest = hashlib.sha256((text or "").encode("utf-8", errors="ignore")).digest()
+    repeats = (dim // len(digest)) + 1
+    raw = (digest * repeats)[:dim]
+    vec = np.frombuffer(raw, dtype=np.uint8).astype(np.float32)
+    vec = (vec / 255.0) - 0.5
+    norm = float(np.linalg.norm(vec))
+    if norm > 0:
+        vec /= norm
+    return vec.tolist()
+
+
+def embed_one(text: str) -> list[float]:
+    """Return a single embedding vector."""
+    model = _get_model()
+    if model is None:
+        return _pseudo_embed(text)
+    vec = model.encode([text or ""], normalize_embeddings=True)[0]
+    return [float(x) for x in vec]
+
+
+def embed_many(texts: Iterable[str]) -> list[list[float]]:
+    """Return a batch of embedding vectors."""
+    items = [t or "" for t in texts]
+    if not items:
+        return []
+    model = _get_model()
+    if model is None:
+        return [_pseudo_embed(t) for t in items]
+    vectors = model.encode(items, normalize_embeddings=True, batch_size=32)
+    return [[float(x) for x in v] for v in vectors]
+
+
+def cosine_similarity(a: list[float], b: list[float]) -> float:
+    """Cosine similarity between two vectors; assumes they may not be unit-normed."""
+    if not a or not b:
+        return 0.0
+    va = np.asarray(a, dtype=np.float32)
+    vb = np.asarray(b, dtype=np.float32)
+    denom = float(np.linalg.norm(va)) * float(np.linalg.norm(vb))
+    if denom == 0.0:
+        return 0.0
+    return float(np.dot(va, vb) / denom)

devmem/ingest.py

@@ -0,0 +1,184 @@
+"""Repository scanning and memory ingestion pipeline for devmem."""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import os
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable
+
+from devmem.live_backend import LiveBackendConfig
+from devmem.storage.milvus_store import MilvusStore
+from devmem.storage.neptune_store import NeptuneStore
+
+logger = logging.getLogger(__name__)
+
+_ALLOWED_EXTENSIONS = {
+    ".py", ".md", ".toml", ".json", ".yaml", ".yml", ".js", ".ts", ".tsx", ".jsx",
+    ".css", ".html", ".sql", ".sh", ".txt", ".ini", ".cfg", ".conf", ".rst", ".csv", ".svg",
+}
+_ALLOWED_FILENAMES = {
+    "Makefile",
+    "Dockerfile",
+    "Jenkinsfile",
+    "Procfile",
+}
+
+_SKIP_PREFIXES = {
+    ".git/",
+    "node_modules/",
+    ".venv/",
+    "venv/",
+    "data/",
+    "logs/",
+    "__pycache__/",
+    ".pytest_cache/",
+    "react-app/node_modules/",
+}
+
+
+@dataclass
+class IngestResult:
+    files_seen: int = 0
+    files_indexed: int = 0
+    files_skipped: int = 0
+    chunks_written: int = 0
+    kg_file_nodes: int = 0
+
+
+def _is_binary(blob: bytes) -> bool:
+    return b"\x00" in blob
+
+
+def _sha256(blob: bytes) -> str:
+    return hashlib.sha256(blob).hexdigest()
+
+
+def _chunk_text(text: str, *, chunk_size: int = 1800, overlap: int = 200) -> list[str]:
+    if not text.strip():
+        return []
+    if chunk_size <= overlap:
+        raise ValueError("chunk_size must be greater than overlap")
+
+    chunks: list[str] = []
+    start = 0
+    n = len(text)
+    while start < n:
+        end = min(start + chunk_size, n)
+        chunks.append(text[start:end])
+        if end >= n:
+            break
+        start = max(0, end - overlap)
+    return chunks
+
+
+def _tracked_files(repo_path: Path) -> list[Path]:
+    try:
+        proc = subprocess.run(
+            ["git", "-C", str(repo_path), "ls-files", "-z"],
+            check=True,
+            capture_output=True,
+        )
+        raw = proc.stdout.decode("utf-8", errors="ignore")
+        files = [repo_path / p for p in raw.split("\x00") if p]
+        return [p for p in files if p.is_file()]
+    except Exception:
+        logger.warning("git ls-files failed; falling back to filesystem walk", exc_info=True)
+        files: list[Path] = []
+        for root, _dirs, names in os.walk(repo_path):
+            for name in names:
+                files.append(Path(root) / name)
+        return files
+
+
+def _should_index(rel_path: str, suffix: str, size_bytes: int) -> bool:
+    if size_bytes > 512 * 1024:
+        return False
+    path_obj = Path(rel_path)
+    if suffix.lower() not in _ALLOWED_EXTENSIONS and path_obj.name not in _ALLOWED_FILENAMES:
+        return False
+    norm = rel_path.replace("\\", "/")
+    for prefix in _SKIP_PREFIXES:
+        if norm.startswith(prefix):
+            return False
+    return True
+
+
+def ingest_repository(
+    *,
+    repo_path: Path,
+    project_id: str,
+    project_name: str,
+    repo_id: str,
+    namespace: str,
+    cfg: LiveBackendConfig,
+) -> IngestResult:
+    """Scan repository and write memory records to Milvus and Neptune."""
+    result = IngestResult()
+
+    milvus = MilvusStore(cfg, namespace=namespace, project_id=project_id, repo_id=repo_id)
+    neptune = NeptuneStore(cfg, namespace=namespace)
+
+    try:
+        milvus.connect()
+        neptune.connect()
+        neptune.health_check()
+
+        neptune.upsert_project(
+            project_id=project_id,
+            name=project_name,
+            repo_path=str(repo_path),
+            repo_id=repo_id,
+        )
+        milvus.clear_project_data()
+
+        tracked = _tracked_files(repo_path)
+        result.files_seen = len(tracked)
+
+        for file_path in tracked:
+            rel_path = file_path.relative_to(repo_path).as_posix()
+            suffix = file_path.suffix.lower()
+            try:
+                blob = file_path.read_bytes()
+            except Exception:
+                logger.debug("Skipping unreadable file path=%s", rel_path, exc_info=True)
+                result.files_skipped += 1
+                continue
+
+            if _is_binary(blob):
+                result.files_skipped += 1
+                continue
+
+            if not _should_index(rel_path, suffix, len(blob)):
+                result.files_skipped += 1
+                continue
+
+            text = blob.decode("utf-8", errors="ignore")
+            sha = _sha256(blob)
+            chunks = _chunk_text(text)
+            if not chunks:
+                result.files_skipped += 1
+                continue
+
+            neptune.upsert_file(
+                project_id=project_id,
+                repo_id=repo_id,
+                path=rel_path,
+                ext=suffix,
+                sha=sha,
+                size_bytes=len(blob),
+            )
+            result.kg_file_nodes += 1
+
+            inserted = milvus.replace_file_chunks(path=rel_path, sha=sha, chunks=chunks)
+            result.chunks_written += inserted
+            result.files_indexed += 1
+
+        milvus.flush()
+        return result
+    finally:
+        neptune.close()
+        milvus.close()