shellbrain 0.1.30__tar.gz → 0.1.32__tar.gz

{shellbrain-0.1.30 → shellbrain-0.1.32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shellbrain
-Version: 0.1.30
+Version: 0.1.32
 Summary: Repo-scoped Shellbrain CLI with explicit evidence-backed writes.
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{shellbrain-0.1.30 → shellbrain-0.1.32}/app/core/entities/inner_agents.py

@@ -14,6 +14,9 @@ class _StrictModel(BaseModel):
 InnerAgentName = Literal["build_context", "build_knowledge", "teach"]
 InnerAgentProviderName = str
 InnerAgentReasoningLevel = Literal["none", "minimal", "low", "medium", "high", "xhigh"]
+BuildContextStrategy = Literal[
+    "deterministic_synthesis", "deterministic_only", "autonomous"
+]
 TokenCaptureQuality = Literal["exact", "estimated"]
 InnerAgentRunStatus = Literal[
     "ok",
@@ -28,12 +31,12 @@ InnerAgentRunStatus = Literal[
 class BuildContextSettings(_StrictModel):
     """Typed model/runtime settings for the build_context recall agent."""
 
+    strategy: BuildContextStrategy = "deterministic_synthesis"
     provider: InnerAgentProviderName = Field(min_length=1)
     model: str = Field(min_length=1)
     reasoning: InnerAgentReasoningLevel
     timeout_seconds: int = Field(ge=1, le=600)
     max_private_reads: int = Field(default=0, ge=0, le=10)
-    max_candidate_tokens: int = Field(ge=1, le=200_000)
     max_brief_tokens: int | None = Field(default=None, ge=1, le=100_000)
 
 

shellbrain-0.1.32/app/core/policies/concepts/search_text.py

@@ -0,0 +1,124 @@
+"""Pure concept aggregate text rendering for retrieval embeddings."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from app.core.entities.concepts import (
+    Anchor,
+    Concept,
+    ConceptClaim,
+    ConceptGrounding,
+    ConceptLifecycleStatus,
+    ConceptMemoryLink,
+    ConceptRelation,
+)
+
+
+def build_concept_embedding_text(bundle: dict[str, Any]) -> str:
+    """Render one concept bundle into deterministic aggregate retrieval text."""
+
+    concept: Concept = bundle["concept"]
+    parts: list[str] = [
+        concept.slug,
+        concept.name,
+        concept.kind.value,
+    ]
+    if concept.scope_note:
+        parts.append(concept.scope_note)
+    parts.extend(
+        alias.alias
+        for alias in sorted(
+            bundle.get("aliases", ()), key=lambda item: item.normalized_alias
+        )
+    )
+    parts.extend(_claim_parts(bundle.get("claims", ())))
+    parts.extend(_relation_parts(bundle.get("relations", ()), concept_id=concept.id))
+    parts.extend(_grounding_parts(bundle.get("groundings", ()), bundle.get("anchors", ())))
+    parts.extend(_memory_link_parts(bundle.get("memory_links", ())))
+    return " ".join(_clean_parts(parts))
+
+
+def _claim_parts(claims: list[ConceptClaim]) -> list[str]:
+    parts: list[str] = []
+    for claim in sorted(claims, key=lambda item: (item.claim_type.value, item.text)):
+        if claim.lifecycle.status is not ConceptLifecycleStatus.ACTIVE:
+            continue
+        parts.extend((claim.claim_type.value, claim.text))
+    return parts
+
+
+def _relation_parts(
+    relations: list[ConceptRelation], *, concept_id: str
+) -> list[str]:
+    parts: list[str] = []
+    for relation in sorted(
+        relations,
+        key=lambda item: (
+            item.predicate.value,
+            item.subject_concept_id,
+            item.object_concept_id,
+        ),
+    ):
+        if relation.lifecycle.status is not ConceptLifecycleStatus.ACTIVE:
+            continue
+        neighbor_id = (
+            relation.object_concept_id
+            if relation.subject_concept_id == concept_id
+            else relation.subject_concept_id
+        )
+        parts.extend((relation.predicate.value, neighbor_id))
+    return parts
+
+
+def _grounding_parts(
+    groundings: list[ConceptGrounding], anchors: list[Anchor]
+) -> list[str]:
+    anchor_by_id = {anchor.id: anchor for anchor in anchors}
+    parts: list[str] = []
+    for grounding in sorted(
+        groundings, key=lambda item: (item.role.value, item.anchor_id)
+    ):
+        if grounding.lifecycle.status is not ConceptLifecycleStatus.ACTIVE:
+            continue
+        anchor = anchor_by_id.get(grounding.anchor_id)
+        parts.append(grounding.role.value)
+        if anchor is not None:
+            parts.append(anchor.kind.value)
+            parts.extend(_locator_scalars(anchor.locator_json))
+    return parts
+
+
+def _memory_link_parts(memory_links: list[ConceptMemoryLink]) -> list[str]:
+    parts: list[str] = []
+    for link in sorted(memory_links, key=lambda item: (item.role.value, item.memory_id)):
+        if link.lifecycle.status is not ConceptLifecycleStatus.ACTIVE:
+            continue
+        parts.extend((link.role.value, link.memory_id))
+    return parts
+
+
+def _locator_scalars(value: object) -> tuple[str, ...]:
+    scalars: list[str] = []
+
+    def _walk(item: object) -> None:
+        if isinstance(item, dict):
+            for key in sorted(item):
+                _walk(item[key])
+            return
+        if isinstance(item, (list, tuple)):
+            for nested in item:
+                _walk(nested)
+            return
+        if item is None or isinstance(item, bool):
+            return
+        text = str(item).strip()
+        if text:
+            scalars.append(text)
+
+    _walk(value)
+    return tuple(scalars)
+
+
+def _clean_parts(parts: list[str]) -> tuple[str, ...]:
+    return tuple(part.strip() for part in parts if part and part.strip())

{shellbrain-0.1.30 → shellbrain-0.1.32}/app/core/policies/retrieval/bm25.py

@@ -20,7 +20,7 @@ _COVERAGE_THRESHOLD_BY_MODE = {
 class BM25Document:
     """Normalized document representation used for BM25 scoring."""
 
-    memory_id: str
+    document_id: str
     terms: tuple[str, ...]
 
 
@@ -28,7 +28,7 @@ class BM25Document:
 class BM25ScoredDocument:
     """Scored lexical candidate with query-coverage metadata."""
 
-    memory_id: str
+    document_id: str
     score: float
     coverage: float
 
@@ -47,7 +47,7 @@ def score_documents(
         return []
 
     term_frequencies = {
-        document.memory_id: Counter(document.terms) for document in documents
+        document.document_id: Counter(document.terms) for document in documents
     }
     corpus_size = len(documents)
     average_length = sum(len(document.terms) for document in documents) / corpus_size
@@ -59,7 +59,7 @@ def score_documents(
             sum(
                 1
                 for document in documents
-                if term in term_frequencies[document.memory_id]
+                if term in term_frequencies[document.document_id]
             ),
             corpus_size,
         )
@@ -71,7 +71,7 @@ def score_documents(
 
     scored: list[BM25ScoredDocument] = []
     for document in documents:
-        frequencies = term_frequencies[document.memory_id]
+        frequencies = term_frequencies[document.document_id]
         matched_terms = tuple(
             term for term in normalized_query_terms if term in frequencies
         )
@@ -93,30 +93,36 @@ def score_documents(
         if score > 0.0:
             scored.append(
                 BM25ScoredDocument(
-                    memory_id=document.memory_id,
+                    document_id=document.document_id,
                     score=score,
                     coverage=matched_query_weight / total_query_weight,
                 )
             )
 
-    return sorted(scored, key=lambda item: (-item.score, item.memory_id))
+    return sorted(scored, key=lambda item: (-item.score, item.document_id))
 
 
 def admit_scored_documents(
     scored_documents: Sequence[BM25ScoredDocument],
     *,
     mode: Literal["ambient", "targeted"],
+    output_id_key: Literal["memory_id", "concept_id"] = "memory_id",
+    coverage_threshold: float | None = None,
 ) -> list[dict[str, object]]:
     """Apply the mode-aware weighted query-coverage gate to scored lexical candidates."""
 
-    threshold = _COVERAGE_THRESHOLD_BY_MODE[mode]
+    threshold = (
+        float(coverage_threshold)
+        if coverage_threshold is not None
+        else _COVERAGE_THRESHOLD_BY_MODE[mode]
+    )
     admitted = [
-        {"memory_id": document.memory_id, "score": document.score}
+        {output_id_key: document.document_id, "score": document.score}
         for document in scored_documents
         if document.coverage >= threshold
     ]
     return sorted(
-        admitted, key=lambda item: (-float(item["score"]), str(item["memory_id"]))
+        admitted, key=lambda item: (-float(item["score"]), str(item[output_id_key]))
     )
 
 

{shellbrain-0.1.30 → shellbrain-0.1.32}/app/core/policies/retrieval/fusion_rrf.py

@@ -10,6 +10,7 @@ def fuse_with_rrf(
     keyword: list[dict[str, Any]],
     *,
     retrieval_defaults: Mapping[str, float] | None = None,
+    id_key: str = "memory_id",
 ) -> list[dict[str, Any]]:
     """This function merges lane candidates using reciprocal-rank fusion."""
 
@@ -25,11 +26,11 @@ def fuse_with_rrf(
 
     for lane_name, candidates in (("semantic", semantic), ("keyword", keyword)):
         for rank, candidate in enumerate(candidates, start=1):
-            memory_id = candidate["memory_id"]
+            item_id = candidate[id_key]
             entry = fused.setdefault(
-                memory_id,
+                item_id,
                 {
-                    "memory_id": memory_id,
+                    id_key: item_id,
                     "rrf_score": 0.0,
                     "rank_semantic": None,
                     "rank_keyword": None,
@@ -40,5 +41,5 @@ def fuse_with_rrf(
 
     return sorted(
         fused.values(),
-        key=lambda item: (-float(item["rrf_score"]), str(item["memory_id"])),
+        key=lambda item: (-float(item["rrf_score"]), str(item[id_key])),
     )

{shellbrain-0.1.30 → shellbrain-0.1.32}/app/core/ports/db/concept_repositories.py

@@ -85,5 +85,13 @@ class IConceptsRepo(ABC):
         """This method returns concept-link matches for displayed memory ids."""
 
     @abstractmethod
-    def list_concept_search_rows(self, *, repo_id: str) -> Sequence[dict[str, Any]]:
-        """This method returns active concept text rows for query matching."""
+    def upsert_embedding(
+        self,
+        *,
+        concept_id: str,
+        repo_id: str,
+        model: str,
+        vector: Sequence[float],
+        source_hash: str,
+    ) -> None:
+        """This method inserts or updates one aggregate concept embedding."""

{shellbrain-0.1.30 → shellbrain-0.1.32}/app/core/ports/db/retrieval_repositories.py

@@ -49,6 +49,35 @@ class IKeywordRetrievalRepo(ABC):
         """This method returns visible text rows for lexical ranking."""
 
 
+class IConceptSemanticRetrievalRepo(ABC):
+    """This interface defines semantic-lane retrieval against concept embeddings."""
+
+    @abstractmethod
+    def query_concepts_semantic(
+        self,
+        *,
+        repo_id: str,
+        query_vector: Sequence[float],
+        limit: int,
+        query_model: str | None = None,
+    ) -> Sequence[dict[str, Any]]:
+        """This method returns concept semantic retrieval candidates with scores."""
+
+
+class IConceptKeywordRetrievalRepo(ABC):
+    """This interface defines concept keyword-lane corpus access."""
+
+    @abstractmethod
+    def list_concept_keyword_corpus(
+        self,
+        *,
+        repo_id: str,
+        query_terms: Sequence[str] | None = None,
+        candidate_limit: int | None = None,
+    ) -> Sequence[dict[str, Any]]:
+        """This method returns active concept text rows for lexical ranking."""
+
+
 class IReadPolicyRepo(ABC):
     """This interface defines read-path visibility and explicit expansion queries."""
 

{shellbrain-0.1.30 → shellbrain-0.1.32}/app/core/ports/db/unit_of_work.py

@@ -17,6 +17,8 @@ from app.core.ports.db.memory_repositories import (
 from app.core.ports.db.problem_runs import IProblemRunsRepo
 from app.core.ports.embeddings.retrieval import IVectorSearch
 from app.core.ports.db.retrieval_repositories import (
+    IConceptKeywordRetrievalRepo,
+    IConceptSemanticRetrievalRepo,
     IKeywordRetrievalRepo,
     IReadPolicyRepo,
     ISemanticRetrievalRepo,
@@ -35,6 +37,8 @@ class IUnitOfWork(ABC):
     evidence: IEvidenceRepo
     semantic_retrieval: ISemanticRetrievalRepo
     keyword_retrieval: IKeywordRetrievalRepo
+    concept_semantic_retrieval: IConceptSemanticRetrievalRepo
+    concept_keyword_retrieval: IConceptKeywordRetrievalRepo
     read_policy: IReadPolicyRepo
     guidance: IPendingUtilityCandidatesRepo
     knowledge_build_runs: IKnowledgeBuildRunsRepo

{shellbrain-0.1.30 → shellbrain-0.1.32}/app/core/ports/host_apps/inner_agents.py

@@ -24,7 +24,7 @@ class _StrictModel(BaseModel):
 
 
 class InnerAgentRunRequest(_StrictModel):
-    """One autonomous read-only inner-agent synthesis request."""
+    """One build_context inner-agent request."""
 
     agent_name: InnerAgentName
     provider: InnerAgentProviderName = Field(min_length=1)
@@ -32,11 +32,12 @@ class InnerAgentRunRequest(_StrictModel):
     reasoning: InnerAgentReasoningLevel
     timeout_seconds: int = Field(ge=1, le=600)
     max_private_reads: int = Field(default=0, ge=0, le=10)
-    max_candidate_tokens: int = Field(ge=1, le=200_000)
     max_brief_tokens: int | None = Field(default=None, ge=1, le=100_000)
     query: str = Field(min_length=1)
     current_problem: dict[str, str]
     repo_root: str | None = None
+    synthesis_only: bool = False
+    deterministic_pack: dict[str, Any] | None = None
 
 
 class InnerAgentRunResult(_StrictModel):

{shellbrain-0.1.30 → shellbrain-0.1.32}/app/core/use_cases/concepts/add/execute.py

@@ -6,8 +6,10 @@ from typing import Any
 
 from app.core.errors import DomainValidationError, ErrorCode, ErrorDetail
 from app.core.entities.concepts import Concept, ConceptKind, ConceptStatus
+from app.core.ports.embeddings.provider import IEmbeddingProvider
 from app.core.ports.system.idgen import IIdGenerator
 from app.core.ports.db.unit_of_work import IUnitOfWork
+from app.core.use_cases.concepts.embeddings import upsert_concept_embeddings
 from app.core.use_cases.concepts.add.request import ConceptAddRequest
 from app.core.use_cases.concepts.add.result import ConceptAddResult
 from app.core.use_cases.concepts.reference_checks import require_missing_concept
@@ -18,6 +20,8 @@ def add_concepts(
     uow: IUnitOfWork,
     *,
     id_generator: IIdGenerator,
+    embedding_provider: IEmbeddingProvider | None = None,
+    embedding_model: str | None = None,
 ) -> ConceptAddResult:
     """Create concept containers, failing when any target concept already exists."""
 
@@ -39,6 +43,7 @@ def add_concepts(
         seen_slugs.add(normalized_slug)
 
     results: list[dict[str, Any]] = []
+    concept_ids: list[str] = []
     for action, normalized_slug in zip(request.actions, normalized_slugs, strict=True):
         concept = uow.concepts.add_concept(
             Concept(
@@ -52,8 +57,16 @@ def add_concepts(
             ),
             aliases=action.aliases,
         )
+        concept_ids.append(concept.id)
         results.append(
             {"type": action.type, "concept_id": concept.id, "slug": concept.slug}
         )
+    upsert_concept_embeddings(
+        repo_id=request.repo_id,
+        concept_ids=concept_ids,
+        uow=uow,
+        embedding_provider=embedding_provider,
+        embedding_model=embedding_model,
+    )
 
     return ConceptAddResult(added_count=len(results), results=results)

shellbrain-0.1.32/app/core/use_cases/concepts/embeddings.py

@@ -0,0 +1,40 @@
+"""Concept aggregate embedding maintenance helpers."""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Sequence
+
+from app.core.ports.db.unit_of_work import IUnitOfWork
+from app.core.ports.embeddings.provider import IEmbeddingProvider
+from app.core.policies.concepts.search_text import build_concept_embedding_text
+
+
+def upsert_concept_embeddings(
+    *,
+    repo_id: str,
+    concept_ids: Sequence[str],
+    uow: IUnitOfWork,
+    embedding_provider: IEmbeddingProvider | None,
+    embedding_model: str | None,
+) -> int:
+    """Refresh aggregate concept embeddings for touched concepts when enabled."""
+
+    if embedding_provider is None or embedding_model is None:
+        return 0
+    updated = 0
+    for concept_id in dict.fromkeys(str(value) for value in concept_ids):
+        bundle = uow.concepts.get_concept_bundle(repo_id=repo_id, concept_ref=concept_id)
+        if bundle is None:
+            continue
+        text = build_concept_embedding_text(bundle)
+        source_hash = hashlib.sha256(text.encode("utf-8")).hexdigest()
+        uow.concepts.upsert_embedding(
+            concept_id=concept_id,
+            repo_id=repo_id,
+            model=embedding_model,
+            vector=embedding_provider.embed(text),
+            source_hash=source_hash,
+        )
+        updated += 1
+    return updated

{shellbrain-0.1.30 → shellbrain-0.1.32}/app/core/use_cases/concepts/update/execute.py

@@ -38,8 +38,10 @@ from app.core.entities.concepts import (
     ConceptSourceKind,
     ConceptStatus,
 )
+from app.core.ports.embeddings.provider import IEmbeddingProvider
 from app.core.ports.system.idgen import IIdGenerator
 from app.core.ports.db.unit_of_work import IUnitOfWork
+from app.core.use_cases.concepts.embeddings import upsert_concept_embeddings
 from app.core.policies.concepts.relation_rules import validate_relation_shape
 from app.core.use_cases.concepts.containment_checks import validate_contains_relation
 from app.core.use_cases.concepts.reference_checks import (
@@ -57,23 +59,43 @@ def update_concepts(
     uow: IUnitOfWork,
     *,
     id_generator: IIdGenerator,
+    embedding_provider: IEmbeddingProvider | None = None,
+    embedding_model: str | None = None,
 ) -> ConceptUpdateResult:
     """Update existing concepts and truth-bearing graph records."""
 
     results: list[dict[str, Any]] = []
+    touched_concept_ids: set[str] = set()
     for action in request.actions:
         try:
             if isinstance(action, UpdateConceptAction):
-                results.append(_update_concept(request.repo_id, action, uow))
+                results.append(
+                    _update_concept(
+                        request.repo_id,
+                        action,
+                        uow,
+                        touched_concept_ids=touched_concept_ids,
+                    )
+                )
             elif isinstance(action, AddRelationAction):
                 results.append(
                     _add_relation(
-                        request.repo_id, action, uow, id_generator=id_generator
+                        request.repo_id,
+                        action,
+                        uow,
+                        id_generator=id_generator,
+                        touched_concept_ids=touched_concept_ids,
                     )
                 )
             elif isinstance(action, AddClaimAction):
                 results.append(
-                    _add_claim(request.repo_id, action, uow, id_generator=id_generator)
+                    _add_claim(
+                        request.repo_id,
+                        action,
+                        uow,
+                        id_generator=id_generator,
+                        touched_concept_ids=touched_concept_ids,
+                    )
                 )
             elif isinstance(action, EnsureAnchorAction):
                 results.append(
@@ -84,13 +106,21 @@ def update_concepts(
             elif isinstance(action, AddGroundingAction):
                 results.append(
                     _add_grounding(
-                        request.repo_id, action, uow, id_generator=id_generator
+                        request.repo_id,
+                        action,
+                        uow,
+                        id_generator=id_generator,
+                        touched_concept_ids=touched_concept_ids,
                     )
                 )
             elif isinstance(action, LinkMemoryAction):
                 results.append(
                     _link_memory(
-                        request.repo_id, action, uow, id_generator=id_generator
+                        request.repo_id,
+                        action,
+                        uow,
+                        id_generator=id_generator,
+                        touched_concept_ids=touched_concept_ids,
                     )
                 )
             else:  # pragma: no cover - discriminated contract should make this impossible.
@@ -109,13 +139,25 @@ def update_concepts(
             raise DomainValidationError(
                 [ErrorDetail(code=ErrorCode.SEMANTIC_ERROR, message=str(exc))]
             ) from exc
+    upsert_concept_embeddings(
+        repo_id=request.repo_id,
+        concept_ids=tuple(sorted(touched_concept_ids)),
+        uow=uow,
+        embedding_provider=embedding_provider,
+        embedding_model=embedding_model,
+    )
     return ConceptUpdateResult(updated_count=len(results), results=results)
 
 
 def _update_concept(
-    repo_id: str, action: UpdateConceptAction, uow: IUnitOfWork
+    repo_id: str,
+    action: UpdateConceptAction,
+    uow: IUnitOfWork,
+    *,
+    touched_concept_ids: set[str],
 ) -> dict[str, Any]:
     existing = require_concept(repo_id, action.concept, uow)
+    touched_concept_ids.add(existing.id)
     concept = Concept(
         id=existing.id,
         repo_id=existing.repo_id,
@@ -141,9 +183,11 @@ def _add_relation(
     uow: IUnitOfWork,
     *,
     id_generator: IIdGenerator,
+    touched_concept_ids: set[str],
 ) -> dict[str, Any]:
     subject = require_concept(repo_id, action.subject, uow)
     object_concept = require_concept(repo_id, action.object, uow)
+    touched_concept_ids.update((subject.id, object_concept.id))
     predicate = ConceptRelationPredicate(action.predicate)
     validate_relation_shape(
         subject=subject, predicate=predicate, object_concept=object_concept
@@ -182,8 +226,10 @@ def _add_claim(
     uow: IUnitOfWork,
     *,
     id_generator: IIdGenerator,
+    touched_concept_ids: set[str],
 ) -> dict[str, Any]:
     concept = require_concept(repo_id, action.concept, uow)
+    touched_concept_ids.add(concept.id)
     claim = uow.concepts.add_claim(
         ConceptClaim(
             id=id_generator.new_id(),
@@ -229,8 +275,10 @@ def _add_grounding(
     uow: IUnitOfWork,
     *,
     id_generator: IIdGenerator,
+    touched_concept_ids: set[str],
 ) -> dict[str, Any]:
     concept = require_concept(repo_id, action.concept, uow)
+    touched_concept_ids.add(concept.id)
     if action.anchor.id:
         anchor = require_anchor(repo_id, action.anchor.id, uow)
     else:
@@ -265,9 +313,11 @@ def _link_memory(
     uow: IUnitOfWork,
     *,
     id_generator: IIdGenerator,
+    touched_concept_ids: set[str],
 ) -> dict[str, Any]:
     concept = require_concept(repo_id, action.concept, uow)
     memory = require_visible_memory(repo_id, action.memory_id, uow)
+    touched_concept_ids.add(concept.id)
     memory_link = uow.concepts.add_memory_link(
         ConceptMemoryLink(
             id=id_generator.new_id(),