shellbrain 0.1.0__py3-none-any.whl

app/__init__.py

@@ -0,0 +1 @@
+"""This package contains the shellbrain system application code."""

app/__main__.py

@@ -0,0 +1,7 @@
+"""Allow `python -m app` to invoke the public CLI."""
+
+from app.periphery.cli.main import main
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

app/boot/__init__.py

@@ -0,0 +1 @@
+"""This package contains factory functions that wire core to periphery."""

app/boot/admin_db.py

@@ -0,0 +1,88 @@
+"""Boot helpers for privileged admin database actions and safety settings."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from app.boot.config import get_config_provider
+from app.boot.home import get_machine_backups_dir
+from app.periphery.admin.machine_state import try_load_machine_config
+
+
+def get_admin_db_dsn() -> str:
+    """Resolve the privileged admin DSN from environment-backed runtime config."""
+
+    machine_config, machine_error = try_load_machine_config()
+    if machine_error:
+        raise RuntimeError(
+            "Shellbrain machine config is unreadable. Rerun `shellbrain init` to repair it."
+        )
+    if machine_config is not None:
+        return machine_config.database.admin_dsn
+
+    runtime = get_config_provider().get_runtime()
+    database = runtime.get("database")
+    if not isinstance(database, dict):
+        raise RuntimeError("runtime.database must be configured")
+    admin_dsn_env = database.get("admin_dsn_env")
+    if not isinstance(admin_dsn_env, str) or not admin_dsn_env:
+        raise RuntimeError("runtime.database.admin_dsn_env must be configured")
+    dsn = os.getenv(admin_dsn_env)
+    if not dsn:
+        raise RuntimeError(f"{admin_dsn_env} is not set")
+    return dsn
+
+
+def get_optional_admin_db_dsn() -> str | None:
+    """Resolve the privileged admin DSN when present, otherwise return None."""
+
+    machine_config, machine_error = try_load_machine_config()
+    if machine_error:
+        return None
+    if machine_config is not None:
+        return machine_config.database.admin_dsn
+
+    runtime = get_config_provider().get_runtime()
+    database = runtime.get("database")
+    if not isinstance(database, dict):
+        return None
+    admin_dsn_env = database.get("admin_dsn_env")
+    if not isinstance(admin_dsn_env, str) or not admin_dsn_env:
+        return None
+    return os.getenv(admin_dsn_env)
+
+
+def get_backup_dir() -> Path:
+    """Resolve the on-disk backup directory, defaulting outside the repo tree."""
+
+    machine_config, machine_error = try_load_machine_config()
+    if machine_error:
+        return get_machine_backups_dir()
+    if machine_config is not None:
+        return Path(machine_config.backups.root).expanduser().resolve()
+    return Path(os.getenv("SHELLBRAIN_BACKUP_DIR", str(get_machine_backups_dir()))).expanduser().resolve()
+
+
+def get_backup_mirror_dir() -> Path | None:
+    """Resolve the optional mirrored backup directory."""
+
+    configured = os.getenv("SHELLBRAIN_BACKUP_MIRROR_DIR")
+    if not configured:
+        return None
+    return Path(configured).expanduser().resolve()
+
+
+def should_fail_on_unsafe_app_role() -> bool:
+    """Return whether app commands should fail instead of warning on unsafe DB roles."""
+
+    configured = os.getenv("SHELLBRAIN_FAIL_ON_UNSAFE_DB_ROLE")
+    if configured is None or not configured.strip():
+        return True
+    return configured.strip().lower() not in {"0", "false", "no", "off"}
+
+
+def get_instance_mode_default() -> str:
+    """Resolve the default instance mode used when stamping metadata for the current DB."""
+
+    return os.getenv("SHELLBRAIN_INSTANCE_MODE", "live").strip().lower() or "live"

app/boot/config.py

@@ -0,0 +1,14 @@
+"""This module defines boot-time helpers that load YAML-backed configuration providers."""
+
+from functools import lru_cache
+from pathlib import Path
+
+from app.config.loader import YamlConfigProvider
+
+
+@lru_cache(maxsize=1)
+def get_config_provider() -> YamlConfigProvider:
+    """This function returns the shared YAML configuration provider instance."""
+
+    defaults_dir = Path(__file__).resolve().parents[1] / "config" / "defaults"
+    return YamlConfigProvider(defaults_dir)

app/boot/create_policy.py

@@ -0,0 +1,52 @@
+"""Boot-time helpers for normalized create-policy settings."""
+
+from typing import Any
+
+from app.boot.config import get_config_provider
+from app.core.contracts.errors import ErrorCode, ErrorDetail
+
+
+_SUPPORTED_GATES = ("schema", "semantic", "integrity")
+_SUPPORTED_SCOPES = ("repo", "global")
+
+
+def get_create_policy_settings() -> dict[str, Any]:
+    """Return normalized create-policy settings from YAML config."""
+
+    policy = get_config_provider().get_create_policy()
+    configured_gates = policy.get("gates")
+    if not isinstance(configured_gates, list) or not configured_gates:
+        raise ValueError("create_policy.gates must be a non-empty list")
+    gates = [str(gate) for gate in configured_gates if str(gate) in _SUPPORTED_GATES]
+    if len(gates) != len(configured_gates):
+        raise ValueError("create_policy.gates contains unsupported values")
+    if "schema" not in gates:
+        raise ValueError("create_policy.gates must include schema")
+    configured_defaults = policy.get("defaults")
+    if not isinstance(configured_defaults, dict):
+        raise ValueError("create_policy.defaults must be a mapping")
+    scope = configured_defaults.get("scope")
+    if not isinstance(scope, str) or scope not in _SUPPORTED_SCOPES:
+        raise ValueError("create_policy.defaults.scope must be repo or global")
+    return {
+        "gates": gates,
+        "defaults": {
+            "scope": scope,
+        },
+    }
+
+
+def get_create_hydration_defaults() -> dict[str, Any]:
+    """Return normalized create defaults used by CLI hydration."""
+
+    return dict(get_create_policy_settings()["defaults"])
+
+
+def validate_create_policy_settings() -> list[ErrorDetail]:
+    """Return structured config errors for unsupported create-policy settings."""
+
+    try:
+        get_create_policy_settings()
+    except ValueError as exc:
+        return [ErrorDetail(code=ErrorCode.INTERNAL_ERROR, message=str(exc), field="create_policy.gates")]
+    return []

app/boot/db.py

@@ -0,0 +1,70 @@
+"""This module defines boot-time factory helpers for database engine and sessions."""
+
+import os
+from pathlib import Path
+
+from app.boot.config import get_config_provider
+from app.periphery.admin.machine_state import try_load_machine_config
+from app.periphery.db.engine import get_engine
+from app.periphery.db.session import get_session_factory
+
+
+def get_db_dsn() -> str:
+    """This function resolves the database DSN from environment configuration."""
+
+    machine_config, machine_error = try_load_machine_config()
+    if machine_error:
+        raise RuntimeError(
+            "Shellbrain machine config is unreadable. Rerun `shellbrain init` to repair it."
+        )
+    if machine_config is not None:
+        return machine_config.database.app_dsn
+
+    runtime = get_config_provider().get_runtime()
+    database = runtime.get("database")
+    if not isinstance(database, dict):
+        raise RuntimeError("runtime.database must be configured")
+    dsn_env = database.get("dsn_env")
+    if not isinstance(dsn_env, str) or not dsn_env:
+        raise RuntimeError("runtime.database.dsn_env must be configured")
+    dsn = os.getenv(dsn_env)
+    if not dsn:
+        raise RuntimeError(f"{dsn_env} is not set")
+    return dsn
+
+
+def get_optional_db_dsn() -> str | None:
+    """Resolve the application DSN when present, otherwise return None."""
+
+    machine_config, machine_error = try_load_machine_config()
+    if machine_error:
+        return None
+    if machine_config is not None:
+        return machine_config.database.app_dsn
+
+    runtime = get_config_provider().get_runtime()
+    database = runtime.get("database")
+    if not isinstance(database, dict):
+        return None
+    dsn_env = database.get("dsn_env")
+    if not isinstance(dsn_env, str) or not dsn_env:
+        return None
+    return os.getenv(dsn_env)
+
+
+def get_engine_instance():
+    """This function builds a shared SQLAlchemy engine for the application."""
+
+    return get_engine(get_db_dsn())
+
+
+def get_session_factory_instance():
+    """This function builds a reusable SQLAlchemy session factory for the app."""
+
+    return get_session_factory(get_engine_instance())
+
+
+def get_defaults_dir() -> Path:
+    """This function returns the path to bundled YAML default configuration files."""
+
+    return Path(__file__).resolve().parents[1] / "config" / "defaults"

app/boot/embeddings.py

@@ -0,0 +1,55 @@
+"""This module defines boot-time wiring for embedding provider construction."""
+
+from app.boot.home import get_machine_models_dir
+from app.boot.config import get_config_provider
+from app.core.interfaces.embeddings import IEmbeddingProvider
+from app.periphery.admin.machine_state import load_machine_config
+from app.periphery.embeddings.local_provider import SentenceTransformersEmbeddingProvider
+
+
+def _get_embedding_config() -> dict:
+    """This function returns runtime embedding configuration values."""
+
+    runtime = get_config_provider().get_runtime()
+    values = runtime.get("embeddings")
+    if not isinstance(values, dict):
+        raise ValueError("runtime.embeddings must be configured")
+    return values
+
+
+def get_embedding_model_name() -> str:
+    """This function resolves the model name persisted alongside embedding vectors."""
+
+    config = _get_embedding_config()
+    provider = config.get("provider")
+    model = config.get("model")
+    if not isinstance(provider, str) or not provider:
+        raise ValueError("runtime.embeddings.provider must be configured")
+    if not isinstance(model, str) or not model:
+        raise ValueError("runtime.embeddings.model must be configured")
+    if provider == "sentence_transformers":
+        return model
+    raise ValueError(f"Unsupported embedding provider: {provider}")
+
+
+def get_embedding_provider() -> IEmbeddingProvider:
+    """This function constructs the configured local embedding provider."""
+
+    config = _get_embedding_config()
+    provider = config.get("provider")
+    model = config.get("model")
+    if not isinstance(provider, str) or not provider:
+        raise ValueError("runtime.embeddings.provider must be configured")
+    if not isinstance(model, str) or not model:
+        raise ValueError("runtime.embeddings.model must be configured")
+    if provider == "sentence_transformers":
+        machine_config = load_machine_config()
+        cache_folder = str(get_machine_models_dir())
+        if machine_config is not None:
+            cache_folder = machine_config.embeddings.cache_path
+            if machine_config.embeddings.readiness_state != "ready":
+                raise RuntimeError(
+                    "Shellbrain embeddings are not ready. Rerun `shellbrain init` to finish model setup."
+                )
+        return SentenceTransformersEmbeddingProvider(model=model, cache_folder=cache_folder)
+    raise ValueError(f"Unsupported embedding provider: {provider}")

app/boot/home.py

@@ -0,0 +1,45 @@
+"""Helpers for locating Shellbrain machine-owned runtime directories."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+
+def get_shellbrain_home() -> Path:
+    """Return the machine-owned Shellbrain home root."""
+
+    configured = os.getenv("SHELLBRAIN_HOME")
+    if configured:
+        return Path(configured).expanduser().resolve()
+    return Path("~/.shellbrain").expanduser().resolve()
+
+
+def get_machine_config_path() -> Path:
+    """Return the machine configuration file path."""
+
+    return get_shellbrain_home() / "config.toml"
+
+
+def get_machine_lock_path() -> Path:
+    """Return the machine-scoped init lock path."""
+
+    return get_shellbrain_home() / "init.lock"
+
+
+def get_machine_models_dir() -> Path:
+    """Return the machine-owned embedding model cache path."""
+
+    return get_shellbrain_home() / "models"
+
+
+def get_machine_backups_dir() -> Path:
+    """Return the machine-owned default backup directory."""
+
+    return get_shellbrain_home() / "backups"
+
+
+def get_machine_postgres_data_dir() -> Path:
+    """Return the managed Postgres bind-mounted data directory."""
+
+    return get_shellbrain_home() / "postgres-data"

app/boot/migrations.py

@@ -0,0 +1,61 @@
+"""Packaged Alembic bootstrap helpers for installed-shellbrain database migrations."""
+
+from __future__ import annotations
+
+from importlib.resources import as_file, files
+
+from alembic import command
+from alembic.config import Config
+
+from app.boot.admin_db import get_admin_db_dsn, get_backup_dir, get_backup_mirror_dir, get_instance_mode_default
+from app.boot.db import get_optional_db_dsn
+from app.periphery.admin.destructive_guard import backup_and_verify_before_destructive_action
+from app.periphery.admin.instance_guard import ensure_instance_metadata, fetch_instance_metadata
+from app.periphery.admin.privileges import reconcile_app_role_privileges
+
+
+def upgrade_database(revision: str = "head") -> None:
+    """Apply packaged Alembic migrations to the configured database."""
+
+    config = Config()
+    admin_dsn = get_admin_db_dsn()
+    if _database_has_shellbrain_objects(admin_dsn):
+        backup_and_verify_before_destructive_action(
+            admin_dsn=admin_dsn,
+            backup_root=get_backup_dir(),
+            mirror_root=get_backup_mirror_dir(),
+        )
+    config.set_main_option("sqlalchemy.url", admin_dsn)
+    with as_file(files("app").joinpath("migrations")) as migrations_path:
+        config.set_main_option("script_location", str(migrations_path))
+        command.upgrade(config, revision)
+    if fetch_instance_metadata(admin_dsn) is None:
+        ensure_instance_metadata(
+            admin_dsn,
+            instance_mode=get_instance_mode_default(),
+            created_by="app.admin.migrate",
+            notes="Stamped by packaged migration runner.",
+        )
+    app_dsn = get_optional_db_dsn()
+    if app_dsn:
+        reconcile_app_role_privileges(admin_dsn=admin_dsn, app_dsn=app_dsn)
+
+
+def _database_has_shellbrain_objects(admin_dsn: str) -> bool:
+    """Return whether the target database already contains Shellbrain-managed tables."""
+
+    import psycopg
+
+    with psycopg.connect(admin_dsn.replace("+psycopg", "")) as conn:
+        with conn.cursor() as cur:
+            cur.execute(
+                """
+                SELECT EXISTS (
+                  SELECT 1
+                  FROM information_schema.tables
+                  WHERE table_schema = 'public'
+                    AND table_name IN ('memories', 'episodes', 'episode_events', 'operation_invocations')
+                )
+                """
+            )
+            return bool(cur.fetchone()[0])

app/boot/read_policy.py

@@ -0,0 +1,179 @@
+"""Boot-time helpers for resolving YAML-backed read-policy settings."""
+
+from copy import deepcopy
+from typing import Any
+
+from app.boot.config import get_config_provider
+
+
+_SUPPORTED_MODES = ("targeted", "ambient")
+_SUPPORTED_BUCKETS = ("direct", "explicit", "implicit")
+_EXPAND_INT_FIELDS = ("semantic_hops", "max_association_depth")
+_EXPAND_BOOL_FIELDS = (
+    "include_problem_links",
+    "include_fact_update_links",
+    "include_association_links",
+)
+_EXPAND_FLOAT_FIELDS = ("min_association_strength",)
+
+
+def _require_mapping(value: Any, *, field: str) -> dict[str, Any]:
+    """Require one config node to be a mapping."""
+
+    if not isinstance(value, dict):
+        raise ValueError(f"{field} must be a mapping")
+    return value
+
+
+def _require_bool(mapping: dict[str, Any], key: str, *, field: str) -> bool:
+    """Require one config value to be a boolean."""
+
+    value = mapping.get(key)
+    if not isinstance(value, bool):
+        raise ValueError(f"{field}.{key} must be a boolean")
+    return value
+
+
+def _require_int(mapping: dict[str, Any], key: str, *, field: str) -> int:
+    """Require one config value to be an integer."""
+
+    value = mapping.get(key)
+    if isinstance(value, bool) or not isinstance(value, int):
+        raise ValueError(f"{field}.{key} must be an integer")
+    return int(value)
+
+
+def _require_float(mapping: dict[str, Any], key: str, *, field: str) -> float:
+    """Require one config value to be numeric."""
+
+    value = mapping.get(key)
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        raise ValueError(f"{field}.{key} must be numeric")
+    return float(value)
+
+
+def _require_mode(value: Any, *, field: str) -> str:
+    """Require one config value to be a supported read mode."""
+
+    if not isinstance(value, str) or value not in _SUPPORTED_MODES:
+        raise ValueError(f"{field} must be one of: {', '.join(_SUPPORTED_MODES)}")
+    return value
+
+
+def get_read_settings() -> dict[str, Any]:
+    """Return normalized read settings from YAML-backed runtime and policy config."""
+
+    config_provider = get_config_provider()
+    read_policy = _require_mapping(config_provider.get_read_policy(), field="read_policy")
+    runtime = _require_mapping(config_provider.get_runtime(), field="runtime")
+    cli_defaults = _require_mapping(runtime.get("cli"), field="runtime.cli")
+    limits = _require_mapping(read_policy.get("limits"), field="read_policy.limits")
+    expansion = _require_mapping(read_policy.get("expansion"), field="read_policy.expansion")
+    quotas = _require_mapping(read_policy.get("quotas"), field="read_policy.quotas")
+    weights = _require_mapping(read_policy.get("weights"), field="read_policy.weights")
+    fusion = _require_mapping(read_policy.get("fusion"), field="read_policy.fusion")
+
+    settings = {
+        "default_mode": _require_mode(cli_defaults.get("default_mode"), field="runtime.cli.default_mode"),
+        "include_global": _require_bool(cli_defaults, "include_global", field="runtime.cli"),
+        "limits_by_mode": {
+            mode: _require_int(limits, mode, field="read_policy.limits")
+            for mode in _SUPPORTED_MODES
+        },
+        "expand": {
+            **{
+                key: _require_int(expansion, key, field="read_policy.expansion")
+                for key in _EXPAND_INT_FIELDS
+            },
+            **{
+                key: _require_bool(expansion, key, field="read_policy.expansion")
+                for key in _EXPAND_BOOL_FIELDS
+            },
+            **{
+                key: _require_float(expansion, key, field="read_policy.expansion")
+                for key in _EXPAND_FLOAT_FIELDS
+            },
+        },
+        "quotas_by_mode": {
+            mode: {
+                bucket: _require_int(
+                    _require_mapping(quotas.get(mode), field=f"read_policy.quotas.{mode}"),
+                    bucket,
+                    field=f"read_policy.quotas.{mode}",
+                )
+                for bucket in _SUPPORTED_BUCKETS
+            }
+            for mode in _SUPPORTED_MODES
+        },
+        "retrieval": {
+            "semantic_weight": _require_float(weights, "semantic", field="read_policy.weights"),
+            "keyword_weight": _require_float(weights, "keyword", field="read_policy.weights"),
+            "k_rrf": _require_float(fusion, "k_rrf", field="read_policy.fusion"),
+        },
+    }
+    return deepcopy(settings)
+
+
+def get_read_hydration_defaults() -> dict[str, Any]:
+    """Return the read defaults expected by CLI hydration."""
+
+    settings = get_read_settings()
+    return {
+        "default_mode": settings["default_mode"],
+        "include_global": settings["include_global"],
+        "limits_by_mode": deepcopy(settings["limits_by_mode"]),
+        "expand": deepcopy(settings["expand"]),
+    }
+
+
+def get_retrieval_defaults() -> dict[str, float]:
+    """Return normalized retrieval defaults for fusion and seed retrieval."""
+
+    return dict(get_read_settings()["retrieval"])
+
+
+def resolve_read_limit(*, mode: str, explicit_limit: int | None) -> int:
+    """Resolve the effective read limit from explicit payload or mode-based config."""
+
+    if explicit_limit is not None:
+        return int(explicit_limit)
+    settings = get_read_settings()
+    resolved_mode = _require_mode(mode, field="read.mode")
+    return int(settings["limits_by_mode"][resolved_mode])
+
+
+def resolve_read_quotas(*, mode: str) -> dict[str, int]:
+    """Resolve the configured context-pack quotas for one read mode."""
+
+    settings = get_read_settings()
+    resolved_mode = _require_mode(mode, field="read.mode")
+    quotas = settings["quotas_by_mode"][resolved_mode]
+    return {bucket: int(value) for bucket, value in quotas.items()}
+
+
+def resolve_read_payload_defaults(payload: dict[str, Any]) -> dict[str, Any]:
+    """Resolve effective read payload defaults from YAML-backed settings."""
+
+    settings = get_read_settings()
+    resolved = dict(payload)
+    mode = resolved.get("mode")
+    if mode is None:
+        mode = settings["default_mode"]
+    resolved["mode"] = _require_mode(mode, field="read.mode")
+    if resolved.get("include_global") is None:
+        resolved["include_global"] = settings["include_global"]
+    if resolved.get("limit") is None:
+        resolved["limit"] = settings["limits_by_mode"][resolved["mode"]]
+
+    incoming_expand = resolved.get("expand")
+    merged_expand = deepcopy(settings["expand"])
+    if incoming_expand is None:
+        resolved["expand"] = merged_expand
+        return resolved
+    if not isinstance(incoming_expand, dict):
+        raise ValueError("read.expand must be a mapping")
+    for key, value in incoming_expand.items():
+        if value is not None:
+            merged_expand[key] = value
+    resolved["expand"] = merged_expand
+    return resolved

app/boot/repos.py

@@ -0,0 +1,15 @@
+"""This module defines boot-time helpers that expose repository-ready unit-of-work factories."""
+
+from app.boot.db import get_session_factory_instance
+from app.boot.embeddings import get_embedding_provider
+from app.periphery.db.uow import PostgresUnitOfWork
+from app.periphery.embeddings.query_vector_search import EmbeddingBackedVectorSearch
+
+
+def get_uow() -> PostgresUnitOfWork:
+    """This function creates a fresh unit-of-work instance with bound repositories."""
+
+    return PostgresUnitOfWork(
+        get_session_factory_instance(),
+        vector_search_factory=lambda: EmbeddingBackedVectorSearch(get_embedding_provider()),
+    )

app/boot/retrieval.py

@@ -0,0 +1,3 @@
+"""This module defines boot-time helpers for retrieval repository wiring."""
+
+from app.boot.read_policy import get_retrieval_defaults

app/boot/thresholds.py

@@ -0,0 +1,19 @@
+"""Boot-time helpers for normalized retrieval threshold settings."""
+
+from app.boot.config import get_config_provider
+
+
+def get_threshold_settings() -> dict[str, float]:
+    """Return normalized retrieval thresholds from YAML config."""
+
+    thresholds = get_config_provider().get_thresholds()
+    semantic_threshold = thresholds.get("semantic_threshold")
+    keyword_threshold = thresholds.get("keyword_threshold")
+    if isinstance(semantic_threshold, bool) or not isinstance(semantic_threshold, (int, float)):
+        raise ValueError("thresholds.semantic_threshold must be numeric")
+    if isinstance(keyword_threshold, bool) or not isinstance(keyword_threshold, (int, float)):
+        raise ValueError("thresholds.keyword_threshold must be numeric")
+    return {
+        "semantic_threshold": float(semantic_threshold),
+        "keyword_threshold": float(keyword_threshold),
+    }

app/boot/update_policy.py

@@ -0,0 +1,34 @@
+"""Boot-time helpers for normalized update-policy settings."""
+
+from typing import Any
+
+from app.boot.config import get_config_provider
+from app.core.contracts.errors import ErrorCode, ErrorDetail
+
+
+_SUPPORTED_GATES = ("schema", "semantic", "integrity")
+
+
+def get_update_policy_settings() -> dict[str, Any]:
+    """Return normalized update-policy settings from YAML config."""
+
+    policy = get_config_provider().get_update_policy()
+    configured_gates = policy.get("gates")
+    if not isinstance(configured_gates, list) or not configured_gates:
+        raise ValueError("update_policy.gates must be a non-empty list")
+    gates = [str(gate) for gate in configured_gates if str(gate) in _SUPPORTED_GATES]
+    if len(gates) != len(configured_gates):
+        raise ValueError("update_policy.gates contains unsupported values")
+    if "schema" not in gates:
+        raise ValueError("update_policy.gates must include schema")
+    return {"gates": gates}
+
+
+def validate_update_policy_settings() -> list[ErrorDetail]:
+    """Return structured config errors for unsupported update-policy settings."""
+
+    try:
+        get_update_policy_settings()
+    except ValueError as exc:
+        return [ErrorDetail(code=ErrorCode.INTERNAL_ERROR, message=str(exc), field="update_policy.gates")]
+    return []