apsimo 1.3.0__py3-none-any.whl

apsimo/secrets/backends/onepassword.py

@@ -0,0 +1,232 @@
+"""1Password secrets backend — supports op CLI and 1Password Connect Server API."""
+
+from __future__ import annotations
+
+import json
+import logging
+import shutil
+import subprocess
+from dataclasses import dataclass, field
+
+from apsimo.secrets.backends.base import SecretsBackend
+from apsimo.secrets.types import SecretType, VAULT_NAME
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class OPItem:
+    """Represents a 1Password item as used by Colony."""
+
+    item_id: str
+    title: str       # Secret key (e.g., "OPENAI_API_KEY")
+    value: str
+    category: str    # Derived from SecretType
+    tags: list[str] = field(default_factory=list)
+
+
+class OnePasswordBackend(SecretsBackend):
+    """1Password secrets backend.
+
+    Supports two modes:
+    1. CLI mode (default): uses ``op`` CLI — requires 1Password desktop app
+       or 1Password CLI with biometric unlock configured.
+    2. Connect mode: uses 1Password Connect Server API — suitable for
+       headless/server deployments. Requires OP_CONNECT_HOST and OP_CONNECT_TOKEN.
+
+    Vault structure:
+      - Default vault name: "Colony" (overridable in config)
+      - Items organised by section: LLM Keys, Infrastructure, Messaging
+        Gateways, Mesh / Federation, Email, Backup, Push Notifications,
+        Colony Core.
+
+    Auto-detects mode based on available configuration.
+    """
+
+    name = "1password"
+
+    # 1Password item sections by SecretType
+    SECTION_MAP: dict[SecretType, str] = {
+        SecretType.LLM_API_KEY: "LLM Keys",
+        SecretType.NEO4J_CREDENTIAL: "Infrastructure",
+        SecretType.GATEWAY_TOKEN: "Messaging Gateways",
+        SecretType.MESH_PAIRING_KEY: "Mesh / Federation",
+        SecretType.EMAIL_CREDENTIAL: "Email",
+        SecretType.BACKUP_PASSPHRASE: "Backup",
+        SecretType.COLONY_TOKEN: "Colony Core",
+        SecretType.FEDERATION_CERT: "Mesh / Federation",
+        SecretType.PUSH_CREDENTIAL: "Push Notifications",
+        SecretType.CALDAV_CREDENTIAL: "Calendar",
+        SecretType.HEALTH_CREDENTIAL: "Health",
+        SecretType.OTHER: "Miscellaneous",
+    }
+
+    def __init__(
+        self,
+        vault_name: str = VAULT_NAME,
+        connect_host: str | None = None,
+        connect_token: str | None = None,
+    ) -> None:
+        self._vault = vault_name
+        self._connect_host = connect_host
+        self._connect_token = connect_token
+        self._use_connect = bool(connect_host and connect_token)
+        self._cache: dict[str, str] = {}  # in-process cache
+
+    def is_available(self) -> bool:
+        if self._use_connect:
+            return bool(self._connect_host and self._connect_token)
+        return shutil.which("op") is not None
+
+    def get(self, key: str, default: str | None = None) -> str | None:
+        if key in self._cache:
+            return self._cache[key]
+        if self._use_connect:
+            return self._get_via_connect(key, default)
+        return self._get_via_cli(key, default)
+
+    def set(self, key: str, value: str, *, secret_type: SecretType | None = None) -> None:
+        if self._use_connect:
+            self._set_via_connect(key, value, secret_type)
+        else:
+            self._set_via_cli(key, value, secret_type)
+        self._cache[key] = value
+
+    def delete(self, key: str) -> None:
+        try:
+            self._run_op(["item", "delete", self._item_title(key), f"--vault={self._vault}"])
+        except subprocess.CalledProcessError as exc:
+            logger.debug("1Password delete failed for %s: %s", key, exc)
+        self._cache.pop(key, None)
+
+    def list(self) -> list[str]:
+        try:
+            result = self._run_op([
+                "item", "list",
+                f"--vault={self._vault}",
+                "--format=json",
+                "--tags=colony",
+            ])
+            items = json.loads(result)
+            prefix = "colony:"
+            return [
+                item["title"][len(prefix):]
+                if item["title"].startswith(prefix)
+                else item["title"]
+                for item in items
+            ]
+        except (subprocess.CalledProcessError, json.JSONDecodeError) as exc:
+            logger.debug("1Password list failed: %s", exc)
+            return []
+
+    # -------------------------------------------------------------------------
+    # CLI helpers
+    # -------------------------------------------------------------------------
+
+    def _get_via_cli(self, key: str, default: str | None) -> str | None:
+        try:
+            result = self._run_op([
+                "item", "get", self._item_title(key),
+                f"--vault={self._vault}",
+                "--fields=password",
+                "--format=json",
+            ])
+            data = json.loads(result)
+            # op returns {"value": "..."} or {"password": "..."} depending on version
+            value = data.get("value") or data.get("password")
+            if value:
+                self._cache[key] = value
+            return value or default
+        except (subprocess.CalledProcessError, json.JSONDecodeError):
+            return default
+
+    def _set_via_cli(
+        self, key: str, value: str, secret_type: SecretType | None
+    ) -> None:
+        section = self.SECTION_MAP.get(secret_type or SecretType.OTHER, "Miscellaneous")
+        title = self._item_title(key)
+
+        # Try update first, create if item not found
+        try:
+            self._run_op([
+                "item", "edit", title,
+                f"--vault={self._vault}",
+                f"password={value}",
+            ])
+        except subprocess.CalledProcessError:
+            self._run_op([
+                "item", "create",
+                "--category=login",
+                f"--title={title}",
+                f"--vault={self._vault}",
+                f"--section={section}",
+                f"password={value}",
+                "--tags=colony",
+            ])
+
+    def _run_op(self, args: list[str]) -> str:
+        result = subprocess.run(
+            ["op"] + args,
+            capture_output=True,
+            text=True,
+            check=True,
+            timeout=10,
+        )
+        return result.stdout.strip()
+
+    @staticmethod
+    def _item_title(key: str) -> str:
+        """Map a secret key to a 1Password item title."""
+        return f"colony:{key}"
+
+    # -------------------------------------------------------------------------
+    # 1Password Connect Server API helpers
+    # -------------------------------------------------------------------------
+
+    def _get_via_connect(self, key: str, default: str | None) -> str | None:
+        """Retrieve via 1Password Connect Server REST API."""
+        import urllib.request
+        url = f"{self._connect_host}/v1/vaults/{self._vault}/items"
+        req = urllib.request.Request(
+            url,
+            headers={"Authorization": f"Bearer {self._connect_token}"},
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=5) as resp:
+                items = json.loads(resp.read())
+                target = self._item_title(key)
+                for item in items:
+                    if item.get("title") == target:
+                        fields = item.get("fields", [])
+                        if fields:
+                            value = fields[0].get("value")
+                            if value:
+                                self._cache[key] = value
+                                return value
+        except Exception as exc:
+            logger.debug("1Password Connect get failed for %s: %s", key, exc)
+        return default
+
+    def _set_via_connect(
+        self, key: str, value: str, secret_type: SecretType | None
+    ) -> None:
+        """Store via 1Password Connect Server REST API."""
+        import urllib.request
+        section = self.SECTION_MAP.get(secret_type or SecretType.OTHER, "Miscellaneous")
+        payload = json.dumps({
+            "title": self._item_title(key),
+            "category": "LOGIN",
+            "tags": ["colony"],
+            "sections": [{"label": section}],
+            "fields": [{"label": "password", "value": value, "type": "CONCEALED"}],
+        }).encode()
+        req = urllib.request.Request(
+            f"{self._connect_host}/v1/vaults/{self._vault}/items",
+            data=payload,
+            headers={
+                "Authorization": f"Bearer {self._connect_token}",
+                "Content-Type": "application/json",
+            },
+            method="POST",
+        )
+        urllib.request.urlopen(req, timeout=5)

apsimo/secrets/cli.py

@@ -0,0 +1,191 @@
+"""CLI command handlers for `colony secrets`."""
+
+from __future__ import annotations
+
+import sys
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import argparse
+
+
+# -------------------------------------------------------------------------
+# Section grouping for pretty-print
+# -------------------------------------------------------------------------
+
+_SECTION_KEYS: dict[str, list[str]] = {
+    "LLM Keys": [
+        "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "OPENROUTER_API_KEY",
+        "GEMINI_API_KEY", "GROQ_API_KEY", "TOGETHER_API_KEY",
+    ],
+    "Infrastructure": ["NEO4J_URI", "NEO4J_USERNAME", "NEO4J_PASSWORD", "COLONY_TOKEN_SECRET"],
+    "Messaging Gateways": [
+        "TELEGRAM_BOT_TOKEN", "DISCORD_BOT_TOKEN", "SLACK_BOT_TOKEN",
+        "WHATSAPP_API_KEY", "TWILIO_ACCOUNT_SID", "TWILIO_AUTH_TOKEN", "SIGNAL_API_KEY",
+    ],
+    "Email": ["EMAIL_IMAP_PASSWORD", "EMAIL_SMTP_PASSWORD"],
+    "Backup": ["BACKUP_PASSPHRASE", "S3_ACCESS_KEY", "S3_SECRET_KEY"],
+    "Push Notifications": ["APNS_AUTH_KEY", "APNS_KEY_ID", "APNS_TEAM_ID", "FCM_SERVICE_ACCOUNT_JSON"],
+    "Calendar / Health": ["CALDAV_PASSWORD", "OURA_PERSONAL_ACCESS_TOKEN"],
+    "1Password Connect": ["OP_CONNECT_HOST", "OP_CONNECT_TOKEN"],
+}
+
+
+def _get_manager():
+    from apsimo.secrets.manager import SecretsManager
+    return SecretsManager()
+
+
+def cmd_secrets_list(args: "argparse.Namespace") -> None:
+    """colony secrets list — show configured secrets grouped by category."""
+    sm = _get_manager()
+    configured = set(sm.list())
+
+    print(f"\nBackend: {sm.backend_name}\n")
+
+    unconfigured: list[str] = []
+    for section, keys in _SECTION_KEYS.items():
+        section_configured = [k for k in keys if k in configured]
+        section_missing = [k for k in keys if k not in configured]
+        unconfigured.extend(section_missing)
+        if section_configured:
+            print(f"{section} ({len(section_configured)} configured):")
+            for k in section_configured:
+                print(f"  \u2713 {k}")
+            print()
+
+    if unconfigured:
+        shown = unconfigured[:10]
+        remaining = len(unconfigured) - len(shown)
+        print("Not configured (run `colony secrets set KEY VALUE`):")
+        for k in shown:
+            print(f"  \u2717 {k}")
+        if remaining > 0:
+            print(f"  ... ({remaining} more)")
+        print()
+
+
+def cmd_secrets_get(args: "argparse.Namespace") -> None:
+    """colony secrets get KEY — retrieve a secret."""
+    sm = _get_manager()
+    value = sm.get(args.key)
+    if value is None:
+        print(f"Secret '{args.key}' not found in {sm.backend_name} backend.", file=sys.stderr)
+        sys.exit(1)
+    print(value)
+
+
+def cmd_secrets_set(args: "argparse.Namespace") -> None:
+    """colony secrets set KEY VALUE — store a secret."""
+    from apsimo.secrets.types import infer_secret_type
+    sm = _get_manager()
+    secret_type = infer_secret_type(args.key)
+    sm.set(args.key, args.value, secret_type=secret_type)
+    print(f"\u2713 Set {args.key} in {sm.backend_name} backend.")
+
+
+def cmd_secrets_delete(args: "argparse.Namespace") -> None:
+    """colony secrets delete KEY — remove a secret."""
+    sm = _get_manager()
+    sm.delete(args.key)
+    print(f"\u2713 Deleted {args.key} from {sm.backend_name} backend.")
+
+
+def cmd_secrets_backend(args: "argparse.Namespace") -> None:
+    """colony secrets backend — show current backend."""
+    sm = _get_manager()
+    print(f"Current secrets backend: {sm.backend_name}")
+    if sm.backend_name == "1password":
+        print("  Storage: 1Password vault")
+    elif sm.backend_name == "keyring":
+        print("  Storage: System keychain (macOS Keychain / Linux SecretService)")
+    else:
+        print("  Storage: ~/.colony/.env (plaintext, mode 0600)")
+    print("\nChange backend: colony setup  (re-run onboarding wizard)")
+
+
+def cmd_secrets_status(args: "argparse.Namespace") -> None:
+    """colony secrets status — check backend availability."""
+    from apsimo.secrets.backends.env import EnvBackend
+    from apsimo.secrets.backends.keyring import KeyringBackend
+    from apsimo.secrets.backends.onepassword import OnePasswordBackend
+
+    sm = _get_manager()
+    print(f"\nActive backend: {sm.backend_name}")
+    print()
+
+    backends = [
+        ("1password", OnePasswordBackend()),
+        ("keyring", KeyringBackend()),
+        ("env", EnvBackend()),
+    ]
+    for name, backend in backends:
+        mark = "\u2713" if backend.is_available() else "\u2717"
+        active = " (active)" if name == sm.backend_name else ""
+        print(f"  {mark} {name}{active}")
+
+    print()
+    configured = sm.list()
+    print(f"Configured secrets: {len(configured)}")
+
+
+def cmd_secrets_migrate(args: "argparse.Namespace") -> None:
+    """colony secrets migrate — move secrets between backends."""
+    from apsimo.secrets.backends.env import EnvBackend
+    from apsimo.secrets.backends.keyring import KeyringBackend
+    from apsimo.secrets.backends.onepassword import OnePasswordBackend
+    from apsimo.secrets.migration import SecretsMigrator
+
+    _backend_map = {
+        "env": EnvBackend,
+        "keyring": KeyringBackend,
+        "1password": OnePasswordBackend,
+    }
+
+    from_name = args.from_backend
+    to_name = args.to_backend
+
+    if from_name not in _backend_map:
+        print(f"Unknown source backend: {from_name}", file=sys.stderr)
+        sys.exit(1)
+    if to_name not in _backend_map:
+        print(f"Unknown target backend: {to_name}", file=sys.stderr)
+        sys.exit(1)
+
+    source = _backend_map[from_name]()
+    target = _backend_map[to_name]()
+
+    if not target.is_available():
+        print(
+            f"Target backend '{to_name}' is not available on this system.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    migrator = SecretsMigrator(source, target)
+
+    dry_run: bool = getattr(args, "dry_run", False)
+    delete_source: bool = getattr(args, "delete_source", False)
+
+    if dry_run:
+        keys = migrator.preview()
+        if not keys:
+            print(f"No secrets found in '{from_name}' backend.")
+            return
+        print(f"Would migrate {len(keys)} secrets from '{from_name}' to '{to_name}':")
+        print("  " + ", ".join(keys))
+        print("\nRun without --dry-run to proceed.")
+        return
+
+    print(f"Migrating secrets from '{from_name}' to '{to_name}'...")
+    result = migrator.migrate(dry_run=False, delete_from_source=delete_source)
+
+    for key in result.migrated:
+        print(f"  \u2713 {key}")
+    for key, err in result.failed:
+        print(f"  \u2717 {key}: {err}", file=sys.stderr)
+
+    print(f"\n{result.summary()}")
+
+    if delete_source and result.migrated:
+        print(f"Removed {len(result.migrated)} secrets from '{from_name}' backend.")

apsimo/secrets/manager.py

@@ -0,0 +1,160 @@
+"""SecretsManager — unified secrets access layer for Colony.
+
+All Colony code MUST use SecretsManager to access secrets.
+Direct reads from os.environ or .env files are DEPRECATED in V2
+and will be removed in V3.
+
+Usage::
+
+    sm = SecretsManager()
+    api_key = sm.get("OPENAI_API_KEY")
+    sm.set("OPENAI_API_KEY", "sk-...")
+    keys = sm.list()
+
+The backend is loaded from ~/.colony/config.yaml on first instantiation
+and cached for the process lifetime.  Override ``COLONY_SECRETS_BACKEND``
+env var to ``env``, ``keyring``, or ``1password`` for testing.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from apsimo.secrets.backends.base import SecretsBackend
+from apsimo.secrets.types import SecretType
+
+logger = logging.getLogger(__name__)
+
+_BACKEND_REGISTRY: dict[str, type[SecretsBackend]] = {}
+
+
+def _register_backends() -> None:
+    """Lazy-load and register all known backends."""
+    global _BACKEND_REGISTRY
+    if _BACKEND_REGISTRY:
+        return
+    from apsimo.secrets.backends.env import EnvBackend
+    from apsimo.secrets.backends.keyring import KeyringBackend
+    from apsimo.secrets.backends.onepassword import OnePasswordBackend
+
+    _BACKEND_REGISTRY = {
+        "env": EnvBackend,
+        "keyring": KeyringBackend,
+        "1password": OnePasswordBackend,
+    }
+
+
+class SecretsManager:
+    """Unified secrets access layer for Colony.
+
+    Singleton — one instance per process, backend determined from config.
+    Use ``SecretsManager.reset()`` in tests to clear cached state.
+    """
+
+    _instance: "SecretsManager | None" = None
+
+    def __new__(cls) -> "SecretsManager":
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            cls._instance._backend = cls._load_backend()
+        return cls._instance
+
+    @classmethod
+    def _load_backend(cls) -> SecretsBackend:
+        _register_backends()
+
+        # Allow override via env var (useful for testing and CI)
+        backend_name = os.environ.get("COLONY_SECRETS_BACKEND")
+
+        if not backend_name:
+            backend_name = cls._read_config_backend()
+
+        if not backend_name:
+            backend_name = "env"
+
+        if backend_name not in _BACKEND_REGISTRY:
+            logger.warning(
+                "Unknown secrets backend %r — falling back to 'env'", backend_name
+            )
+            backend_name = "env"
+
+        backend_cls = _BACKEND_REGISTRY[backend_name]
+        backend = backend_cls()
+
+        if not backend.is_available():
+            logger.warning(
+                "Secrets backend %r is not available on this system — "
+                "falling back to 'env'. Run `colony secrets status` for details.",
+                backend_name,
+            )
+            backend = _BACKEND_REGISTRY["env"]()
+
+        return backend
+
+    @classmethod
+    def _read_config_backend(cls) -> str | None:
+        """Read secrets.backend from ~/.colony/config.yaml."""
+        try:
+            import yaml
+            from pathlib import Path
+
+            colony_home = Path(os.environ.get("COLONY_HOME", Path.home() / ".colony"))
+            config_path = colony_home / "config.yaml"
+            if not config_path.exists():
+                return None
+            with open(config_path, encoding="utf-8") as f:
+                cfg = yaml.safe_load(f) or {}
+            return cfg.get("secrets", {}).get("backend")
+        except Exception as exc:
+            logger.debug("Could not read secrets backend from config: %s", exc)
+            return None
+
+    # -------------------------------------------------------------------------
+    # Public API
+    # -------------------------------------------------------------------------
+
+    def get(self, key: str, default: str | None = None) -> str | None:
+        """Retrieve a secret by key. Returns default if not found."""
+        return self._backend.get(key, default)
+
+    def get_required(self, key: str) -> str:
+        """Retrieve a secret by key. Raises KeyError if not found."""
+        value = self._backend.get(key)
+        if value is None:
+            raise KeyError(
+                f"Required secret '{key}' not found in {self._backend.name} backend. "
+                f"Run `colony secrets set {key} <value>` to configure it."
+            )
+        return value
+
+    def set(self, key: str, value: str, *, secret_type: SecretType | None = None) -> None:
+        """Store a secret."""
+        self._backend.set(key, value, secret_type=secret_type)
+
+    def delete(self, key: str) -> None:
+        """Remove a secret."""
+        self._backend.delete(key)
+
+    def list(self) -> list[str]:
+        """List all known secret keys (not values)."""
+        return self._backend.list()
+
+    @property
+    def backend_name(self) -> str:
+        return self._backend.name
+
+    @property
+    def backend(self) -> SecretsBackend:
+        return self._backend
+
+    @classmethod
+    def reset(cls) -> None:
+        """Clear the singleton — for testing only."""
+        cls._instance = None
+
+    @classmethod
+    def with_backend(cls, backend: SecretsBackend) -> "SecretsManager":
+        """Create a SecretsManager instance with a specific backend — for testing."""
+        instance = object.__new__(cls)
+        instance._backend = backend
+        return instance

apsimo/secrets/migration.py

@@ -0,0 +1,101 @@
+"""Auto-migration utilities for moving secrets between backends."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+
+from apsimo.secrets.backends.base import SecretsBackend
+from apsimo.secrets.types import infer_secret_type
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class MigrationResult:
+    """Result summary from a secrets migration run."""
+
+    migrated: list[str] = field(default_factory=list)
+    skipped: list[str] = field(default_factory=list)
+    failed: list[tuple[str, str]] = field(default_factory=list)
+    would_migrate: list[str] = field(default_factory=list)
+
+    @property
+    def success_count(self) -> int:
+        return len(self.migrated)
+
+    @property
+    def failed_count(self) -> int:
+        return len(self.failed)
+
+    def summary(self) -> str:
+        parts = []
+        if self.would_migrate:
+            parts.append(f"Would migrate {len(self.would_migrate)} secrets (dry run)")
+        if self.migrated:
+            parts.append(f"Migrated {len(self.migrated)} secrets")
+        if self.skipped:
+            parts.append(f"Skipped {len(self.skipped)} (no value)")
+        if self.failed:
+            parts.append(f"Failed {len(self.failed)}")
+        return ", ".join(parts) if parts else "Nothing to migrate"
+
+
+class SecretsMigrator:
+    """Detect existing secrets and migrate them between backends.
+
+    Called by the onboarding wizard when a user changes their secrets backend,
+    and optionally on first startup after a V2 upgrade if the user has a .env
+    file but has selected 1Password or keyring as their backend.
+    """
+
+    def __init__(self, source: SecretsBackend, target: SecretsBackend) -> None:
+        self._source = source
+        self._target = target
+
+    def preview(self) -> list[str]:
+        """Return list of secret keys that would be migrated."""
+        return self._source.list()
+
+    def migrate(
+        self,
+        keys: list[str] | None = None,
+        *,
+        dry_run: bool = False,
+        delete_from_source: bool = False,
+    ) -> MigrationResult:
+        """Migrate secrets from source to target backend.
+
+        Args:
+            keys: Specific keys to migrate.  None = migrate all from source.
+            dry_run: If True, only report what would be migrated without writing.
+            delete_from_source: If True, delete from source after verifying target.
+
+        Returns:
+            MigrationResult with counts of migrated, skipped, and failed keys.
+        """
+        to_migrate = keys if keys is not None else self._source.list()
+        result = MigrationResult()
+
+        for key in to_migrate:
+            value = self._source.get(key)
+            if value is None:
+                result.skipped.append(key)
+                continue
+            if dry_run:
+                result.would_migrate.append(key)
+                continue
+            try:
+                secret_type = infer_secret_type(key)
+                self._target.set(key, value, secret_type=secret_type)
+                # Verify the write succeeded
+                if self._target.get(key) != value:
+                    raise RuntimeError("Read-back verification failed")
+                result.migrated.append(key)
+                if delete_from_source:
+                    self._source.delete(key)
+            except Exception as exc:
+                logger.warning("Failed to migrate secret %s: %s", key, exc)
+                result.failed.append((key, str(exc)))
+
+        return result