hyperagents 1.2.3rc5.post3__py3-none-any.whl

hyperclaw/teamsetup/secrets.py

@@ -0,0 +1,313 @@
+"""Where onboarding keeps secrets: the OS secret store if there is one, else
+an owner-only file.
+
+Backends (first that applies; ``HYPERCLAW_SECRET_STORE=keychain|secret-tool|file``
+overrides):
+
+- ``keychain`` (macOS): generic passwords, service ``hyperclaw``, account =
+  the secret's name. Written through ``security -i`` on stdin, so a value is
+  never on a command line or in the process list.
+- ``secret-tool`` (Linux with the Secret Service): attributes
+  ``service hyperclaw account <name>``; the value goes on stdin.
+- ``file``: ``$HYPERCLAW_ROOT/secrets/team-secrets.json``, directory 0700,
+  file 0600. A file readable by group or others is refused.
+
+Values are stored base64url-encoded, so any value (a JSON token bundle, for
+instance) is safe on the ``security -i`` command line.
+
+Some secrets are read by the rest of HyperAgents from the environment
+(``ANTHROPIC_API_KEY``, ``TELEGRAM_BOT_TOKEN``). The non-secret map
+``$HYPERCLAW_ROOT/config/secret-env.json`` names them; ``export_env()``
+copies them into the process environment at start-up (never overriding a
+value already set). Every entry point that loads ``.env`` calls it.
+
+Nothing here prints, logs or returns a value in an error message.
+"""
+from __future__ import annotations
+
+import base64
+import json
+import logging
+import os
+import re
+import shutil
+import stat
+import subprocess
+import sys
+from pathlib import Path
+from typing import Callable, Optional
+
+log = logging.getLogger("hyperclaw.teamsetup.secrets")
+
+SERVICE = "hyperclaw"
+STORE_ENV = "HYPERCLAW_SECRET_STORE"
+_NAME = re.compile(r"^[a-z0-9_.-]{1,64}$")
+
+
+class SecretStoreError(Exception):
+    """A secret couldn't be stored or read. The text never holds a value."""
+
+
+def _root() -> Path:
+    from hyperclaw.userconfig import root
+    return root()
+
+
+def _enc(value: str) -> str:
+    return base64.urlsafe_b64encode(value.encode("utf-8")).decode("ascii")
+
+
+def _dec(raw: str) -> str:
+    raw = (raw or "").strip()
+    if not raw:
+        return ""
+    try:
+        return base64.urlsafe_b64decode(raw.encode("ascii")).decode("utf-8")
+    except Exception:
+        return ""
+
+
+def _check_name(name: str) -> str:
+    if not _NAME.match(name or ""):
+        raise SecretStoreError("invalid secret name")
+    return name
+
+
+# ── file backend ──────────────────────────────────────────────────────────
+
+class FileStore:
+    name = "file"
+    where = "an owner-only file (0600)"
+
+    def __init__(self, path: Optional[Path] = None):
+        self.path = path or (_root() / "secrets" / "team-secrets.json")
+
+    def _read(self) -> dict:
+        p = self.path
+        if not p.exists():
+            return {}
+        if stat.S_IMODE(p.stat().st_mode) & 0o077:
+            raise SecretStoreError("the secrets file is readable by other accounts "
+                                   "(it must be 0600)")
+        try:
+            data = json.loads(p.read_text(encoding="utf-8"))
+        except Exception:
+            raise SecretStoreError("the secrets file is not valid JSON")
+        return data if isinstance(data, dict) else {}
+
+    def _write(self, data: dict) -> None:
+        d = self.path.parent
+        d.mkdir(parents=True, exist_ok=True)
+        try:
+            os.chmod(d, 0o700)
+        except OSError:
+            pass
+        tmp = self.path.with_suffix(".tmp")
+        fd = os.open(str(tmp), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+        with os.fdopen(fd, "w", encoding="utf-8") as f:
+            json.dump(data, f)
+        os.chmod(tmp, 0o600)
+        os.replace(tmp, self.path)
+
+    def get(self, name: str) -> str:
+        return _dec(str(self._read().get(_check_name(name)) or ""))
+
+    def set(self, name: str, value: str) -> None:
+        data = self._read()
+        data[_check_name(name)] = _enc(value)
+        self._write(data)
+
+    def delete(self, name: str) -> bool:
+        data = self._read()
+        had = data.pop(_check_name(name), None) is not None
+        if had:
+            self._write(data)
+        return had
+
+
+# ── macOS keychain ────────────────────────────────────────────────────────
+
+Runner = Callable[..., subprocess.CompletedProcess]
+
+
+class KeychainStore:
+    name = "keychain"
+    where = "the macOS keychain"
+
+    def __init__(self, runner: Optional[Runner] = None):
+        self.run = runner or subprocess.run
+
+    def get(self, name: str) -> str:
+        r = self.run(["security", "find-generic-password", "-s", SERVICE, "-a",
+                      _check_name(name), "-w"], capture_output=True, text=True)
+        if r.returncode != 0:
+            return ""
+        return _dec(r.stdout or "")
+
+    def set(self, name: str, value: str) -> None:
+        cmd = f"add-generic-password -U -s {SERVICE} -a {_check_name(name)} -w {_enc(value)}\n"
+        r = self.run(["security", "-i"], input=cmd, capture_output=True, text=True)
+        if r.returncode != 0:
+            raise SecretStoreError(f"the keychain refused the item (exit {r.returncode})")
+
+    def delete(self, name: str) -> bool:
+        r = self.run(["security", "delete-generic-password", "-s", SERVICE, "-a",
+                      _check_name(name)], capture_output=True, text=True)
+        return r.returncode == 0
+
+
+# ── Linux Secret Service ──────────────────────────────────────────────────
+
+class SecretToolStore:
+    name = "secret-tool"
+    where = "the system keyring"
+
+    def __init__(self, runner: Optional[Runner] = None):
+        self.run = runner or subprocess.run
+
+    def get(self, name: str) -> str:
+        r = self.run(["secret-tool", "lookup", "service", SERVICE, "account", _check_name(name)],
+                     capture_output=True, text=True)
+        return _dec(r.stdout or "") if r.returncode == 0 else ""
+
+    def set(self, name: str, value: str) -> None:
+        r = self.run(["secret-tool", "store", f"--label=HyperAgents {name}", "service", SERVICE,
+                      "account", _check_name(name)], input=_enc(value), capture_output=True,
+                     text=True)
+        if r.returncode != 0:
+            raise SecretStoreError(f"the keyring refused the item (exit {r.returncode})")
+
+    def delete(self, name: str) -> bool:
+        r = self.run(["secret-tool", "clear", "service", SERVICE, "account", _check_name(name)],
+                     capture_output=True, text=True)
+        return r.returncode == 0
+
+
+def backend_name() -> str:
+    env = os.environ.get(STORE_ENV, "").strip().lower()
+    if env in ("keychain", "secret-tool", "file"):
+        return env
+    if sys.platform == "darwin" and shutil.which("security"):
+        return "keychain"
+    if sys.platform.startswith("linux") and shutil.which("secret-tool") \
+            and os.environ.get("DBUS_SESSION_BUS_ADDRESS"):
+        return "secret-tool"
+    return "file"
+
+
+def store(name: str = ""):
+    n = name or backend_name()
+    if n == "keychain":
+        return KeychainStore()
+    if n == "secret-tool":
+        return SecretToolStore()
+    return FileStore()
+
+
+# ── the env map (non-secret) ──────────────────────────────────────────────
+
+def env_map_path() -> Path:
+    return _root() / "config" / "secret-env.json"
+
+
+def env_map() -> dict:
+    """{"store": backend, "env": {ENV_NAME: secret name}}."""
+    try:
+        data = json.loads(env_map_path().read_text(encoding="utf-8"))
+        if isinstance(data, dict) and isinstance(data.get("env"), dict):
+            return data
+    except Exception:
+        pass
+    return {"store": "", "env": {}}
+
+
+def _write_env_map(data: dict) -> None:
+    p = env_map_path()
+    p.parent.mkdir(parents=True, exist_ok=True)
+    fd = os.open(str(p), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    with os.fdopen(fd, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=1, sort_keys=True)
+
+
+def put(name: str, value: str, *, env: str = "", backend=None) -> str:
+    """Store a secret; with ``env``, also export it to that variable now and
+    at every start. Returns where it was stored (for the ✓ line)."""
+    st = backend or store(env_map().get("store") or "")
+    st.set(name, value)
+    if env:
+        m = env_map()
+        m["store"] = st.name
+        m["env"][env] = name
+        _write_env_map(m)
+        os.environ[env] = value
+    return st.where
+
+
+def link_env(name: str, env: str, backend=None) -> bool:
+    """Export an already stored secret to ``env`` (now and at every start).
+    False when the secret isn't stored."""
+    st = backend or store(env_map().get("store") or "")
+    value = get(name, backend=st)
+    if not value:
+        return False
+    m = env_map()
+    m["store"] = st.name
+    m["env"][env] = name
+    _write_env_map(m)
+    os.environ[env] = value
+    return True
+
+
+def unlink_env(env: str) -> None:
+    """Stop exporting ``env`` (the secret itself stays stored)."""
+    m = env_map()
+    if m["env"].pop(env, None) is not None:
+        _write_env_map(m)
+    os.environ.pop(env, None)
+
+
+def get(name: str, backend=None) -> str:
+    st = backend or store(env_map().get("store") or "")
+    try:
+        return st.get(name)
+    except SecretStoreError:
+        return ""
+
+
+def remove(name: str, backend=None) -> bool:
+    st = backend or store(env_map().get("store") or "")
+    m = env_map()
+    gone = [k for k, v in m["env"].items() if v == name]
+    for k in gone:
+        m["env"].pop(k, None)
+    if gone:
+        _write_env_map(m)
+    try:
+        return st.delete(name)
+    except SecretStoreError:
+        return False
+
+
+def export_env(backend=None) -> list:
+    """Copy the mapped secrets into os.environ (setdefault). Returns the
+    variable names exported. Never raises; a missing map costs nothing."""
+    try:
+        m = env_map()
+        if not m["env"]:
+            return []
+        st = backend or store(m.get("store") or "")
+        done = []
+        for env_name, secret_name in m["env"].items():
+            if os.environ.get(env_name):
+                continue
+            try:
+                v = st.get(secret_name)
+            except SecretStoreError:
+                v = ""
+            if v:
+                os.environ[env_name] = v
+                done.append(env_name)
+        return done
+    except Exception as e:  # noqa: BLE001 -- start-up must never fail here
+        log.warning("secret store: couldn't export secrets (%s)", type(e).__name__)
+        return []

hyperclaw/teamsetup/selfcheck.py

@@ -0,0 +1,295 @@
+"""``hyperclaw self-check``: a report a team member can send Vaughn.
+
+It writes ``$HYPERCLAW_ROOT/reports/self-check-<UTC time>.txt`` (0600) with:
+version, OS, install method and mode, the onboarding steps, one live check
+per piece (the same "✓ Connected" lines as onboarding), the background
+service state, which secrets are stored (names only), and the recent
+warnings and errors from the logs.
+
+Removed from the whole report, whatever their source:
+
+- keys and tokens: every value held in the secret store, the digest token
+  store, the Google token file and the secret-looking ``.env`` entries is
+  replaced literally; known shapes (Anthropic/OpenAI/xAI keys, Telegram bot
+  tokens, Slack, GitLab, Google OAuth, GitHub, AWS, JWT, bearer, password=)
+  are replaced by pattern as a second line of defence;
+- email addresses;
+- message content: logs keep warnings and errors only; text after message
+  markers (``User:``, ``text=``, ...) and anything in quotes is dropped;
+  conversation files are never read;
+- file paths with the user name (home is shown as ``~``, the account name as
+  ``<user>``) and long numbers such as chat ids.
+
+``tests/unit/teamsetup/test_selfcheck.py`` plants a fake secret of every kind and
+proves none reaches the report.
+"""
+from __future__ import annotations
+
+import datetime as _dt
+import getpass
+import json
+import os
+import platform
+import re
+import sys
+from pathlib import Path
+from typing import Callable, Iterable, Optional
+
+LOG_FILES = ("logs/service.log", "logs/telegram.err.log", "logs/hyperclaw.log",
+             "logs/telegram.log")
+LOG_TAIL = 400            # lines read from the end of each log
+LOG_KEEP = 60             # warning/error lines kept per log
+_LEVEL = re.compile(r"\b(WARNING|ERROR|CRITICAL)\b")
+
+_PATTERNS = [
+    (re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)",
+                re.S), "[key removed]"),
+    (re.compile(r"\bsk-ant-[A-Za-z0-9_-]{8,}"), "[key removed]"),
+    (re.compile(r"\bsk-(?:proj-)?[A-Za-z0-9_-]{16,}"), "[key removed]"),
+    (re.compile(r"\bxai-[A-Za-z0-9_-]{16,}"), "[key removed]"),
+    (re.compile(r"\brpa_[A-Za-z0-9_-]{8,}"), "[key removed]"),
+    (re.compile(r"\bHCT\d+\.[A-Za-z0-9_.-]{16,}"), "[item removed]"),
+    (re.compile(r"\b\d{5,}:[A-Za-z0-9_-]{30,}"), "[token removed]"),
+    (re.compile(r"\bbot\d{5,}:[A-Za-z0-9_-]{20,}"), "[token removed]"),
+    (re.compile(r"\bxox[abeoprsc]-[A-Za-z0-9-]{8,}"), "[token removed]"),
+    (re.compile(r"\bxoxe[.-][A-Za-z0-9.-]{8,}"), "[token removed]"),
+    (re.compile(r"\bxapp-[A-Za-z0-9-]{8,}"), "[token removed]"),
+    (re.compile(r"\bgl[a-z]{1,8}-[A-Za-z0-9_.-]{16,}"), "[token removed]"),
+    (re.compile(r"\bya29\.[A-Za-z0-9_.-]{8,}"), "[token removed]"),
+    (re.compile(r"\b1//[A-Za-z0-9_-]{8,}"), "[token removed]"),
+    (re.compile(r"\bGOCSPX-[A-Za-z0-9_-]{8,}"), "[secret removed]"),
+    (re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}"), "[token removed]"),
+    (re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b"), "[key removed]"),
+    (re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}"),
+     "[token removed]"),
+    (re.compile(r"(?i)\b(bearer\s+)[A-Za-z0-9._~+/=-]{8,}"), r"\1[token removed]"),
+    (re.compile(r"(?i)\b((?:password|passwd|secret|api[_-]?key|access[_-]?token|"
+                r"refresh[_-]?token|token|key|code|code_verifier)\s*[:=]\s*)"
+                r"(?!\[)(\"[^\"]*\"|'[^']*'|\S+)"), r"\1[removed]"),
+    (re.compile(r"\b([a-z][a-z0-9+.-]*://[^:/\s@]+:)[^@\s]+(@)"), r"\1[removed]\2"),
+    (re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}"), "[email removed]"),
+    # any account's home folder, not only the current one
+    (re.compile(r"(/(?:Users|home))/(?!Shared/)[^/\s]+"), r"\1/<user>"),
+    (re.compile(r"(?<![\w.])-?\d{6,}(?![\w.])"), "[number removed]"),
+]
+# Text after these markers is message content.
+_CONTENT = re.compile(r"(?i)\b(user|assistant|reply|message|text|prompt|content|said|"
+                      r"caption|body|query)\s*[:=]\s*.*$")
+_QUOTED = re.compile(r"\"[^\"]*\"|'[^']*'|“[^”]*”|‘[^’]*’")
+_SECRET_ENV = re.compile(r"(KEY|TOKEN|SECRET|PASSWORD|PASS|DATABASE_URL|DSN|CREDENTIAL)",
+                         re.I)
+
+
+def known_secret_values() -> list:
+    """Every secret value this install holds (to remove literally)."""
+    vals = set()
+    from hyperclaw.teamsetup import secrets
+    try:
+        for name in ("model_key", "hyper_key", "google_client_secret", "telegram_bot_token",
+                     "slack_user_token"):
+            v = secrets.get(name)
+            if v:
+                vals.add(v)
+                if name == "slack_user_token":
+                    try:
+                        b = json.loads(v)
+                        vals.update(str(x) for k, x in b.items()
+                                    if k in ("access_token", "refresh_token") and x)
+                    except ValueError:
+                        pass
+    except Exception:
+        pass
+    try:
+        from hyperclaw.digest import tokens
+        for src in ("gitlab", "slack"):
+            for backend in {tokens.backend_name(), "file"}:
+                try:
+                    v = tokens.store(backend).get(src)
+                except Exception:
+                    v = ""
+                if v:
+                    vals.add(v)
+    except Exception:
+        pass
+    try:
+        from hyperclaw.google_workspace import auth as ga
+        tok = ga.load_token()
+        vals.update(str(tok.get(k)) for k in ("refresh_token", "access_token") if tok.get(k))
+        cp = ga.client_path()
+        if cp.is_file():
+            vals.add(str((json.loads(cp.read_text()).get("installed") or {})
+                         .get("client_secret") or ""))
+    except Exception:
+        pass
+    try:
+        from hyperclaw.brief import store as bstore
+        tok = bstore.read_json(bstore.token_path())
+        vals.update(str(tok.get(k)) for k in ("refresh_token", "access_token") if tok.get(k))
+    except Exception:
+        pass
+    try:
+        from hyperclaw.userconfig import env_read
+        for k, v in env_read().items():
+            if v and _SECRET_ENV.search(k):
+                vals.add(v)
+    except Exception:
+        pass
+    for k, v in os.environ.items():
+        if v and _SECRET_ENV.search(k) and len(v) >= 8:
+            vals.add(v)
+    return sorted((v for v in vals if len(v) >= 6), key=len, reverse=True)
+
+
+class Redactor:
+    def __init__(self, values: Optional[Iterable[str]] = None, home: str = "",
+                 user: str = ""):
+        self.values = list(values if values is not None else known_secret_values())
+        self.home = home or str(Path.home())
+        self.user = user or _user()
+
+    def __call__(self, text: str) -> str:
+        out = str(text or "")
+        for v in self.values:
+            out = out.replace(v, "[secret removed]")
+        if self.home and self.home not in ("/", ""):
+            out = out.replace(self.home, "~")
+        for pat, repl in _PATTERNS:
+            out = pat.sub(repl, out)
+        if self.user and len(self.user) >= 3:
+            out = re.sub(rf"(?<![A-Za-z0-9]){re.escape(self.user)}(?![A-Za-z0-9])", "<user>",
+                         out)
+        return out
+
+    def log_line(self, line: str) -> str:
+        """A log line with message content removed, then redacted."""
+        s = _CONTENT.sub(lambda m: m.group(1) + ": [message removed]", line)
+        s = _QUOTED.sub("[text removed]", s)
+        return self(s)[:300]
+
+
+def _user() -> str:
+    try:
+        return getpass.getuser()
+    except Exception:
+        return os.environ.get("USER", "")
+
+
+def log_section(redact: Redactor) -> list:
+    from hyperclaw.userconfig import root
+    out = []
+    for rel in LOG_FILES:
+        p = root() / rel
+        if not p.exists():
+            continue
+        try:
+            with open(p, "rb") as f:
+                f.seek(0, 2)
+                size = f.tell()
+                f.seek(max(0, size - 256 * 1024))
+                lines = f.read().decode("utf-8", errors="replace").splitlines()[-LOG_TAIL:]
+        except OSError:
+            continue
+        kept = [redact.log_line(x) for x in lines if _LEVEL.search(x)][-LOG_KEEP:]
+        out.append(f"--- {rel}: {len(lines)} recent lines, {len(kept)} warnings/errors shown "
+                   "(message text removed)")
+        out += kept
+    return out or ["(no logs yet)"]
+
+
+def _secret_names() -> list:
+    from hyperclaw.teamsetup import secrets
+    out = []
+    st = secrets.env_map().get("store") or secrets.backend_name()
+    for name in ("model_key", "hyper_key", "google_client_secret", "telegram_bot_token",
+                 "slack_user_token"):
+        try:
+            present = bool(secrets.get(name))
+        except Exception:
+            present = False
+        out.append(f"{name}: {'stored' if present else 'not stored'}")
+    out.append(f"secret store: {st}")
+    return out
+
+
+def build(*, checks_fn: Optional[Callable[[], list]] = None,
+          service_fn: Optional[Callable[[], dict]] = None,
+          redact: Optional[Redactor] = None, now: Optional[_dt.datetime] = None) -> str:
+    from hyperclaw import __version__, distribution
+    from hyperclaw.deployment import mode
+    from hyperclaw.teamsetup import onboard, profile, service
+    redact = redact or Redactor()
+    now = now or _dt.datetime.now(_dt.timezone.utc)
+    try:
+        from hyperclaw.updates.install import detect
+        method = detect().kind
+    except Exception:
+        method = "other"
+    prof = profile.load()
+    lines = [
+        "HyperAgents self-check",
+        f"time (UTC): {now.strftime('%Y-%m-%d %H:%M')}",
+        f"version: {__version__}",
+        f"python: {platform.python_version()}",
+        f"system: {platform.system()} {platform.mac_ver()[0] or platform.release()} "
+        f"({platform.machine()})",
+        f"install: {method}; mode: {mode()}; team profile: "
+        f"{prof.get('name', 'none') if prof else 'none or not verified'}",
+        f"install command: {distribution.install_command(__version__)}"
+        + (f" (this install is the old package; {distribution.migration_line(__version__)})"
+           if distribution.is_legacy_install() else ""),
+        "",
+        "Onboarding steps:",
+    ]
+    st = onboard.load_state()["steps"]
+    for s in onboard.STEPS:
+        info = st.get(s) or {}
+        lines.append(f"  {s}: {info.get('status') or 'not started'}"
+                     + (f" ({info['code']})" if info.get("code") else ""))
+    lines += ["", "Connections:"]
+    try:
+        conn = checks_fn() if checks_fn else onboard.summary_lines()
+    except Exception as e:  # noqa: BLE001
+        conn = [f"(checks couldn't run: {type(e).__name__})"]
+    lines += [f"  {x}" for x in conn]
+    try:
+        svc = service_fn() if service_fn else service.status()
+    except Exception as e:  # noqa: BLE001
+        svc = {"error": type(e).__name__}
+    lines += ["", "Background service: " + ", ".join(f"{k}={v}" for k, v in svc.items()),
+              "scheduler jobs: " + (os.environ.get("HYPERCLAW_SCHEDULER_JOBS", "")
+                                    or "(set in the service only)"),
+              "", "Stored secrets (names only):"]
+    lines += [f"  {x}" for x in _secret_names()]
+    lines += ["", "Recent warnings and errors:"]
+    lines += log_section(redact)
+    lines += ["", "Removed from this report: keys, tokens, email addresses, message text, "
+                  "and your user name in file paths."]
+    return redact("\n".join(lines)) + "\n"
+
+
+def write(text: str, out: Optional[Path] = None) -> Path:
+    from hyperclaw.userconfig import root
+    if out is None:
+        stamp = _dt.datetime.now(_dt.timezone.utc).strftime("%Y%m%d-%H%M%S")
+        out = root() / "reports" / f"self-check-{stamp}.txt"
+    out.parent.mkdir(parents=True, exist_ok=True)
+    fd = os.open(str(out), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    with os.fdopen(fd, "w", encoding="utf-8") as f:
+        f.write(text)
+    return out
+
+
+def main(out: Optional[str] = None, *, echo: Callable[[str], None] = print) -> Path:
+    from hyperclaw.userconfig import load_env_into_process
+    load_env_into_process()
+    text = build()
+    p = write(text, Path(out).expanduser() if out else None)
+    shown = str(p).replace(str(Path.home()), "~")
+    echo(f"Self-check written to {shown}")
+    echo("It has no keys, tokens, email addresses or messages in it. Send that file to Vaughn "
+         "(attach it in a direct message).")
+    return p
+
+
+if __name__ == "__main__":
+    main(sys.argv[1] if len(sys.argv) > 1 else None)