secretref 0.1.0__py3-none-any.whl

secretref/__init__.py

@@ -0,0 +1,99 @@
+"""Config files that name their secrets instead of containing them.
+
+An `.env` file works until the config needs a list, a nested section, or a
+number that is really a number. JSON and YAML handle all three — but then the
+secrets are sitting in a file next to the code, which is how they end up in a
+repository.
+
+So a secret in a config file here is a *reference*:
+
+    {
+      "providers": [
+        {"name": "openai",    "api_key": {"env": "OPENAI_API_KEY"}},
+        {"name": "anthropic", "api_key": {"file": "/run/secrets/anthropic"}},
+        {"name": "internal",  "api_key": {"command": "op read op://vault/key"}}
+      ],
+      "timeout_seconds": 30,
+      "languages": ["en", "fil"]
+    }
+
+The file is committable, because it says where each secret lives rather than
+what it is. Resolution is lazy, so a config naming twenty secrets reads none
+of them at load — which is what lets one file serve development and
+production.
+
+    from secretref import load
+
+    config = load("config.json")
+    key = config.secret("providers.0.api_key").resolve()
+
+And the part that makes it worth a library rather than a helper function:
+
+    from secretref import audit
+
+    for finding in audit(config):
+        print(finding)
+    # error: providers.0.api_key: a literal secret inside a repository
+    #   try: move it to an environment variable and reference it
+
+`audit` reports a literal secret as an **error** when it finds a repository
+above the file, because such a value is one commit from a remote. Outside a
+repository the same value is a warning. That check is the reason this exists:
+the failure it prevents is not a crash but a leak, which no test catches.
+
+No dependencies. A library that reads your secrets should be auditable in an
+afternoon.
+"""
+
+from secretref.auditors import audit, repository_root
+from secretref.cache import ResolutionCache
+from secretref.constants import REDACTED
+from secretref.exceptions import (
+    CommandNotAllowedError,
+    ConfigError,
+    FileNotAllowedError,
+    ResolutionError,
+    SecretrefError,
+    VariableNotAllowedError,
+)
+from secretref.models import (
+    Config,
+    Finding,
+    SecretRef,
+    Severity,
+    Source,
+)
+from secretref.policies import CommandPolicy, EnvironmentPolicy, FilePolicy
+
+__all__ = [
+    "REDACTED",
+    "CommandNotAllowedError",
+    "CommandPolicy",
+    "Config",
+    "ConfigError",
+    "EnvironmentPolicy",
+    "FileNotAllowedError",
+    "FilePolicy",
+    "Finding",
+    "ResolutionCache",
+    "ResolutionError",
+    "SecretRef",
+    "SecretrefError",
+    "Severity",
+    "Source",
+    "VariableNotAllowedError",
+    "__version__",
+    "audit",
+    "find_config",
+    "load",
+    "loads",
+    "repository_root",
+]
+__version__ = "0.1.0"
+
+# The public API stays functions, because `load("config.json")` is what a
+# caller wants to write. Config holds the implementation, so the pieces live
+# with the type they build rather than in a module of their own.
+load = Config.load
+loads = Config.loads
+find_config = Config.find

secretref/__main__.py

@@ -0,0 +1,11 @@
+"""Lets the tool run as `python -m secretref`, not only as `secretref`.
+
+Useful when the package is installed somewhere the console script is not on
+PATH, which is common in a container.
+"""
+
+from __future__ import annotations
+
+from secretref.cli import run
+
+run()

secretref/auditors/__init__.py

@@ -0,0 +1,56 @@
+"""Checking a config file for secrets it should not be holding.
+
+The failure this prevents is not a crash. A hardcoded key works perfectly —
+right up until the repository is public. Nothing breaks, no test fails, and
+the problem is discovered by someone else.
+
+So the central question is not "is this value valid" but "could this value
+already have been published". That is why the severity depends on whether
+there is a repository above the file:
+
+    ERROR    a literal secret inside a working copy. One commit from a
+             remote, and git history keeps it after the fix.
+    WARNING  a literal secret outside one. Careless, but contained.
+    NOTE     a reference that currently resolves to nothing. Usually a typo
+             in a variable name, occasionally a deliberately absent option.
+
+What this is not: a repository scanner. It reads one config file and knows
+nothing about history, other files, or anything already committed. For that,
+gitleaks and detect-secrets answer a different question and answer it better.
+"""
+
+from __future__ import annotations
+
+from secretref.auditors.auditor import Auditor
+from secretref.auditors.placeholder import Placeholder
+from secretref.auditors.secret_key import SecretKey
+from secretref.models.config import Config
+from secretref.models.finding import Finding
+from secretref.utils.repository import Repository
+
+__all__ = [
+    "Auditor",
+    "Placeholder",
+    "SecretKey",
+    "audit",
+    "repository_root",
+]
+
+
+def audit(config: Config, *, resolve: bool = False) -> list[Finding]:
+    """Every problem in a config file, most severe first.
+
+    Does not resolve by default, which is a security boundary rather than a
+    performance one: resolving a command reference runs it, and auditing a
+    config you did not write is the main use for this. A CI check on a pull
+    request would otherwise have been remote code execution.
+
+    Pass `resolve=True` for your own config at startup, where confirming
+    that a secret is actually reachable is worth more than the caution.
+    """
+    return Auditor(config, resolve=resolve).run()
+
+
+# `repository_root` was the audit's own helper before the VCS walk was
+# separated out. Kept as a name here because it is public API.
+repository_root = Repository.root

secretref/auditors/auditor.py

@@ -0,0 +1,185 @@
+"""Running the checks over one config file."""
+
+from __future__ import annotations
+
+from secretref.auditors.known_format import KnownFormat
+from secretref.auditors.placeholder import Placeholder
+from secretref.auditors.secret_key import SecretKey
+from secretref.models.config import Config
+from secretref.models.finding import Finding, Severity
+from secretref.models.source import Source
+from secretref.utils.repository import Repository
+from secretref.utils.tree import Tree
+
+
+class Auditor:
+    """Carries the config and the repository verdict through the checks.
+
+    A class because every check needs both, and threading them through
+    arguments made the functions longer than the checks they performed.
+    """
+
+    def __init__(self, config: Config, *, resolve: bool = False) -> None:
+        """Audit a config, optionally checking that its secrets resolve.
+
+        `resolve` is off by default, and that default is a security
+        boundary rather than a performance one. Resolving a command
+        reference *runs* it, so an audit that resolved would execute
+        whatever the config asked for — and auditing a config you did not
+        write is the main use for this. A CI check on a pull request would
+        have been remote code execution.
+
+        Turn it on for your own config at startup, where knowing a secret
+        is actually reachable is worth more than the caution.
+        """
+        self._config = config
+        self._resolve = resolve
+        self._repository = Repository.root(config.path) if config.path else None
+
+    def run(self) -> list[Finding]:
+        """Run every check and order the results by severity."""
+        findings = [*self._reference_findings(), *self._literal_findings()]
+        order = {Severity.ERROR: 0, Severity.WARNING: 1, Severity.NOTE: 2}
+        findings.sort(key=lambda finding: (order[finding.severity], finding.path))
+        return findings
+
+    def _may_check(self, reference) -> bool:
+        """Whether this reference can be verified without doing anything.
+
+        A reference with side effects is only resolved when the caller has
+        said the config is theirs.
+        """
+        return self._resolve or not reference.has_side_effects()
+
+    # -- checks -----------------------------------------------------------
+
+    def _reference_findings(self) -> list[Finding]:
+        """Problems with values that are already references."""
+        findings = []
+        for path, reference in self._config.secrets().items():
+            if reference.source is Source.LITERAL:
+                findings.append(self._literal_finding(path, reference.locator))
+            elif broken := reference.malformed():
+                # Checked first: a reference that cannot be parsed will not
+                # work whatever the policy says, and naming the syntax error
+                # is more useful than reporting it as unverified.
+                findings.append(
+                    Finding(
+                        severity=Severity.ERROR,
+                        path=path,
+                        message=f"{reference.described()} is {broken}",
+                        remedy="fix the reference in the config",
+                    )
+                )
+            elif refused := reference.refusal():
+                # A policy refusal is a config error, not a deployment one:
+                # the config asks for something it is not permitted to do,
+                # and no amount of setting variables will change that. An
+                # error whatever the repository, because the application
+                # will not start.
+                findings.append(
+                    Finding(
+                        severity=Severity.ERROR,
+                        path=path,
+                        message=f"refused by policy — {refused.rstrip('.')}",
+                        remedy="widen the policy, or point the reference somewhere it permits",
+                    )
+                )
+            elif not self._may_check(reference):
+                findings.append(
+                    Finding(
+                        severity=Severity.NOTE,
+                        path=path,
+                        message=f"{reference.described()} was not verified",
+                        remedy=(
+                            "resolving it would run a command, which an audit "
+                            "does not do. Pass resolve=True if this config is "
+                            "yours and you want it checked"
+                        ),
+                    )
+                )
+            elif not reference.is_resolvable():
+                findings.append(
+                    Finding(
+                        severity=Severity.NOTE,
+                        path=path,
+                        message=f"{reference.described()} resolves to nothing",
+                        remedy='set it, or add "required": false if it is optional',
+                    )
+                )
+        return findings
+
+    def _literal_findings(self) -> list[Finding]:
+        """Plain strings that hold a secret.
+
+        Two independent checks, because each catches what the other misses.
+        A published credential format is recognized from the value, whatever
+        the key is called — which is the case key names cannot cover, since
+        nothing stops a live key sitting under `tracking.ref`. A
+        secret-looking key name catches the rest: an internal password has no
+        published shape to match.
+        """
+        findings = []
+        for path, value in Tree.strings(self._config.data):
+            if identified := KnownFormat.identify(value):
+                findings.append(self._known_format_finding(path, identified))
+            elif self._looks_like_a_secret(path, value):
+                findings.append(self._literal_finding(path, value))
+        return findings
+
+    @staticmethod
+    def _looks_like_a_secret(path: str, value: str) -> bool:
+        """Whether a key name suggests a credential and the value could be one."""
+        if not SecretKey.matches(path):
+            return False
+        return not (Placeholder.too_short_to_judge(value) or Placeholder.matches(value))
+
+    def _known_format_finding(self, path: str, identified: str) -> Finding:
+        """One credential matching a published format.
+
+        Always an error, and always regardless of the key name or whether a
+        repository was found. A value matching an issuer's published shape is
+        a real credential — there is no reading of it as a placeholder, and
+        the right response is to rotate it rather than to move it.
+        """
+        inside = (
+            " inside a repository, so git history keeps it"
+            if self._repository is not None
+            else " in a config file"
+        )
+        return Finding(
+            severity=Severity.ERROR,
+            path=path,
+            message=f"{identified}{inside}",
+            remedy=(
+                "rotate this credential, then reference it instead of storing "
+                'it: {"env": "YOUR_VAR"}'
+            ),
+        )
+
+    def _literal_finding(self, path: str, value: str) -> Finding:
+        """One literal, with a severity that depends on the repository."""
+        if Placeholder.is_noise(value):
+            return Finding(
+                severity=Severity.NOTE,
+                path=path,
+                message="a literal value, which looks like a placeholder",
+                remedy="replace it with a reference before deploying",
+            )
+        if self._repository is not None:
+            return Finding(
+                severity=Severity.ERROR,
+                path=path,
+                message="a literal secret inside a repository",
+                remedy=(
+                    "move it to an environment variable and reference it: "
+                    '{"env": "YOUR_VAR"} — and rotate it, since git history '
+                    "keeps the old value"
+                ),
+            )
+        return Finding(
+            severity=Severity.WARNING,
+            path=path,
+            message="a literal secret in a config file",
+            remedy='reference it instead of storing it: {"env": "YOUR_VAR"}',
+        )

secretref/auditors/known_format.py

@@ -0,0 +1,42 @@
+"""Whether a value is a credential whose issuer publishes its shape."""
+
+from __future__ import annotations
+
+import re
+from typing import ClassVar
+
+from secretref.patterns import KNOWN_SECRET_FORMATS
+
+
+class KnownFormat:
+    """Whether a value matches a published credential format.
+
+    The one kind of secret that can be identified from the value alone. Every
+    other check in this package starts from the key name, which is the thing
+    an audit cannot depend on: a live Stripe key under `payments.api_key` is
+    found either way, while the same key under `tracking.ref` is found only
+    here.
+
+    A match is reported as an error regardless of the key, and regardless of
+    whether the config is inside a repository — a key matching a published
+    format is a real credential, not a placeholder someone typed.
+    """
+
+    FORMATS: ClassVar[tuple[tuple[str, re.Pattern[str]], ...]] = KNOWN_SECRET_FORMATS
+
+    @classmethod
+    def identify(cls, value: str) -> str:
+        """Name the credential format this value matches, or return empty.
+
+        The first match wins. The patterns are specific enough that two
+        matching the same value would mean one of them is wrong.
+        """
+        for name, pattern in cls.FORMATS:
+            if pattern.search(value):
+                return name
+        return ""
+
+    @classmethod
+    def matches(cls, value: str) -> bool:
+        """Whether this value matches any published credential format."""
+        return bool(cls.identify(value))

secretref/auditors/placeholder.py

@@ -0,0 +1,41 @@
+"""Whether a value is filler rather than a real secret."""
+
+from __future__ import annotations
+
+import re
+from typing import ClassVar
+
+from secretref.constants import MIN_SECRET_LENGTH
+from secretref.patterns import PLACEHOLDERS
+
+
+class Placeholder:
+    """Whether a value is filler rather than a real secret."""
+
+    PATTERNS: ClassVar[tuple[re.Pattern[str], ...]] = PLACEHOLDERS
+
+    @classmethod
+    def matches(cls, value: str) -> bool:
+        """Whether a value is obviously filler.
+
+        Generous on purpose. A false negative here costs one spurious error,
+        which a reader dismisses. A false positive reports a real secret as
+        filler and hides it, which is the failure that matters.
+        """
+        candidate = value.strip()
+        return any(pattern.search(candidate) for pattern in cls.PATTERNS)
+
+    @classmethod
+    def too_short_to_judge(cls, value: str) -> bool:
+        """Whether a value is too short to be worth reporting.
+
+        A short string under a secret-looking key is more often a flag or an
+        abbreviation than a credential, and reporting those is how an audit
+        trains people to ignore it.
+        """
+        return len(value.strip()) < MIN_SECRET_LENGTH
+
+    @classmethod
+    def is_noise(cls, value: str) -> bool:
+        """Whether a value should be a note rather than an error."""
+        return cls.matches(value) or cls.too_short_to_judge(value)

secretref/auditors/secret_key.py

@@ -0,0 +1,25 @@
+"""Whether a config path names a credential."""
+
+from __future__ import annotations
+
+import re
+from typing import ClassVar
+
+from secretref.patterns import SECRET_KEY
+
+
+class SecretKey:
+    """Whether a config path names a credential."""
+
+    PATTERN: ClassVar[re.Pattern[str]] = SECRET_KEY
+
+    @classmethod
+    def matches(cls, dotted: str) -> bool:
+        """Whether a path's final segment names a credential.
+
+        Only the final segment, so "providers.0.api_key" is judged on
+        "api_key". A parent path saying "providers" tells you nothing about
+        whether this particular leaf holds a secret.
+        """
+        last = dotted.rsplit(".", 1)[-1]
+        return bool(cls.PATTERN.search(last))

secretref/cache.py

@@ -0,0 +1,130 @@
+"""One shared store of already-resolved values.
+
+A thin wrapper over `utils.TtlStore`, which holds the mechanics. What is left
+here is the part that is actually about secrets:
+
+    the key is (source, locator), so an env var and a file path that happen
+    to share a name do not collide
+
+    an empty resolution is never stored, because caching "this resolved to
+    nothing" would hide a secret that appears later — the opposite of helpful
+    during a deployment
+
+    it is a singleton, because two references to the same variable should not
+    each spawn their own subprocess, and a per-reference cache caches nothing
+
+That last point is the only place in this library where a singleton is the
+right answer, and the contrast is the useful part. `Tree`, `Repository` and
+the parsers are classmethods on classes nobody constructs — stronger than a
+singleton, since there is no instance to mutate. `Config`, `SecretRef` and
+`Finding` must not be singletons at all, because more than one is the normal
+case. Here the shared state *is* the feature.
+
+Off by default, because caching changes two things a caller should decide:
+
+    a secret stays in memory after the only read that needed it
+
+    a rotated secret keeps serving the old value until the entry expires,
+    which is the usual reason to set a ttl rather than leave it unbounded
+"""
+
+from __future__ import annotations
+
+from threading import Lock
+from typing import ClassVar
+
+from secretref.utils.ttl_store import TtlStore
+
+
+class ResolutionCache:
+    """A process-wide store of resolved secrets, keyed by reference."""
+
+    _instance: ClassVar[ResolutionCache | None] = None
+    _singleton_lock: ClassVar[Lock] = Lock()
+
+    def __new__(cls) -> ResolutionCache:
+        """Return the one instance, creating it on first use.
+
+        Double-checked under a lock, because two threads resolving at startup
+        would otherwise each build a store and one would win — leaving the
+        other thread's writes in an object nothing else can see.
+        """
+        if cls._instance is None:
+            with cls._singleton_lock:
+                if cls._instance is None:
+                    instance = super().__new__(cls)
+                    instance._initialize()
+                    cls._instance = instance
+        return cls._instance
+
+    def _initialize(self) -> None:
+        """Set up the store. Called once, by __new__."""
+        self._store: TtlStore[tuple[str, str], str] = TtlStore()
+        self._enabled = False
+
+    # -- configuration -----------------------------------------------------
+
+    def enable(self, ttl_seconds: float | None = None) -> None:
+        """Start caching, optionally expiring entries after `ttl_seconds`.
+
+        A ttl is how a long-running process picks up a rotated secret without
+        restarting. Without one an entry lives until the cache is cleared,
+        which suits a short-lived process and not much else.
+        """
+        self._enabled = True
+        self._store.ttl = ttl_seconds
+
+    def disable(self) -> None:
+        """Stop caching and drop everything held."""
+        self._enabled = False
+        self._store.clear()
+
+    def clear(self) -> None:
+        """Drop everything held, without changing whether caching is on.
+
+        What to call after rotating a secret, and what a test fixture calls
+        so one test cannot see another's values.
+        """
+        self._store.clear()
+
+    @property
+    def enabled(self) -> bool:
+        """Whether resolved values are being kept."""
+        return self._enabled
+
+    # -- use ---------------------------------------------------------------
+
+    def get(self, source: str, locator: str) -> str | None:
+        """Return a cached value for this reference, or None.
+
+        None means "resolve it" rather than "it resolved to nothing" — an
+        empty resolution is never stored, so the two cannot be confused.
+        """
+        if not self._enabled:
+            return None
+        return self._store.get((source, locator))
+
+    def put(self, source: str, locator: str, value: str) -> None:
+        """Store a resolved value.
+
+        An empty value is not stored. Caching "this resolved to nothing"
+        would make a secret that appears later invisible until the cache is
+        cleared, which is the opposite of helpful during a deployment.
+        """
+        if not self._enabled or not value:
+            return
+        self._store.put((source, locator), value)
+
+    @property
+    def hits(self) -> int:
+        """Lookups served from the store."""
+        return self._store.hits
+
+    @property
+    def misses(self) -> int:
+        """Lookups that had to resolve."""
+        return self._store.misses
+
+    def stats(self) -> dict[str, int | bool | float | None]:
+        """Counters for logging. Contains no values and no locators."""
+        return {"enabled": self._enabled, **self._store.counters()}