worktree-shield 0.1.0__py3-none-any.whl

worktree_shield/collectors/compose.py

@@ -0,0 +1,315 @@
+"""Docker Compose: project names, container names, published ports and mounts.
+
+Files are chosen as compose-go chooses them: the first of :data:`COMPOSE_FILES` in a
+directory, plus the first of :data:`OVERRIDE_FILES` next to it. They are loaded with
+``yaml.SafeLoader`` (pure Python: deep nesting raises ``RecursionError`` instead of
+overflowing the C stack), which builds only plain data. Nothing is passed to Docker.
+
+The project name follows compose-go's precedence: ``COMPOSE_PROJECT_NAME`` (from
+``.env``, when env files are read), then the last non-empty ``name:``, then the name of
+the directory, normalised. It prefixes containers, networks and named volumes, so two
+worktrees with the same project name share all of them; a ``name:`` in the file, or a
+compose file in a subdirectory such as ``infra/``, gives every worktree the same one.
+
+Overrides are merged the simple way: a service's ``container_name`` is the last one
+set; ports and volumes of all files are listed together.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path, PurePosixPath
+from typing import Any
+
+import yaml
+
+from worktree_shield.collectors.base import (
+    Collected,
+    Scope,
+    is_absolute_path,
+    path_key,
+    port_key,
+)
+from worktree_shield.models import (
+    Policy,
+    ResourceClaim,
+    ResourceKind,
+    SkippedFile,
+    SourceRef,
+    UnreadableValue,
+)
+from worktree_shield.parsers.compose import parse_mount, published_ports
+from worktree_shield.parsers.dotenv import parse_dotenv
+from worktree_shield.parsers.interpolate import Interpolated, interpolate
+
+COMPOSE_FILES = ("compose.yaml", "compose.yml", "docker-compose.yml", "docker-compose.yaml")
+OVERRIDE_FILES = (
+    "compose.override.yml",
+    "compose.override.yaml",
+    "docker-compose.override.yml",
+    "docker-compose.override.yaml",
+)
+_PROJECT_CHARS = re.compile(r"[a-z0-9_-]")
+
+
+def normalize_project_name(name: str) -> str:
+    """compose-go's NormalizeProjectName: lower case, ``[a-z0-9_-]`` only, no leading
+    ``_`` or ``-``."""
+    return "".join(_PROJECT_CHARS.findall(name.lower())).lstrip("_-")
+
+
+@dataclass
+class _Document:
+    relative: PurePosixPath
+    node: yaml.Node
+    data: Any
+
+
+@dataclass
+class _Service:
+    container_name: tuple[_Document, Any] | None = None
+    ports: list[tuple[_Document, int, Any]] = field(default_factory=list)
+    volumes: list[tuple[_Document, int, Any]] = field(default_factory=list)
+
+
+def collect_compose(scope: Scope, directory: PurePosixPath, policy: Policy, out: Collected) -> None:
+    """The Compose resources of the project in ``directory``, if it has one."""
+    main = _first(scope, directory, COMPOSE_FILES)
+    if main is None:
+        return
+    documents = _load(scope, directory / main, out)
+    if documents is None:
+        return  # unreadable (recorded as skipped): even the project name is unknown
+    override = _first(scope, directory, OVERRIDE_FILES)
+    if override is not None:
+        documents += _load(scope, directory / override, out) or []
+    variables = _env_variables(scope, directory, out) if policy.include_env_files else {}
+    project, source, detail = _project(scope, directory, main, documents, variables)
+    out.claims.append(
+        ResourceClaim(
+            ResourceKind.COMPOSE_PROJECT,
+            f"compose-project://{project}",
+            scope.worktree,
+            source,
+            detail,
+        )
+    )
+    _Project(scope, project, variables, documents, out).collect()
+
+
+def _first(scope: Scope, directory: PurePosixPath, names: tuple[str, ...]) -> str | None:
+    folder = Path(scope.root, *directory.parts)
+    return next((name for name in names if (folder / name).is_file()), None)
+
+
+def _load(scope: Scope, relative: PurePosixPath, out: Collected) -> list[_Document] | None:
+    """Every YAML document of a file, as nodes (for lines) and plain data; ``None``
+    when the file could not be read (recorded in ``out.skipped``)."""
+    text = scope.read(relative, out)
+    if text is None:
+        return None
+    documents: list[_Document] = []
+    loader = yaml.SafeLoader(text)
+    try:
+        while loader.check_node():  # also consumes the stream start get_node() expects
+            node = loader.get_node()
+            if node is None:  # pragma: no cover - check_node() promised a document
+                break
+            documents.append(_Document(relative, node, loader.construct_document(node)))
+    except yaml.YAMLError:
+        out.skipped.append(SkippedFile(scope.source(relative), "is not valid YAML"))
+        return None
+    except RecursionError:
+        out.skipped.append(SkippedFile(scope.source(relative), "is nested too deeply"))
+        return None
+    finally:
+        loader.dispose()
+    return documents
+
+
+def _env_variables(scope: Scope, directory: PurePosixPath, out: Collected) -> dict[str, str]:
+    """The variables of ``.env`` next to the compose file, which Compose reads for
+    interpolation; references to earlier variables are resolved as Compose does."""
+    text = scope.read(directory / ".env", out)
+    variables: dict[str, str] = {}
+    for entry in parse_dotenv(text or ""):
+        if not entry.reference:
+            variables[entry.key] = entry.value
+            continue
+        resolved = interpolate(entry.value, variables)
+        if resolved.value is not None:
+            variables[entry.key] = resolved.value
+    return variables
+
+
+def _project(
+    scope: Scope,
+    directory: PurePosixPath,
+    first_file: str,
+    documents: list[_Document],
+    variables: dict[str, str],
+) -> tuple[str, SourceRef, str]:
+    """The project name, where it comes from and how to describe it."""
+    from_env = variables.get("COMPOSE_PROJECT_NAME", "")
+    if from_env:
+        source = scope.source(directory / ".env")
+        return normalize_project_name(from_env), source, f"COMPOSE_PROJECT_NAME={from_env} (.env)"
+    named: tuple[str, _Document] | None = None
+    for document in documents:
+        if isinstance(document.data, dict) and isinstance(document.data.get("name"), str):
+            value = interpolate(document.data["name"], variables).value
+            if value:
+                named = (value, document)
+    if named is not None:
+        value, document = named
+        source = scope.source(document.relative, _line(document.node, "name"))
+        return normalize_project_name(value), source, f"name: {value}"
+    folder = Path(scope.root, *directory.parts).name
+    return (
+        normalize_project_name(folder),
+        scope.source(directory / first_file),
+        f"directory name {folder}",
+    )
+
+
+class _Project:
+    def __init__(
+        self,
+        scope: Scope,
+        project: str,
+        variables: dict[str, str],
+        documents: list[_Document],
+        out: Collected,
+    ) -> None:
+        self.scope, self.project, self.variables = scope, project, variables
+        self.documents, self.out = documents, out
+
+    def collect(self) -> None:
+        services: dict[str, _Service] = {}
+        volumes: dict[str, Any] = {}
+        for document in self.documents:
+            data = document.data if isinstance(document.data, dict) else {}
+            if isinstance(data.get("volumes"), dict):
+                volumes.update(data["volumes"])
+            entries = data.get("services")
+            if not isinstance(entries, dict):
+                continue
+            for name, service in entries.items():
+                if not isinstance(name, str) or not isinstance(service, dict):
+                    continue
+                merged = services.setdefault(name, _Service())
+                if "container_name" in service:
+                    merged.container_name = (document, service["container_name"])
+                for key, target in (("ports", merged.ports), ("volumes", merged.volumes)):
+                    if isinstance(service.get(key), list):
+                        target.extend((document, i, e) for i, e in enumerate(service[key]))
+        for name, service in services.items():
+            if service.container_name is not None:
+                self.container_name(name, *service.container_name)
+            for document, index, entry in service.ports:
+                self.port(name, document, index, entry)
+            for document, index, entry in service.volumes:
+                self.mount(name, document, index, entry, volumes)
+
+    def claim(self, kind: ResourceKind, key: str, source: SourceRef, detail: str) -> None:
+        self.out.claims.append(ResourceClaim(kind, key, self.scope.worktree, source, detail))
+
+    def unreadable(self, kind: ResourceKind, name: str, source: SourceRef, reason: str) -> None:
+        self.out.unreadable.append(UnreadableValue(kind, source, name, reason))
+
+    def resolve(self, value: Any) -> tuple[Any, str | None]:
+        """The value with its variables filled in, or ``None`` and the reason why not.
+        Strings inside a long-syntax mapping are filled in too."""
+        if isinstance(value, str):
+            result = interpolate(value, self.variables)
+            return (result.value, None) if result.value is not None else (None, _why(result))
+        if isinstance(value, dict):
+            resolved: dict[Any, Any] = {}
+            for key, item in value.items():
+                item_value, reason = self.resolve(item)
+                if reason is not None:
+                    return None, reason
+                resolved[key] = item_value
+            return resolved, None
+        return value, None
+
+    def source(self, document: _Document, *path: str | int) -> SourceRef:
+        return self.scope.source(document.relative, _line(document.node, "services", *path))
+
+    def container_name(self, service: str, document: _Document, value: Any) -> None:
+        source = self.source(document, service, "container_name")
+        label = f"services.{service}.container_name"
+        name, reason = self.resolve(value)
+        if reason is not None:
+            self.unreadable(ResourceKind.CONTAINER_NAME, label, source, reason)
+        elif isinstance(name, str) and name:
+            key = f"container://{name}"
+            self.claim(ResourceKind.CONTAINER_NAME, key, source, f"{label}: {name}")
+
+    def port(self, service: str, document: _Document, index: int, entry: Any) -> None:
+        source = self.source(document, service, "ports", index)
+        label = f"services.{service}.ports"
+        value, reason = self.resolve(entry)
+        ports = published_ports(value) if reason is None else None
+        if ports is None:
+            why = reason or "is not a valid port mapping"
+            self.unreadable(ResourceKind.LISTEN_PORT, f"{label}[{index}]", source, why)
+            return
+        shown = f"published {value.get('published')}" if isinstance(value, dict) else str(value)
+        for port in ports:
+            key = port_key(port.host_ip, port.port, protocol=port.protocol)
+            self.claim(ResourceKind.LISTEN_PORT, key, source, f"{label}: {shown}")
+
+    def mount(
+        self, service: str, document: _Document, index: int, entry: Any, volumes: dict[str, Any]
+    ) -> None:
+        source = self.source(document, service, "volumes", index)
+        label = f"services.{service}.volumes"
+        value, reason = self.resolve(entry)
+        if reason is not None:  # ${DATA_DIR}:/data: a host path nobody can tell
+            self.unreadable(ResourceKind.ABSOLUTE_PATH, f"{label}[{index}]", source, reason)
+            return
+        mount = parse_mount(value)
+        if mount is None or mount.read_only or mount.source is None:
+            return  # read-only sharing is no collision; anonymous volumes are per container
+        if mount.kind == "bind":
+            if mount.source.startswith("~") or is_absolute_path(mount.source):
+                key = path_key(mount.source)
+                self.claim(ResourceKind.ABSOLUTE_PATH, key, source, f"{label}: {mount.source}")
+            return
+        if mount.kind != "volume":
+            return
+        settings = volumes.get(mount.source)
+        settings = settings if isinstance(settings, dict) else {}
+        fixed = settings.get("name")
+        external = settings.get("external") is True
+        if isinstance(fixed, str) and fixed:
+            name = interpolate(fixed, self.variables).value or fixed
+        elif external:
+            name = mount.source
+        else:
+            name = f"{self.project}_{mount.source}"
+        detail = f"{label}: {mount.source}" + (" (external)" if external else "")
+        self.claim(ResourceKind.VOLUME, f"volume://{name}", source, detail)
+
+
+def _why(result: Interpolated) -> str:
+    if result.missing:
+        return f"depends on {', '.join(result.missing)}, which no file that was read sets"
+    return "has an invalid ${...} reference"
+
+
+def _line(node: yaml.Node, *path: str | int) -> int | None:
+    """The line of the value at ``path`` in a node tree, or ``None`` if not found."""
+    for step in path:
+        found: yaml.Node | None = None
+        if isinstance(node, yaml.MappingNode) and isinstance(step, str):
+            pairs = ((k, v) for k, v in node.value if isinstance(k, yaml.ScalarNode))
+            found = next((v for k, v in pairs if k.value == step), None)
+        elif isinstance(node, yaml.SequenceNode) and isinstance(step, int):
+            found = node.value[step] if step < len(node.value) else None
+        if found is None:
+            return None  # reached through a merge key or an odd shape
+        node = found
+    return node.start_mark.line + 1

worktree_shield/collectors/envfiles.py

@@ -0,0 +1,130 @@
+"""Ports, databases and absolute paths declared in ``.env`` files.
+
+Templates (``.env.example``, ``.env.sample``, ``.env.template``) are always read: they
+are committed and meant to be shared. ``.env`` and ``.env.local`` hold real values,
+often secrets, and are read only with ``include_env_files``; then a key they set
+replaces the templates' value for it, as dotenv tools load ``.env.local`` over ``.env``
+and never read a template. Either way a value leaves
+this module only as a port number, a database without credentials or a path; the
+values of every other variable are dropped.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path, PurePosixPath
+
+from worktree_shield.collectors.base import (
+    Collected,
+    Scope,
+    is_absolute_path,
+    path_key,
+    port_key,
+)
+from worktree_shield.models import (
+    Policy,
+    ResourceClaim,
+    ResourceKind,
+    SkippedFile,
+    SourceRef,
+    UnreadableValue,
+)
+from worktree_shield.names import (
+    host_variable,
+    is_mutable_path_variable,
+    literal_host,
+    port_number,
+    port_variable,
+)
+from worktree_shield.parsers.dburl import DatabaseURLError, parse_database_url
+from worktree_shield.parsers.dotenv import EnvEntry, parse_dotenv
+
+TEMPLATE_FILES = (".env.example", ".env.sample", ".env.template")
+LOCAL_FILES = (".env", ".env.local")
+NOT_READ = "was not read: .env files are read only with --include-env"
+
+
+def collect_env_files(
+    scope: Scope, directory: PurePosixPath, policy: Policy, out: Collected
+) -> None:
+    """The claims of the ``.env`` files in one directory of a worktree."""
+    names = TEMPLATE_FILES + (LOCAL_FILES if policy.include_env_files else ())
+    if not policy.include_env_files:
+        folder = Path(scope.root, *directory.parts)
+        for name in LOCAL_FILES:  # noted, not opened: what runs may differ from what was read
+            if os.path.lexists(folder / name):
+                out.skipped.append(SkippedFile(scope.source(directory / name), NOT_READ))
+    files: list[tuple[PurePosixPath, tuple[EnvEntry, ...]]] = []
+    for name in names:
+        relative = directory / name
+        text = scope.read(relative, out)
+        if text is not None:
+            files.append((relative, parse_dotenv(text)))
+    for index, (relative, entries) in enumerate(files):
+        later = files[index + 1 :]  # in names order: templates, then .env, then .env.local
+        overridden = {
+            e.key.upper() for path, found in later if path.name in LOCAL_FILES for e in found
+        }
+        _collect(scope, relative, entries, out, skip=overridden)
+
+
+def _collect(
+    scope: Scope,
+    relative: PurePosixPath,
+    entries: tuple[EnvEntry, ...],
+    out: Collected,
+    *,
+    skip: set[str],
+) -> None:
+    """The claims of one file, except for keys a higher-priority local file sets."""
+    by_name = {entry.key.upper(): entry for entry in entries}
+    for entry in entries:
+        if entry.reference or entry.key.upper() in skip:
+            continue
+        value = entry.value.strip()
+        source = scope.source(relative, entry.line)
+        try:
+            target = parse_database_url(value)
+        except DatabaseURLError as error:
+            out.unreadable.append(
+                UnreadableValue(ResourceKind.DATABASE, source, entry.key, str(error))
+            )
+            continue
+        if target is not None:
+            if target.engine == "sqlite" and not is_absolute_path(target.database or ""):
+                continue  # relative to each worktree, so never shared
+            detail = entry.key  # the key already names the database
+            out.claims.append(
+                ResourceClaim(ResourceKind.DATABASE, target.key, scope.worktree, source, detail)
+            )
+            continue
+        port = port_number(value)
+        kind = port_variable(entry.key)
+        if port is not None and kind is not None:
+            out.claims.append(_port_claim(scope, source, entry, kind, port, by_name))
+        elif is_mutable_path_variable(entry.key) and is_absolute_path(value):
+            detail = f"{entry.key}={value}"
+            out.claims.append(
+                ResourceClaim(
+                    ResourceKind.ABSOLUTE_PATH, path_key(value), scope.worktree, source, detail
+                )
+            )
+
+
+def _port_claim(
+    scope: Scope,
+    source: SourceRef,
+    entry: EnvEntry,
+    kind: ResourceKind,
+    port: int,
+    by_name: dict[str, EnvEntry],
+) -> ResourceClaim:
+    host_entry = by_name.get(host_variable(entry.key))
+    host = None
+    if host_entry is not None and not host_entry.reference:
+        host = literal_host(host_entry.value)
+    default = "localhost" if kind is ResourceKind.CLIENT_ENDPOINT else "*"
+    detail = f"{entry.key}={port}"
+    if host_entry is not None and host is not None:
+        detail += f" ({host_entry.key}={host_entry.value.strip()})"
+    return ResourceClaim(kind, port_key(host, port, default), scope.worktree, source, detail)

worktree_shield/collectors/layout.py

@@ -0,0 +1,106 @@
+"""Which directories of a worktree hold configuration worth reading.
+
+Monorepos keep a ``package.json`` or ``.env.example`` per app (``apps/web``,
+``services/api``), so the root alone is not enough, but the whole tree is too much:
+the walk lists directories only (no file is opened here), goes at most
+:data:`MAX_DEPTH` levels down, never follows a link to a directory, and skips
+dependencies, build output, test fixtures and examples, which declare ports nobody runs
+from this worktree.
+
+A directory with a ``.git`` entry of its own is another worktree or a submodule and is
+not entered: worktrees often live inside the main one (``repo/worktrees/feature``), and
+their files must not be counted twice, once as the main worktree's.
+"""
+
+from __future__ import annotations
+
+import os
+from collections import deque
+from pathlib import Path, PurePosixPath
+
+from worktree_shield.collectors.base import Collected, Scope
+from worktree_shield.collectors.compose import COMPOSE_FILES, OVERRIDE_FILES
+from worktree_shield.collectors.envfiles import LOCAL_FILES, TEMPLATE_FILES
+from worktree_shield.collectors.vite import CONFIG_FILES as VITE_FILES
+from worktree_shield.models import SkippedFile
+
+MAX_DEPTH = 3
+MAX_DIRECTORIES = 2000
+CONFIG_NAMES = frozenset(
+    (
+        *TEMPLATE_FILES,
+        *LOCAL_FILES,
+        "package.json",
+        "pyproject.toml",
+        *VITE_FILES,
+        *COMPOSE_FILES,
+        *OVERRIDE_FILES,
+        "Procfile",
+        "Procfile.dev",
+        ".devcontainer",
+        ".devcontainer.json",
+    )
+)
+SKIPPED_DIRECTORIES = frozenset(
+    {
+        "node_modules",
+        "bower_components",
+        "vendor",
+        "third_party",
+        "site-packages",
+        "venv",
+        "dist",
+        "build",
+        "out",
+        "target",
+        "coverage",
+        "__pycache__",
+        "fixtures",
+        "__fixtures__",
+        "testdata",
+        "examples",
+    }
+)
+
+
+def config_directories(scope: Scope, out: Collected) -> list[PurePosixPath]:
+    """Directories (relative to the worktree, root first) that contain at least one
+    known configuration file, breadth first. A directory that cannot be listed, or a
+    walk that reaches :data:`MAX_DIRECTORIES`, is recorded in ``out.skipped``."""
+    found: list[PurePosixPath] = []
+    queue: deque[PurePosixPath] = deque([PurePosixPath()])
+    listed = 0
+    while queue:
+        relative = queue.popleft()
+        listed += 1
+        if listed > MAX_DIRECTORIES:
+            reason = f"was not searched: the walk stopped after {MAX_DIRECTORIES} directories"
+            out.skipped.append(SkippedFile(scope.source(relative), reason))
+            break
+        try:
+            with os.scandir(scope.root.joinpath(*relative.parts)) as listing:
+                entries = sorted(listing, key=lambda entry: entry.name)
+        except OSError as error:
+            reason = f"could not be listed ({error.strerror or error})"
+            out.skipped.append(SkippedFile(scope.source(relative), reason))
+            continue
+        if any(entry.name in CONFIG_NAMES for entry in entries):
+            found.append(relative)
+        if len(relative.parts) >= MAX_DEPTH:
+            continue
+        for entry in entries:
+            if _worth_entering(entry):
+                queue.append(relative / entry.name)
+    return found
+
+
+def _worth_entering(entry: os.DirEntry[str]) -> bool:
+    name = entry.name
+    if name.startswith(".") or name in SKIPPED_DIRECTORIES:
+        return False
+    try:
+        if not entry.is_dir(follow_symlinks=False):
+            return False
+        return not os.path.lexists(Path(entry.path, ".git"))  # another worktree or submodule
+    except OSError:
+        return False

worktree_shield/collectors/node.py

@@ -0,0 +1,87 @@
+"""Listen ports in ``package.json``: every script, and npm's ``config.port``.
+
+The file is parsed with :mod:`json`; scripts are scanned as text by
+:func:`~worktree_shield.parsers.commands.listen_ports` and never run. Only the fragment
+that names a port is kept, so other values on the same command line go nowhere.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import replace
+from pathlib import PurePosixPath
+
+from worktree_shield.collectors.base import Collected, Scope, port_key
+from worktree_shield.models import ResourceClaim, ResourceKind, SkippedFile
+from worktree_shield.names import port_number
+from worktree_shield.parsers.commands import listen_ports
+
+_TOKEN = re.compile(r'"(?:[^"\\]|\\.)*"|[{}\[\]:]', re.S)
+
+
+def collect_package_json(scope: Scope, directory: PurePosixPath, out: Collected) -> None:
+    """The listen ports in ``directory/package.json``, if there is one."""
+    relative = directory / "package.json"
+    text = scope.read(relative, out)
+    if text is None:
+        return
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError as error:
+        out.skipped.append(SkippedFile(scope.source(relative, error.lineno), "is not valid JSON"))
+        return
+    if not isinstance(data, dict):
+        return
+    lines = _key_lines(text)
+
+    def claim(port: int, host: str | None, key: tuple[str, str], detail: str) -> ResourceClaim:
+        source = scope.source(relative, lines.get(key))
+        resource = port_key(host, port)
+        return ResourceClaim(ResourceKind.LISTEN_PORT, resource, scope.worktree, source, detail)
+
+    scripts = data.get("scripts")
+    if isinstance(scripts, dict):
+        for name, command in scripts.items():
+            if isinstance(command, str):
+                for mention in listen_ports(command):
+                    found = claim(
+                        mention.port,
+                        mention.host,
+                        ("scripts", name),
+                        f"scripts.{name}: {mention.fragment}",
+                    )
+                    out.claims.append(replace(found, strict=mention.strict))
+    config = data.get("config")
+    if isinstance(config, dict):
+        value = config.get("port")
+        if isinstance(value, (str, int)) and not isinstance(value, bool):
+            port = port_number(str(value))
+            if port is not None:
+                out.claims.append(claim(port, None, ("config", "port"), f"config.port={port}"))
+
+
+def _key_lines(text: str) -> dict[tuple[str, ...], int]:
+    """The line of every object key, by its path (``("scripts", "dev")``); for a key
+    defined twice the later line, as :mod:`json` keeps the later value."""
+    lines: dict[tuple[str, ...], int] = {}
+    tokens = list(_TOKEN.finditer(text))
+    opened_by: list[str | None] = []  # the key whose value each open container is
+    last_key: str | None = None
+    line, position = 1, 0
+    for index, match in enumerate(tokens):
+        piece = match.group()
+        if piece.startswith('"'):
+            if index + 1 < len(tokens) and tokens[index + 1].group() == ":":
+                last_key = json.loads(piece)
+                line += text.count("\n", position, match.start())
+                position = match.start()
+                path = tuple(key for key in opened_by if key is not None)
+                lines[(*path, last_key)] = line
+        elif piece in "{[":
+            opened_by.append(last_key)
+            last_key = None
+        elif piece in "}]":
+            opened_by.pop()  # balanced: json.loads() accepted the text
+            last_key = None
+    return lines