src-auth-perms-sync 0.2.1__py3-none-any.whl

src_auth_perms_sync/permissions/workflow.py

@@ -0,0 +1,526 @@
+"""Shared helpers for repo permission command workflows."""
+
+from __future__ import annotations
+
+import datetime
+import logging
+from collections.abc import Iterator
+from pathlib import Path
+
+import src_py_lib as src
+
+from ..shared import backups, id_codec, saml_groups
+from ..shared import sourcegraph as shared_sourcegraph
+from ..shared import types as shared_types
+from . import mapping as permissions_mapping
+from . import maps as permissions_maps
+from . import snapshot as permission_snapshot
+from . import sourcegraph as permissions_sourcegraph
+from . import types as permission_types
+
+log = logging.getLogger(__name__)
+
+
+def load_discovery(
+    client: src.SourcegraphClient,
+    saml_groups_attribute_name_by_config_id: dict[str, str],
+) -> tuple[
+    list[shared_types.AuthProvider],
+    list[permission_types.ExternalService],
+    dict[tuple[str, str], str],
+]:
+    """Fetch auth providers + external services and resolve the SAML attribute
+    names map, with consistent logging. Shared by --get and --set; returns the
+    raw lists so each caller can transform them as needed (YAML form for --get,
+    keyed-by-id dict for --set).
+
+    Both commands need exactly the same instance state to do their work, so
+    centralizing this avoids drift in which providers/services are considered
+    authoritative or how the per-provider SAML attribute override map is
+    resolved.
+    """
+    log.info("Querying auth providers from %s ...", client.endpoint)
+    providers = shared_sourcegraph.list_auth_providers(client)
+    log.info("Received %d auth providers.", len(providers))
+
+    log.info("Loading external services from %s ...", client.endpoint)
+    services = permissions_sourcegraph.list_external_services(client)
+    log.info("Received %d external services.", len(services))
+
+    saml_attribute_names = saml_groups.attribute_names_by_provider_key(
+        providers, saml_groups_attribute_name_by_config_id
+    )
+    return providers, services, saml_attribute_names
+
+
+def load_repos_by_external_service(
+    client: src.SourcegraphClient,
+    services_by_id: dict[int, permission_types.ExternalService],
+) -> dict[int, list[permission_types.Repository]]:
+    """Fetch repos once per discovered code host connection."""
+    with src.event(
+        "load_repos_by_external_service",
+        external_service_count=len(services_by_id),
+    ) as load_event:
+        expected_repo_count = sum(service["repoCount"] for service in services_by_id.values())
+        load_event["expected_repo_count"] = expected_repo_count
+        log.info(
+            "Loading about %d repo(s) across %d code host connection(s) ...",
+            expected_repo_count,
+            len(services_by_id),
+        )
+
+        repos_by_external_service_id: dict[int, list[permission_types.Repository]] = {}
+        total_repos = 0
+        for external_service_id in sorted(services_by_id):
+            service = services_by_id[external_service_id]
+            repos = permissions_sourcegraph.list_repos_for_external_service(client, service["id"])
+            repos_by_external_service_id[external_service_id] = repos
+            total_repos += len(repos)
+            log.info(
+                "Received %d repo(s) for code host connection %s (id=%d).",
+                len(repos),
+                service["displayName"],
+                external_service_id,
+            )
+        load_event["repo_count"] = total_repos
+        return repos_by_external_service_id
+
+
+def index_repos_by_id(
+    repos_by_external_service_id: dict[int, list[permission_types.Repository]],
+) -> dict[str, permission_types.Repository]:
+    repos_by_id: dict[str, permission_types.Repository] = {}
+    for repos in repos_by_external_service_id.values():
+        for repo in repos:
+            repos_by_id[repo["id"]] = repo
+    return repos_by_id
+
+
+def load_mapping_rules(input_path: Path) -> list[permission_types.MappingRule]:
+    """Load and structurally validate mapping rules from YAML."""
+    config = permissions_maps.load_maps_yaml(input_path)
+    mapping_rules = config.get("maps") or []
+    if mapping_rules:
+        permissions_mapping.validate_mapping_rules(mapping_rules)
+    return mapping_rules
+
+
+def load_mapping_context(
+    client: src.SourcegraphClient,
+    input_path: Path,
+    saml_groups_attribute_name_by_config_id: dict[str, str],
+) -> permission_types.MappingContext | None:
+    """Load maps, providers, services, and repos for permission planning."""
+    mapping_rules = load_mapping_rules(input_path)
+    if not mapping_rules:
+        log.warning("No maps defined in %s — nothing to do.", input_path)
+        return None
+
+    return load_mapping_context_for_rules(
+        client,
+        mapping_rules,
+        saml_groups_attribute_name_by_config_id,
+    )
+
+
+def load_mapping_context_for_rules(
+    client: src.SourcegraphClient,
+    mapping_rules: list[permission_types.MappingRule],
+    saml_groups_attribute_name_by_config_id: dict[str, str],
+) -> permission_types.MappingContext:
+    """Load providers, services, repos, and warning context for mapping rules."""
+    providers, services, saml_groups_attribute_names = load_discovery(
+        client, saml_groups_attribute_name_by_config_id
+    )
+    services_by_id: dict[int, permission_types.ExternalService] = {
+        id_codec.decode_external_service_id(service["id"]): service for service in services
+    }
+    repos_by_external_service_id = load_repos_by_external_service(client, services_by_id)
+    all_repos_by_id = index_repos_by_id(repos_by_external_service_id)
+    log.info(
+        "Received %d unique repo(s) across %d code host connection(s).",
+        len(all_repos_by_id),
+        len(services_by_id),
+    )
+    warn_unknown_external_services(mapping_rules, services_by_id)
+    return permission_types.MappingContext(
+        mapping_rules=mapping_rules,
+        providers=providers,
+        saml_groups_attribute_names=saml_groups_attribute_names,
+        services_by_id=services_by_id,
+        repos_by_external_service_id=repos_by_external_service_id,
+        all_repos_by_id=all_repos_by_id,
+    )
+
+
+def warn_unknown_external_services(
+    mapping_rules: list[permission_types.MappingRule],
+    services_by_id: dict[int, permission_types.ExternalService],
+) -> None:
+    """Warn when maps reference code-host connection IDs absent on the instance."""
+    for external_service_id in sorted(
+        permissions_mapping.referenced_external_service_ids(mapping_rules)
+    ):
+        if external_service_id not in services_by_id:
+            log.warning(
+                "External service id %s is referenced by the maps but "
+                "is not present on the instance — rules using it will "
+                "resolve to zero repos.",
+                external_service_id,
+            )
+
+
+def snapshot_path(
+    input_path: Path,
+    timestamp: str,
+    endpoint: str,
+    command: str,
+    state: str | None = None,
+) -> Path:
+    """Return a path inside the run's artifact directory.
+
+    Example: maps.yaml + endpoint + timestamp + set-apply + before →
+    src-auth-perms-sync-runs/sourcegraph.example.com/runs/2026-04-27-01-54-23-set-apply/before.json.
+    """
+    return backups.backup_path(input_path.name, timestamp, endpoint, command, state)
+
+
+def write_snapshot_pair(
+    input_path: Path,
+    timestamp: str,
+    endpoint: str,
+    command: str,
+    before_snapshot: permission_snapshot.Snapshot,
+    after_snapshot: permission_snapshot.Snapshot,
+) -> tuple[Path, Path, Path]:
+    before_path = snapshot_path(input_path, timestamp, endpoint, command, "before")
+    after_path = snapshot_path(input_path, timestamp, endpoint, command, "after")
+    permission_snapshot.write_snapshot(before_path, before_snapshot)
+    permission_snapshot.write_snapshot(after_path, after_snapshot)
+    diff_path = write_snapshot_diff_file(
+        input_path,
+        timestamp,
+        endpoint,
+        command,
+        before_snapshot,
+        after_snapshot,
+    )
+    return before_path, after_path, diff_path
+
+
+def write_snapshot_diff_file(
+    input_path: Path,
+    timestamp: str,
+    endpoint: str,
+    command: str,
+    before_snapshot: permission_snapshot.Snapshot,
+    after_snapshot: permission_snapshot.Snapshot,
+) -> Path:
+    diff_path = snapshot_path(input_path, timestamp, endpoint, command, "diff")
+    permission_snapshot.write_snapshot_diff_from_snapshots(
+        diff_path,
+        before_snapshot,
+        after_snapshot,
+    )
+    return diff_path
+
+
+def write_user_scoped_snapshot_diff_file(
+    input_path: Path,
+    timestamp: str,
+    endpoint: str,
+    command: str,
+    before_snapshot: permission_snapshot.UserScopedSnapshot,
+    after_snapshot: permission_snapshot.UserScopedSnapshot,
+) -> Path:
+    diff_path = snapshot_path(input_path, timestamp, endpoint, command, "diff")
+    permission_snapshot.write_user_scoped_snapshot_diff(
+        diff_path,
+        permission_snapshot.build_user_scoped_snapshot_diff(before_snapshot, after_snapshot),
+    )
+    return diff_path
+
+
+def maps_backup_path(
+    input_path: Path,
+    timestamp: str,
+    endpoint: str,
+    command: str,
+) -> Path:
+    """Path for the companion copy of the maps YAML used for a backup run."""
+    return backups.backup_path(input_path.name, timestamp, endpoint, command, suffix="yaml")
+
+
+def write_maps_backup(
+    input_path: Path,
+    timestamp: str,
+    endpoint: str,
+    command: str,
+) -> Path | None:
+    """Copy the active maps YAML next to the JSON snapshots for auditability."""
+    if not input_path.exists():
+        log.warning("Could not back up maps file %s because it does not exist.", input_path)
+        return None
+
+    output_path = maps_backup_path(input_path, timestamp, endpoint, command)
+    with src.event(
+        "disk_io",
+        level="DEBUG",
+        op="write",
+        path=str(output_path),
+        file_kind="yaml",
+    ) as disk_event:
+        contents = input_path.read_bytes()
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        output_path.write_bytes(contents)
+        disk_event["bytes"] = len(contents)
+    log.info("Wrote maps backup: %s", output_path)
+    return output_path
+
+
+def projected_snapshot_repo_ids(
+    before_snapshot: permission_snapshot.Snapshot,
+    expected_users: dict[str, tuple[str, ...]],
+) -> list[str]:
+    """Return repo IDs that may appear in a projected full-set after snapshot."""
+    return sorted(set(before_snapshot["repos"]) | set(expected_users))
+
+
+def projected_snapshot_repo_for_id(
+    before_snapshot: permission_snapshot.Snapshot,
+    expected_users: dict[str, tuple[str, ...]],
+    repo_names: dict[str, str],
+    repo_id: str,
+) -> permission_snapshot.RepoSnapshot | None:
+    """Return one projected repo snapshot without cloning the whole snapshot."""
+    if repo_id in expected_users:
+        usernames = expected_users[repo_id]
+        if not usernames:
+            return None
+        return {
+            "name": repo_names[repo_id],
+            "explicit_permissions_users": list(usernames),
+        }
+    return before_snapshot["repos"].get(repo_id)
+
+
+def projected_snapshot_repos(
+    before_snapshot: permission_snapshot.Snapshot,
+    expected_users: dict[str, tuple[str, ...]],
+    repo_names: dict[str, str],
+) -> Iterator[tuple[str, permission_snapshot.RepoSnapshot]]:
+    """Return projected repo entries one repo at a time in stable order."""
+    for repo_id in projected_snapshot_repo_ids(before_snapshot, expected_users):
+        repo = projected_snapshot_repo_for_id(
+            before_snapshot,
+            expected_users,
+            repo_names,
+            repo_id,
+        )
+        if repo is not None:
+            yield repo_id, repo
+
+
+def projected_snapshot_stats(
+    before_snapshot: permission_snapshot.Snapshot,
+    expected_users: dict[str, tuple[str, ...]],
+) -> permission_snapshot.SnapshotStats:
+    """Compute projected stats without materializing the projected snapshot."""
+    users_with_explicit_grants: set[str] = set()
+    total_grants = 0
+    repo_count = 0
+    for repo_id, repo in before_snapshot["repos"].items():
+        if repo_id in expected_users:
+            continue
+        repo_count += 1
+        usernames = repo["explicit_permissions_users"]
+        users_with_explicit_grants.update(usernames)
+        total_grants += len(usernames)
+    for usernames in expected_users.values():
+        if not usernames:
+            continue
+        repo_count += 1
+        users_with_explicit_grants.update(usernames)
+        total_grants += len(usernames)
+    return {
+        "total_users_scanned": before_snapshot["stats"]["total_users_scanned"],
+        "users_with_explicit_grants": len(users_with_explicit_grants),
+        "repos_with_explicit_grants": repo_count,
+        "total_grants": total_grants,
+    }
+
+
+def projected_snapshot_shell(
+    before_snapshot: permission_snapshot.Snapshot,
+    expected_users: dict[str, tuple[str, ...]],
+) -> permission_snapshot.Snapshot:
+    """Return projected snapshot metadata; repo entries are streamed separately."""
+    return {
+        "schema_version": before_snapshot["schema_version"],
+        "captured_at": datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds"),
+        "endpoint": before_snapshot["endpoint"],
+        "bindID_mode": before_snapshot["bindID_mode"],
+        "config_file": before_snapshot["config_file"],
+        "config_sha256": before_snapshot["config_sha256"],
+        "pending_bindIDs": list(before_snapshot["pending_bindIDs"]),
+        "stats": projected_snapshot_stats(before_snapshot, expected_users),
+        "repos": {},
+    }
+
+
+def write_projected_snapshot(
+    path: Path,
+    before_snapshot: permission_snapshot.Snapshot,
+    expected_users: dict[str, tuple[str, ...]],
+    repo_names: dict[str, str],
+) -> permission_snapshot.Snapshot:
+    """Write a projected full-set after snapshot without holding it in memory."""
+    after_snapshot = projected_snapshot_shell(before_snapshot, expected_users)
+    permission_snapshot.write_snapshot_with_repos(
+        path,
+        after_snapshot,
+        projected_snapshot_repos(before_snapshot, expected_users, repo_names),
+    )
+    return after_snapshot
+
+
+def write_projected_snapshot_diff_file(
+    input_path: Path,
+    timestamp: str,
+    endpoint: str,
+    command: str,
+    before_snapshot: permission_snapshot.Snapshot,
+    after_snapshot: permission_snapshot.Snapshot,
+    expected_users: dict[str, tuple[str, ...]],
+    repo_names: dict[str, str],
+) -> Path:
+    """Write a diff for a projected full-set after snapshot."""
+    diff_path = snapshot_path(input_path, timestamp, endpoint, command, "diff")
+    repo_ids = projected_snapshot_repo_ids(before_snapshot, expected_users)
+    permission_snapshot.write_snapshot_diff_from_snapshot_parts(
+        diff_path,
+        before_snapshot,
+        after_snapshot,
+        repo_ids,
+        lambda repo_id: projected_snapshot_repo_for_id(
+            before_snapshot,
+            expected_users,
+            repo_names,
+            repo_id,
+        ),
+    )
+    return diff_path
+
+
+def render_projected_snapshot_diff(
+    before_snapshot: permission_snapshot.Snapshot,
+    after_snapshot: permission_snapshot.Snapshot,
+    expected_users: dict[str, tuple[str, ...]],
+    repo_names: dict[str, str],
+) -> str:
+    """Render a capped diff for a projected full-set after snapshot."""
+    repo_ids = projected_snapshot_repo_ids(before_snapshot, expected_users)
+    return permission_snapshot.render_snapshot_diff_from_snapshot_parts(
+        before_snapshot,
+        after_snapshot,
+        repo_ids,
+        lambda repo_id: projected_snapshot_repo_for_id(
+            before_snapshot,
+            expected_users,
+            repo_names,
+            repo_id,
+        ),
+    )
+
+
+def validate_post_apply(
+    after: permission_snapshot.Snapshot,
+    expected_users: dict[str, tuple[str, ...]],
+    mutated_repo_ids: set[str],
+) -> None:
+    """Post-apply sanity gates. Each failure WARNs/ERRORs but does not raise.
+
+    1. Pending bindIDs: any username we just wrote that didn't resolve to a
+       real User now appears in `usersWithPendingPermissions`. In our use
+       case this should never happen — we enumerate users via the users
+       query before mutating — but it's a cheap safety net.
+
+    2. Per-repo expected vs. actual: for every repo we touched, the
+       after-snapshot's explicit-user list must equal the union we asked
+       for. Disagreement means a partial write, a concurrent mutation by
+       another tool, or a server-side bug.
+    """
+    requested_usernames: set[str] = set()
+    for usernames in expected_users.values():
+        requested_usernames.update(usernames)
+    pending = set(after["pending_bindIDs"])
+    stuck = sorted(requested_usernames & pending)
+    if stuck:
+        log.error(
+            "VALIDATION: %d bindID(s) we just wrote did NOT resolve to "
+            "real users (now pending): %s",
+            len(stuck),
+            ", ".join(stuck),
+        )
+
+    mismatches = 0
+    for repo_id in mutated_repo_ids:
+        expected = list(expected_users.get(repo_id, ()))
+        actual_repo = after["repos"].get(repo_id)
+        actual = actual_repo["explicit_permissions_users"] if actual_repo else []
+        if expected == actual:
+            continue
+        expected_set = set(expected)
+        actual_set = set(actual)
+        mismatches += 1
+        only_expected = sorted(expected_set - actual_set)
+        only_actual = sorted(actual_set - expected_set)
+        log.warning(
+            "VALIDATION MISMATCH on repo id=%d: expected %d users, got %d.  "
+            "Expected-but-missing: %s.  Actual-but-unexpected: %s.",
+            id_codec.decode_repository_id(repo_id),
+            len(expected),
+            len(actual),
+            only_expected or "(none)",
+            only_actual or "(none)",
+        )
+    if mismatches:
+        log.warning(
+            "VALIDATION: %d / %d mutated repo(s) do not reflect the requested state.",
+            mismatches,
+            len(mutated_repo_ids),
+        )
+    else:
+        log.info(
+            "VALIDATION OK: all %d mutated repo(s) match the requested explicit-permissions state.",
+            len(mutated_repo_ids),
+        )
+
+
+def parse_cli_date(value: str, flag_name: str) -> datetime.datetime:
+    """Parse and validate a CLI date argument, returning UTC midnight."""
+    if len(value) != 10 or value[4] != "-" or value[7] != "-":
+        raise SystemExit(f"{flag_name} must use YYYY-MM-DD, got {value!r}.")
+    try:
+        parsed_date = datetime.date.fromisoformat(value)
+    except ValueError as error:
+        raise SystemExit(f"{flag_name} must use YYYY-MM-DD, got {value!r}.") from error
+    return datetime.datetime.combine(parsed_date, datetime.time(), tzinfo=datetime.UTC)
+
+
+def sourcegraph_datetime_filter(value: datetime.datetime) -> str:
+    """Return a Sourcegraph DateTime filter string for a UTC datetime."""
+    return value.isoformat(timespec="seconds").replace("+00:00", "Z")
+
+
+def user_ids_created_on_or_after(client: src.SourcegraphClient, value: str) -> set[str]:
+    """Return Sourcegraph user IDs created on or after the given CLI date."""
+    filter_value = sourcegraph_datetime_filter(parse_cli_date(value, "--created-after"))
+    candidates = permissions_sourcegraph.list_site_user_candidates(client, filter_value)
+    log.info(
+        "Restricting to %d Sourcegraph user(s) created on or after %s.",
+        len(candidates),
+        value,
+    )
+    return {candidate["id"] for candidate in candidates}

src_auth_perms_sync/shared/__init__.py

@@ -0,0 +1 @@
+"""Shared helpers used by auth mapper workflows."""

src_auth_perms_sync/shared/backups.py

@@ -0,0 +1,119 @@
+"""Endpoint-scoped artifact path helpers."""
+
+from __future__ import annotations
+
+import datetime
+import re
+from collections.abc import Generator
+from contextlib import contextmanager
+from contextvars import ContextVar
+from pathlib import Path
+from urllib.parse import urlsplit
+
+ARTIFACTS_DIR_NAME = "src-auth-perms-sync-runs"
+LOG_FILE_NAME = "log.json"
+RUNS_DIR_NAME = "runs"
+
+_CURRENT_RUN_ARTIFACTS_DIRECTORY: ContextVar[Path | None] = ContextVar(
+    "current_run_artifacts_directory",
+    default=None,
+)
+_CURRENT_RUN_TIMESTAMP: ContextVar[str | None] = ContextVar(
+    "current_run_timestamp",
+    default=None,
+)
+
+
+def backup_timestamp() -> str:
+    """Return a filesystem-friendly UTC timestamp."""
+    run_timestamp = _CURRENT_RUN_TIMESTAMP.get()
+    if run_timestamp is not None:
+        return run_timestamp
+    return datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d-%H-%M-%S")
+
+
+@contextmanager
+def run_artifacts_context(run_directory: Path, timestamp: str) -> Generator[None]:
+    """Make backup helpers write into the current CLI run directory."""
+    directory_token = _CURRENT_RUN_ARTIFACTS_DIRECTORY.set(run_directory)
+    timestamp_token = _CURRENT_RUN_TIMESTAMP.set(timestamp)
+    try:
+        yield
+    finally:
+        _CURRENT_RUN_TIMESTAMP.reset(timestamp_token)
+        _CURRENT_RUN_ARTIFACTS_DIRECTORY.reset(directory_token)
+
+
+def artifact_run_directory(timestamp: str, endpoint: str, command: str) -> Path:
+    """Return the artifact directory for one command run."""
+    run_directory = safe_filename_part(f"{timestamp}-{command}")
+    return endpoint_artifacts_directory(endpoint) / RUNS_DIR_NAME / run_directory
+
+
+def backup_path(
+    source_name: str,
+    timestamp: str,
+    endpoint: str,
+    command: str,
+    state: str | None = None,
+    *,
+    suffix: str = "json",
+) -> Path:
+    """Return an artifact path under one directory per endpoint run."""
+    backup_directory = _CURRENT_RUN_ARTIFACTS_DIRECTORY.get() or artifact_run_directory(
+        timestamp,
+        endpoint,
+        command,
+    )
+    if state is None:
+        return backup_directory / safe_filename_part(source_name)
+    return backup_directory / f"{safe_filename_part(state)}.{suffix}"
+
+
+def run_log_path(run_directory: Path) -> Path:
+    """Return the structured log path for a run artifact directory."""
+    return run_directory / LOG_FILE_NAME
+
+
+def endpoint_artifacts_directory(endpoint: str, current_directory: Path | None = None) -> Path:
+    """Return this endpoint's artifact directory under the current working directory."""
+    base_directory = current_directory or Path.cwd()
+    return base_directory / ARTIFACTS_DIR_NAME / endpoint_directory_name(endpoint)
+
+
+def endpoint_directory_name(endpoint: str) -> str:
+    """Return a filesystem-friendly directory name for a Sourcegraph endpoint."""
+    parsed_endpoint = urlsplit(endpoint)
+    hostname = parsed_endpoint.hostname
+    port = _fallback_endpoint_port(parsed_endpoint.netloc)
+    if not hostname:
+        endpoint_without_scheme = endpoint.split("://", 1)[-1]
+        hostname_and_port = endpoint_without_scheme.split("/", 1)[0]
+        hostname = hostname_and_port.split(":", 1)[0]
+        port = _fallback_endpoint_port(hostname_and_port)
+    directory_name = hostname.lower()
+    if port is not None:
+        directory_name = f"{directory_name}-{port}"
+    return safe_filename_part(directory_name)
+
+
+def endpoint_artifact_path(endpoint: str, path: Path) -> Path:
+    """Resolve a user-facing artifact path within the endpoint directory by default."""
+    if path.is_absolute():
+        return path
+    return endpoint_artifacts_directory(endpoint) / path
+
+
+def _fallback_endpoint_port(hostname_and_port: str) -> int | None:
+    """Parse a port from an endpoint netloc that urlsplit could not fully parse."""
+    if ":" not in hostname_and_port:
+        return None
+    raw_port = hostname_and_port.rsplit(":", 1)[1]
+    if not raw_port.isdecimal():
+        return None
+    return int(raw_port)
+
+
+def safe_filename_part(value: str) -> str:
+    """Return a non-empty string safe for backup filenames."""
+    return re.sub(r"[^A-Za-z0-9_.-]+", "_", value).strip("._-") or "unknown"