github-repository-sync 0.0.1__py3-none-any.whl

github_repository_sync-0.0.1.dist-info/METADATA

@@ -0,0 +1,105 @@
+Metadata-Version: 2.4
+Name: github-repository-sync
+Version: 0.0.1
+Summary: Find and clone all repositories for a GitHub organization.
+Author: Kaizten Analytics
+License-Expression: LicenseRef-Proprietary
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# GitHub Organization Repository Sync
+
+`github-repository-sync.py` finds all repositories for a GitHub organization and checks whether they already exist locally under a recursive search root.
+
+Repositories that are missing locally are cloned into `<root>/<org>/<repo>` by default. The script detects existing local repositories by inspecting their Git remotes, so repositories can be located in nested folders.
+
+The script uses named arguments only. There are no positional arguments.
+
+## Dependencies
+
+- Python 3.10 or newer
+- Git
+- GitHub CLI (`gh`), recommended for private repositories
+
+Authentication options:
+
+- Use an authenticated GitHub CLI session with `gh auth login`.
+- Or set `GITHUB_TOKEN` or `GH_TOKEN`.
+- Or pass a token with `--token`.
+
+Without authentication, GitHub only returns public repositories.
+
+## How to use
+
+Run the script with the organization name:
+
+```bash
+./github-repository-sync.py --org kaizten
+```
+
+By default, the script searches recursively under `~` and clones missing
+repositories into `~/kaizten`.
+
+Use `--dry-run` to preview the missing repositories and clone destinations
+without cloning anything:
+
+```bash
+./github-repository-sync.py --org kaizten --dry-run
+```
+
+Use `--root` to search and clone under a different directory:
+
+```bash
+./github-repository-sync.py --org kaizten --root /home/christopher
+```
+
+Use `--clone-dir` to clone missing repositories somewhere other than
+`<root>/<org>`:
+
+```bash
+./github-repository-sync.py --org kaizten --clone-dir /home/christopher/repos
+```
+
+If a clone destination already exists and is not empty, the script chooses the
+next available suffixed directory, such as `repo-1`.
+
+## Examples
+
+Preview missing repositories for `kaizten`:
+
+```bash
+./github-repository-sync.py --org kaizten --dry-run
+```
+
+Clone all missing repositories for `kaizten` under the default root:
+
+```bash
+./github-repository-sync.py --org kaizten
+```
+
+Search under `/home/christopher` and clone missing repositories under
+`/home/christopher/kaizten`:
+
+```bash
+./github-repository-sync.py --org kaizten --root /home/christopher
+```
+
+Clone using SSH URLs:
+
+```bash
+./github-repository-sync.py --org kaizten --protocol ssh
+```
+
+Skip archived repositories:
+
+```bash
+./github-repository-sync.py --org kaizten --exclude-archived
+```
+
+Use an explicit GitHub token:
+
+```bash
+./github-repository-sync.py --org kaizten --token YOUR_GITHUB_TOKEN
+```

github_repository_sync-0.0.1.dist-info/RECORD

@@ -0,0 +1,6 @@
+github_repository_sync.py,sha256=iTazVcIaJJMyZEmHXNNzdnbLA1u8pujnFE0MdcHAWcQ,12384
+github_repository_sync-0.0.1.dist-info/METADATA,sha256=aucU8VOJhcexwrT1bNtC2IFIVF3wCVwDqtZqv_KdaTY,2767
+github_repository_sync-0.0.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+github_repository_sync-0.0.1.dist-info/entry_points.txt,sha256=Jpx2WqZQVFYwUL-wDve5fC-v4l4kUqI4ovX_G59klqk,71
+github_repository_sync-0.0.1.dist-info/top_level.txt,sha256=52t4ReaXYcSgDfrB0PjrelM6Zav7M1bDp_l87TSyGck,23
+github_repository_sync-0.0.1.dist-info/RECORD,,

github_repository_sync-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

github_repository_sync-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+github-repository-sync = github_repository_sync:main

github_repository_sync-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+github_repository_sync

github_repository_sync.py

@@ -0,0 +1,396 @@
+#!/usr/bin/env python3
+"""Find and clone all repositories for a GitHub organization.
+
+The script discovers local clones recursively under a root directory by
+inspecting git remotes. Any organization repository that is not already present
+locally is cloned into <root>/<org>/<repo> by default.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass
+from pathlib import Path
+
+
+GITHUB_API = "https://api.github.com"
+
+
+@dataclass(frozen=True)
+class GitHubRepo:
+    name: str
+    ssh_url: str
+    clone_url: str
+    archived: bool
+    disabled: bool
+
+
+@dataclass(frozen=True)
+class CloneResult:
+    repo_name: str
+    destination: Path
+    status: str
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Recursively find local clones for a GitHub organization and clone missing repositories.",
+    )
+    parser.add_argument(
+        "--org",
+        required=True,
+        help="GitHub organization name. Example: kaizten",
+    )
+    parser.add_argument(
+        "--root",
+        type=Path,
+        default=Path.home(),
+        help="Directory to search recursively and clone into. Defaults to ~.",
+    )
+    parser.add_argument(
+        "--token",
+        default=os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN"),
+        help="GitHub token. Defaults to GITHUB_TOKEN or GH_TOKEN when set.",
+    )
+    parser.add_argument(
+        "--no-gh-cli",
+        action="store_true",
+        help="Do not use the GitHub CLI as an authentication fallback.",
+    )
+    parser.add_argument(
+        "--github-api",
+        default=GITHUB_API,
+        help="GitHub API base URL. Defaults to https://api.github.com.",
+    )
+    parser.add_argument(
+        "--protocol",
+        choices=("https", "ssh"),
+        default="https",
+        help="Clone protocol for missing repositories. Defaults to https.",
+    )
+    parser.add_argument(
+        "--clone-dir",
+        type=Path,
+        help="Directory where missing repositories are cloned. Defaults to <root>/<org>.",
+    )
+    parser.add_argument(
+        "--include-archived",
+        action="store_true",
+        help="Include archived repositories. This is the default; kept for compatibility.",
+    )
+    parser.add_argument(
+        "--exclude-archived",
+        action="store_true",
+        help="Skip archived repositories.",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Print what would be cloned without running git clone.",
+    )
+    return parser.parse_args()
+
+
+def run_git(args: list[str], cwd: Path | None = None) -> str:
+    completed = subprocess.run(
+        ["git", *args],
+        cwd=cwd,
+        check=True,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    return completed.stdout.strip()
+
+
+def is_git_repository(path: Path) -> bool:
+    return (path / ".git").exists()
+
+
+def iter_git_repositories(root: Path) -> list[Path]:
+    repos: list[Path] = []
+    ignored_dir_names = {".git", ".hg", ".svn", "node_modules", ".venv", "venv", "__pycache__"}
+
+    for current_root, dir_names, _file_names in os.walk(root):
+        dir_names[:] = [name for name in dir_names if name not in ignored_dir_names]
+        current_path = Path(current_root)
+
+        if is_git_repository(current_path):
+            repos.append(current_path)
+
+    return repos
+
+
+def normalize_remote_url(url: str) -> tuple[str, str] | None:
+    url = url.strip()
+    if not url:
+        return None
+
+    scp_match = re.match(r"^git@[^:]+:([^/]+)/(.+?)(?:\.git)?$", url)
+    if scp_match:
+        return scp_match.group(1).lower(), scp_match.group(2).lower()
+
+    parsed = urllib.parse.urlparse(url)
+    if parsed.scheme and parsed.netloc:
+        parts = [part for part in parsed.path.strip("/").split("/") if part]
+        if len(parts) >= 2:
+            repo_name = re.sub(r"\.git$", "", parts[1], flags=re.IGNORECASE)
+            return parts[0].lower(), repo_name.lower()
+
+    return None
+
+
+def repository_remotes(repo_path: Path) -> set[tuple[str, str]]:
+    try:
+        output = run_git(["remote", "-v"], cwd=repo_path)
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        return set()
+
+    remotes: set[tuple[str, str]] = set()
+    for line in output.splitlines():
+        parts = line.split()
+        if len(parts) >= 2:
+            normalized = normalize_remote_url(parts[1])
+            if normalized:
+                remotes.add(normalized)
+    return remotes
+
+
+def local_repositories_by_name(root: Path, org: str) -> dict[str, Path]:
+    local: dict[str, Path] = {}
+    expected_owner = org.lower()
+
+    for repo_path in iter_git_repositories(root):
+        for owner, repo_name in repository_remotes(repo_path):
+            if owner == expected_owner:
+                local.setdefault(repo_name, repo_path)
+
+    return local
+
+
+def github_request(url: str, token: str | None) -> tuple[list[dict], str | None]:
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "User-Agent": "github-repository-sync",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+
+    request = urllib.request.Request(url, headers=headers)
+    try:
+        with urllib.request.urlopen(request) as response:
+            payload = json.loads(response.read().decode("utf-8"))
+            next_url = parse_next_link(response.headers.get("Link", ""))
+            return payload, next_url
+    except urllib.error.HTTPError as error:
+        message = error.read().decode("utf-8", errors="replace")
+        raise RuntimeError(f"GitHub API request failed ({error.code}): {message}") from error
+    except urllib.error.URLError as error:
+        raise RuntimeError(f"GitHub API request failed: {error.reason}") from error
+
+
+def parse_next_link(link_header: str) -> str | None:
+    for item in link_header.split(","):
+        section = item.strip()
+        if 'rel="next"' not in section:
+            continue
+        match = re.match(r"<([^>]+)>", section)
+        if match:
+            return match.group(1)
+    return None
+
+
+def fetch_org_repositories(api_base: str, org: str, token: str | None) -> list[GitHubRepo]:
+    encoded_org = urllib.parse.quote(org, safe="")
+    url: str | None = f"{api_base.rstrip('/')}/orgs/{encoded_org}/repos?per_page=100&type=all"
+    repos: list[GitHubRepo] = []
+
+    while url:
+        payload, url = github_request(url, token)
+        for item in payload:
+            repos.append(
+                GitHubRepo(
+                    name=item["name"],
+                    ssh_url=item["ssh_url"],
+                    clone_url=item["clone_url"],
+                    archived=bool(item.get("archived")),
+                    disabled=bool(item.get("disabled")),
+                )
+            )
+
+    return repos
+
+
+def fetch_org_repositories_with_gh(org: str) -> list[GitHubRepo]:
+    if not shutil.which("gh"):
+        return []
+
+    completed = subprocess.run(
+        [
+            "gh",
+            "api",
+            "--paginate",
+            f"/orgs/{org}/repos?per_page=100&type=all",
+            "--jq",
+            ".[] | [.name, .ssh_url, .clone_url, .archived, .disabled] | @tsv",
+        ],
+        check=False,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    if completed.returncode != 0:
+        return []
+
+    repos: list[GitHubRepo] = []
+    for line in completed.stdout.splitlines():
+        if not line.strip():
+            continue
+        fields = line.split("\t")
+        if len(fields) != 5:
+            continue
+        repos.append(
+            GitHubRepo(
+                name=fields[0],
+                ssh_url=fields[1],
+                clone_url=fields[2],
+                archived=fields[3] == "true",
+                disabled=fields[4] == "true",
+            )
+        )
+    return repos
+
+
+def available_clone_destination(clone_dir: Path, repo_name: str) -> Path:
+    destination = clone_dir / repo_name
+    if not destination.exists() or not any(destination.iterdir()):
+        return destination
+
+    suffix = 1
+    while True:
+        candidate = clone_dir / f"{repo_name}-{suffix}"
+        if not candidate.exists() or not any(candidate.iterdir()):
+            return candidate
+        suffix += 1
+
+
+def print_repository_list(title: str, repos: list[CloneResult]) -> None:
+    print(title)
+    for result in repos:
+        print(f"- {result.repo_name}: {result.destination}")
+
+
+def clone_repository(
+    repo: GitHubRepo,
+    org: str,
+    clone_dir: Path,
+    protocol: str,
+    dry_run: bool,
+    use_gh_clone: bool,
+) -> CloneResult:
+    default_destination = clone_dir / repo.name
+    destination = available_clone_destination(clone_dir, repo.name)
+    source = repo.ssh_url if protocol == "ssh" else repo.clone_url
+
+    if destination != default_destination:
+        print(
+            f"warning: {default_destination} is not empty; cloning {repo.name} into {destination}",
+            file=sys.stderr,
+        )
+
+    print(f"clone: {repo.name} -> {destination}")
+    if dry_run:
+        return CloneResult(repo.name, destination, "dry-run")
+
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    command = (
+        ["gh", "repo", "clone", f"{org}/{repo.name}", str(destination)]
+        if use_gh_clone
+        else ["git", "clone", source, str(destination)]
+    )
+    try:
+        subprocess.run(command, check=True)
+        return CloneResult(repo.name, destination, "cloned")
+    except subprocess.CalledProcessError as error:
+        print(f"error: failed to clone {repo.name} ({error})", file=sys.stderr)
+        return CloneResult(repo.name, destination, "failed")
+
+
+def main() -> int:
+    args = parse_args()
+    root = args.root.expanduser().resolve()
+    clone_dir = (args.clone_dir or root / args.org).expanduser().resolve()
+
+    if not root.exists():
+        print(f"error: root directory does not exist: {root}", file=sys.stderr)
+        return 2
+
+    print(f"searching local repositories under {root}")
+    local_repos = local_repositories_by_name(root, args.org)
+    print(f"found {len(local_repos)} local repositories for {args.org}")
+
+    if not args.token and not args.no_gh_cli:
+        org_repos = fetch_org_repositories_with_gh(args.org)
+        if org_repos:
+            source = "GitHub CLI"
+        else:
+            org_repos = fetch_org_repositories(args.github_api, args.org, args.token)
+            source = "GitHub API"
+    else:
+        org_repos = fetch_org_repositories(args.github_api, args.org, args.token)
+        source = "GitHub API"
+
+    active_repos = [
+        repo
+        for repo in org_repos
+        if not repo.disabled and not (args.exclude_archived and repo.archived)
+    ]
+    print(f"found {len(active_repos)} organization repositories from {source}")
+    if not args.token and source == "GitHub API":
+        print(
+            "warning: no GitHub token was provided and GitHub CLI authentication was not usable; "
+            "private repositories will not be listed",
+            file=sys.stderr,
+        )
+
+    missing = [repo for repo in active_repos if repo.name.lower() not in local_repos]
+    if not missing:
+        print("all organization repositories are already present locally")
+        return 0
+
+    print(f"{len(missing)} repositories missing locally")
+    planned = [
+        CloneResult(repo.name, available_clone_destination(clone_dir, repo.name), "pending")
+        for repo in missing
+    ]
+    print_repository_list("missing repositories:", planned)
+
+    results: list[CloneResult] = []
+    use_gh_clone = source == "GitHub CLI" and not args.no_gh_cli
+    for repo in missing:
+        results.append(
+            clone_repository(repo, args.org, clone_dir, args.protocol, args.dry_run, use_gh_clone)
+        )
+
+    print("summary:")
+    for status in ("cloned", "dry-run", "failed"):
+        status_results = [result for result in results if result.status == status]
+        if status_results:
+            print_repository_list(f"{status} repositories ({len(status_results)}):", status_results)
+
+    failures = sum(1 for result in results if result.status == "failed")
+    return 1 if failures else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())