holocron-sync 0.1.0__py3-none-any.whl

holocron/__init__.py

@@ -0,0 +1,3 @@
+from .__main__ import main
+
+__all__ = ["main"]

holocron/__main__.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+import os
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+# Import from local modules
+from .config import parse_args
+from .logger import log
+from .github_provider import get_github_repos
+from .mirror import needs_sync, sync_one_repo
+
+def main():
+    args = parse_args()
+    
+    # Load secrets
+    gh_token = os.environ.get("GITHUB_TOKEN")
+    gl_token = os.environ.get("GITLAB_TOKEN")
+    
+    # Validation logic
+    if not gh_token:
+        print("CRITICAL: Missing GITHUB_TOKEN.")
+        sys.exit(1)
+
+    if not args.backup_only and not gl_token:
+        print("CRITICAL: Missing GITLAB_TOKEN.")
+        print("Please set GITLAB_TOKEN or use --backup-only.")
+        sys.exit(1)
+
+    log("Initializing Holocron...")
+    if args.dry_run:
+        log("!!! DRY RUN MODE ACTIVE !!!")
+
+    # Track the last synced push timestamp for each repo
+    synced_pushes = {}
+
+    while True:
+        repos = get_github_repos(gh_token, args.verbose)
+        log(f"Found {len(repos)} repositories on GitHub.", is_verbose_mode=args.verbose)
+
+        sync_count = 0
+        with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
+            future_to_repo = {}
+            for repo in repos:
+                repo_name = repo['name']
+                pushed_at = repo.get('pushed_at')
+                repo_dir = os.path.join(args.storage, f"{repo_name}.git")
+
+                # If watching, use the smart filter. If running once, sync all.
+                if args.watch:
+                    # 1. Skip if we already synced this exact push
+                    if repo_name in synced_pushes and synced_pushes[repo_name] == pushed_at:
+                        continue
+                    
+                    # 2. Check time window (SKIP if old AND local repo exists)
+                    # If local repo is missing, we MUST sync (bootstrap), regardless of age.
+                    if os.path.exists(repo_dir) and not needs_sync(repo, args.window):
+                         continue
+                    
+                future = executor.submit(sync_one_repo, repo, args, gh_token, gl_token)
+                future_to_repo[future] = repo
+
+            for future in as_completed(future_to_repo):
+                repo = future_to_repo[future]
+                try:
+                    future.result()
+                    sync_count += 1
+                    # Update tracking on success
+                    if repo.get('pushed_at'):
+                        synced_pushes[repo['name']] = repo['pushed_at']
+                except Exception as exc:
+                    log(f"[{repo['name']}] generated an exception: {exc}")
+
+        if sync_count > 0:
+            log(f"Sync cycle complete. Updated {sync_count} repositories.")
+        elif args.verbose:
+            log("No changes detected in this cycle.")
+
+        if not args.watch:
+            break
+            
+        time.sleep(args.interval)
+
+if __name__ == "__main__":
+    main()

holocron/config.py

@@ -0,0 +1,36 @@
+import os
+import argparse
+from dotenv import load_dotenv
+
+# Load env vars from .env file
+load_dotenv()
+
+# --- CONFIGURATION DEFAULTS ---
+# We use Environment Variables for security. 
+# Never hardcode passwords in open source code!
+GITHUB_API_URL = "https://api.github.com"
+GITLAB_API_URL = os.environ.get("GITLAB_API_URL", "http://gitlab.local/api/v4")
+
+def parse_args():
+    """
+    Sets up the command line arguments.
+    This allows the user to run: 'python g2g.py --dry-run'
+    """
+    parser = argparse.ArgumentParser(
+        description="Holocron: GitHub to GitLab/Local Mirroring Tool"
+    )
+    
+    # Flags (True/False options)
+    parser.add_argument("--dry-run", action="store_true", help="Simulate execution without making changes")
+    parser.add_argument("--watch", action="store_true", help="Run continuously in a loop (Daemon mode)")
+    parser.add_argument("--verbose", action="store_true", help="Print detailed logs")
+    
+    # value options
+    parser.add_argument("--interval", type=int, default=60, help="Seconds to wait between checks (default: 60)")
+    parser.add_argument("--window", type=int, default=10, help="Only sync repos updated in the last X minutes")
+    parser.add_argument("--storage", type=str, default="./mirror-data", help="Local path to store git repositories")
+    parser.add_argument("--concurrency", type=int, default=5, help="Number of concurrent sync threads (default: 5)")
+    parser.add_argument("--backup-only", action="store_true", help="Mirror locally only, skip pushing to GitLab")
+    parser.add_argument("--checkout", action="store_true", help="Create a checkout of the repository alongside the mirror")
+
+    return parser.parse_args()

holocron/github_provider.py

@@ -0,0 +1,99 @@
+import requests
+from .logger import log
+from .config import GITHUB_API_URL
+
+def get_all_pages(base_url, headers, verbose, context_name, query_params=None):
+    """Helper to fetch all pages from a GitHub endpoint."""
+    if query_params is None:
+        query_params = {}
+        
+    items = []
+    page = 1
+    query_params['per_page'] = 100
+    
+    log(f"Fetching {context_name}...", is_verbose_mode=verbose)
+    
+    while True:
+        try:
+            # Update page number in params
+            query_params['page'] = page
+            
+            if verbose:
+                # Log the actual request for debugging
+                # (Masking token if we were printing headers, but we aren't)
+                log(f"DEBUG: Requesting page {page} from {base_url} with params {query_params}", verbose_only=True, is_verbose_mode=True)
+
+            r = requests.get(base_url, headers=headers, params=query_params, timeout=20)
+            r.raise_for_status()
+            
+            data = r.json()
+            if not data:
+                if verbose:
+                    log(f"DEBUG: Page {page} empty. stopping.", verbose_only=True, is_verbose_mode=True)
+                break
+            
+            count = len(data)
+            if verbose:
+                log(f"DEBUG: Page {page} returned {count} items.", verbose_only=True, is_verbose_mode=True)
+                
+            items.extend(data)
+            
+            # Optimization: If we got fewer than per_page, we are done
+            if count < query_params['per_page']:
+                break
+                
+            page += 1
+        except Exception as e:
+            log(f"ERROR fetching {context_name}: {e}")
+            break
+    return items
+
+def get_github_repos(token, verbose):
+    """Fetches all repositories from the user AND their organizations."""
+    headers = {'Authorization': f'token {token}'}
+    all_repos = []
+    seen_ids = set()
+
+    # 1. Fetch User Repos (includes explicit access to outside org repos)
+    # Using specific parameters as per GitHub docs to ensure maximal coverage
+    user_repos = get_all_pages(
+        f"{GITHUB_API_URL}/user/repos", 
+        headers, 
+        verbose, 
+        "user repositories (visibility=all, all affiliations)",
+        query_params={
+            "visibility": "all",
+            "affiliation": "owner,collaborator,organization_member"
+        }
+    )
+    
+    for repo in user_repos:
+        if repo['id'] not in seen_ids:
+            all_repos.append(repo)
+            seen_ids.add(repo['id'])
+
+    # 2. Fetch User Organizations
+    orgs = get_all_pages(
+        f"{GITHUB_API_URL}/user/orgs", 
+        headers, 
+        verbose, 
+        "organizations"
+    )
+
+    # 3. Fetch Repos for each Org
+    for org in orgs:
+        org_name = org['login']
+        # type=all includes forks, private, public
+        org_repos = get_all_pages(
+            f"{GITHUB_API_URL}/orgs/{org_name}/repos",
+            headers,
+            verbose,
+            f"repositories for organization '{org_name}'",
+            query_params={"type": "all"}
+        )
+        for repo in org_repos:
+            if repo['id'] not in seen_ids:
+                all_repos.append(repo)
+                seen_ids.add(repo['id'])
+                
+    return all_repos

holocron/logger.py

@@ -0,0 +1,8 @@
+from datetime import datetime
+
+def log(message, verbose_only=False, is_verbose_mode=False):
+    """Simple logger that checks if we are in verbose mode."""
+    if verbose_only and not is_verbose_mode:
+        return
+    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    print(f"[{timestamp}] {message}")

holocron/mirror.py

@@ -0,0 +1,111 @@
+import os
+import subprocess
+from datetime import datetime, timedelta, timezone
+from .logger import log
+from .config import GITLAB_API_URL
+
+def needs_sync(repo, window_minutes):
+    """
+    Returns True if the repo was pushed to within the 'window_minutes'.
+    """
+    pushed_at_str = repo.get('pushed_at')
+    if not pushed_at_str:
+        return False
+        
+    # Parse ISO timestamp (Handles the 'Z' for UTC)
+    pushed_at = datetime.fromisoformat(pushed_at_str.replace('Z', '+00:00'))
+    now = datetime.now(timezone.utc)
+    
+    # Check if the difference is inside our window
+    return (now - pushed_at) < timedelta(minutes=window_minutes)
+
+def sync_one_repo(repo, args, github_token, gitlab_token):
+    name = repo['name']
+    repo_dir = os.path.join(args.storage, f"{name}.git")
+    
+    # 1. Construct Secure URLs (Injecting tokens)
+    # Note: We use the OAuth2 syntax for GitLab
+    gh_clone_url = repo['clone_url'].replace("https://", f"https://oauth2:{github_token}@")
+    
+    # We assume the GitLab group matches the GitHub username or is defined in the URL
+    # For this script, we construct a generic target URL. 
+    # In a real scenario, you might want to customize the group logic.
+    if not args.backup_only:
+        gl_target_url = f"{GITLAB_API_URL.replace('/api/v4', '')}/{name}.git".replace("http://", f"http://oauth2:{gitlab_token}@")
+    else:
+        gl_target_url = None
+
+    # 2. Dry Run Check
+    if args.dry_run:
+        target_msg = gl_target_url if not args.backup_only else "(Local Backup Only)"
+        log(f"[DRY-RUN] Would sync '{name}' -> '{target_msg}'")
+        return
+
+    # 3. Create Storage Directory if needed
+    os.makedirs(args.storage, exist_ok=True)
+
+    # 4. Git Operations
+    try:
+        # Use --quiet to suppress progress bars which mess up parallel logs
+        # We also redirect stderr to DEVNULL to be completely silent unless it fails (which check=True handles by raising)
+        # Note: If git fails, we won't see the error message if we silence stderr perfectly. 
+        # A better approach is capture_output=True and log stderr on error.
+        
+        if not os.path.exists(repo_dir):
+            log(f"[{name}] Cloning new mirror...")
+            try:
+                subprocess.run(["git", "clone", "--mirror", "--quiet", gh_clone_url, repo_dir], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+            except subprocess.CalledProcessError as e:
+                # Decode stderr for the log
+                err_msg = e.stderr.decode().strip() if e.stderr else str(e)
+                raise subprocess.CalledProcessError(e.returncode, e.cmd, output=e.output, stderr=err_msg)
+
+        else:
+            log(f"[{name}] Fetching updates...", verbose_only=True, is_verbose_mode=args.verbose)
+            try:
+                subprocess.run(["git", "-C", repo_dir, "fetch", "--quiet", "-p", "origin"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+            except subprocess.CalledProcessError as e:
+                err_msg = e.stderr.decode().strip() if e.stderr else str(e)
+                raise subprocess.CalledProcessError(e.returncode, e.cmd, output=e.output, stderr=err_msg)
+        
+        # If not backup only, ensure push remote is set (optional but good practice if it changes)
+        if not args.backup_only:
+             subprocess.run(["git", "-C", repo_dir, "remote", "set-url", "--push", "origin", gl_target_url], check=True, stderr=subprocess.DEVNULL)
+
+        # 5. Push to GitLab
+        # We always push to ensure the mirror is exact, UNLESS backup-only mode is on
+        if not args.backup_only:
+            try:
+                subprocess.run(["git", "-C", repo_dir, "push", "--mirror", "--quiet"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+                log(f"[{name}] Successfully synced to GitLab.")
+            except subprocess.CalledProcessError as e:
+                err_msg = e.stderr.decode().strip() if e.stderr else str(e)
+                raise subprocess.CalledProcessError(e.returncode, e.cmd, output=e.output, stderr=err_msg)
+        else:
+            log(f"[{name}] Successfully backed up locally.")
+        
+        # 6. Optional Checkout (Sidecar)
+        if args.checkout:
+            # Derived directly from repo_dir (which ends in .git)
+            checkout_dir = repo_dir.replace(".git", "")
+            
+            if not os.path.exists(checkout_dir):
+                log(f"[{name}] Creating checkout...", verbose_only=True, is_verbose_mode=args.verbose)
+                try:
+                    # Clone from the local mirror
+                    subprocess.run(["git", "clone", "--quiet", repo_dir, checkout_dir], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+                except subprocess.CalledProcessError as e:
+                    err_msg = e.stderr.decode().strip() if e.stderr else str(e)
+                    log(f"[{name}] Failed to create checkout: {err_msg}")
+            else:
+                log(f"[{name}] Updating checkout...", verbose_only=True, is_verbose_mode=args.verbose)
+                try:
+                    # We use pull to update the current branch. 
+                    # If this fails (e.g. merge conflict, though unlikely for a backup), we catch it.
+                    subprocess.run(["git", "-C", checkout_dir, "pull", "--quiet"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+                except subprocess.CalledProcessError as e:
+                    err_msg = e.stderr.decode().strip() if e.stderr else str(e)
+                    log(f"[{name}] Failed to update checkout: {err_msg}")
+        
+    except subprocess.CalledProcessError as e:
+        log(f"ERROR syncing {name}: {e}")

holocron_sync-0.1.0.dist-info/METADATA

@@ -0,0 +1,84 @@
+Metadata-Version: 2.4
+Name: holocron-sync
+Version: 0.1.0
+Summary: Holocron: The Ultimate Git Mirroring Tool
+License-File: LICENSE
+Requires-Python: >=3.9
+Requires-Dist: python-dotenv
+Requires-Dist: requests>=2.28.0
+Description-Content-Type: text/markdown
+
+# Holocron
+> **The "Ultimate" Git Mirroring Tool**
+
+```
+          /\
+         /  \
+        / /\ \
+       / /  \ \
+      / /    \ \
+     /_/______\_\
+     \ \      / /
+      \ \    / /
+       \ \  / /
+        \ \/ /
+         \  /
+          \/
+```
+
+**Holocron** is a powerful Python application designed to mirror your GitHub repositories to a local directory or a self-hosted GitLab instance. It supports parallel syncing, continuous watch mode, and local-only backups (no GitLab required).
+
+## Features
+- **Parallel Syncing**: Sync multiple repositories concurrently for maximum speed.
+- **Continuous Watch Mode**: Polls for changes and syncs only when necessary (smart redundancy checks).
+- **Two-Way Mirroring**: Creates a bare mirror (`.git` folder) for safety AND an optional checkout for visibility.
+- **Dockerized**: Runs as a lightweight container.
+- **Backup Level**: 
+    - Full: GitHub -> Local -> GitLab
+    - Backup-Only: GitHub -> Local (No GitLab token needed)
+
+## Quick Start
+
+### 1. Docker (Recommended)
+```bash
+docker-compose up -d --build
+```
+This starts Holocron in watch mode.
+
+### 2. Manual Run
+```bash
+# Install dependencies
+uv sync
+
+# Run a one-time backup of all your repos locally (visible files)
+export GITHUB_TOKEN=your_token
+uv run python src/holocron.py --backup-only --checkout --concurrency 10
+```
+
+## Configuration
+Holocron uses environment variables for secrets:
+
+| Variable | Description | Required | 
+| :--- | :--- | :--- |
+| `GITHUB_TOKEN` | Your GitHub Personal Access Token (repo scope) | **Yes** |
+| `GITLAB_TOKEN` | Your GitLab Personal Access Token (api scope) | No (if `--backup-only`) |
+| `GITLAB_API_URL` | URL to your GitLab API (default: `http://gitlab.local/api/v4`) | No |
+
+### Command Line Arguments
+| Flag | Default | Description |
+| :--- | :--- | :--- |
+| `--watch` | False | Run continuously in a loop |
+| `--interval` | 60 | Seconds to sleep between checks in watch mode |
+| `--window` | 60 | Sync only repos pushed within the last N minutes |
+| `--backup-only` | False | Mirror locally only, do not push to GitLab |
+| `--checkout` | False | Create a visible working directory alongside the mirror |
+| `--concurrency` | 5 | Number of parallel sync threads |
+| `--storage` | `./mirror-data` | Directory to store repositories |
+| `--dry-run` | False | Print what would happen without doing it |
+| `--verbose` | False | Enable detailed debug logging |
+
+## Development
+Run tests with coverage:
+```bash
+uv run pytest --cov=src
+```

holocron_sync-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+holocron/__init__.py,sha256=9jRVm2gCiDxGHfum8z9JBVNCkzjdCHJ0OApdD8C7SiM,47
+holocron/__main__.py,sha256=Lbk1n_Lwz9BoqX5UUUsI0O4jziTQR1Sh9uHLNAiwwWo,2988
+holocron/config.py,sha256=6ixFtSPGoK0bOilKLcnJmMDyhjmomWBYdfPCwAtUuew,1692
+holocron/github_provider.py,sha256=FS7gDAa_JcMBVRWEkabqh2JsNwqXTeKzBdlp7hRAkCg,3271
+holocron/logger.py,sha256=bLspdJUWhPelKN72DhJAT8jgZuRkWfqF0IiDaacONfA,314
+holocron/mirror.py,sha256=KnolJYwZ02uRIdHJXFmDeB4xgXeidpLdlR3pPMBay7A,5733
+holocron_sync-0.1.0.dist-info/METADATA,sha256=fF35ERZPFooA_B8gT1OWh7hqVUYL2ynMrMyzP-mYk88,2734
+holocron_sync-0.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+holocron_sync-0.1.0.dist-info/entry_points.txt,sha256=9nwV7AF3lX03yqL64yAO-RF0Uy3tBHaZm57JlKecOSY,52
+holocron_sync-0.1.0.dist-info/licenses/LICENSE,sha256=VEobrYN3vjG9NON0lbc7JBIFo6rFe9MY1MuGv6U4F0M,1072
+holocron_sync-0.1.0.dist-info/RECORD,,

holocron_sync-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.28.0
+Root-Is-Purelib: true
+Tag: py3-none-any

holocron_sync-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+holocron = holocron.__main__:main

holocron_sync-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Wouter Bloeyaert
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.