holocron-sync 0.1.0__tar.gz

holocron_sync-0.1.0/.dockerignore

@@ -0,0 +1,7 @@
+mirror-data/
+.venv/
+.git/
+__pycache__/
+.env
+.DS_Store
+*.pyc

holocron_sync-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,31 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "0.4.15"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: uv sync --all-extras --dev
+
+      - name: Run tests
+        run: uv run pytest

holocron_sync-0.1.0/.github/workflows/publish.yaml

@@ -0,0 +1,27 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  pypi-publish:
+    name: Upload to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/holocron-sync
+    permissions:
+      id-token: write # IMPORTANT: this permission is mandatory for Trusted Publishing
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Build package
+        run: uv build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

holocron_sync-0.1.0/.gitignore

@@ -0,0 +1,61 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution / Packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Virtual Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Testing and Coverage
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# G2G-Sync / Holocron specific
+mirror-data/
+*.git
+
+# Environment Variables
+.env
+
+# MacOS
+.DS_Store
+
+# IDEs
+.idea/
+.vscode/
+*.swp
+*.swo

holocron_sync-0.1.0/Dockerfile

@@ -0,0 +1,22 @@
+FROM python:3.12-alpine
+
+# Install git (required for mirroring)
+RUN apk add --no-cache git
+
+WORKDIR /app
+
+# Copy requirements first to leverage cache
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy source code
+COPY src/ src/
+
+# Create directory for mirror data
+RUN mkdir -p mirror-data
+
+# Default entrypoint
+ENTRYPOINT ["python", "-m", "holocron"]
+
+# Default command (can be overridden)
+CMD ["--watch", "--interval", "60"]

holocron_sync-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Wouter Bloeyaert
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

holocron_sync-0.1.0/PKG-INFO

@@ -0,0 +1,84 @@
+Metadata-Version: 2.4
+Name: holocron-sync
+Version: 0.1.0
+Summary: Holocron: The Ultimate Git Mirroring Tool
+License-File: LICENSE
+Requires-Python: >=3.9
+Requires-Dist: python-dotenv
+Requires-Dist: requests>=2.28.0
+Description-Content-Type: text/markdown
+
+# Holocron
+> **The "Ultimate" Git Mirroring Tool**
+
+```
+          /\
+         /  \
+        / /\ \
+       / /  \ \
+      / /    \ \
+     /_/______\_\
+     \ \      / /
+      \ \    / /
+       \ \  / /
+        \ \/ /
+         \  /
+          \/
+```
+
+**Holocron** is a powerful Python application designed to mirror your GitHub repositories to a local directory or a self-hosted GitLab instance. It supports parallel syncing, continuous watch mode, and local-only backups (no GitLab required).
+
+## Features
+- **Parallel Syncing**: Sync multiple repositories concurrently for maximum speed.
+- **Continuous Watch Mode**: Polls for changes and syncs only when necessary (smart redundancy checks).
+- **Two-Way Mirroring**: Creates a bare mirror (`.git` folder) for safety AND an optional checkout for visibility.
+- **Dockerized**: Runs as a lightweight container.
+- **Backup Level**: 
+    - Full: GitHub -> Local -> GitLab
+    - Backup-Only: GitHub -> Local (No GitLab token needed)
+
+## Quick Start
+
+### 1. Docker (Recommended)
+```bash
+docker-compose up -d --build
+```
+This starts Holocron in watch mode.
+
+### 2. Manual Run
+```bash
+# Install dependencies
+uv sync
+
+# Run a one-time backup of all your repos locally (visible files)
+export GITHUB_TOKEN=your_token
+uv run python src/holocron.py --backup-only --checkout --concurrency 10
+```
+
+## Configuration
+Holocron uses environment variables for secrets:
+
+| Variable | Description | Required | 
+| :--- | :--- | :--- |
+| `GITHUB_TOKEN` | Your GitHub Personal Access Token (repo scope) | **Yes** |
+| `GITLAB_TOKEN` | Your GitLab Personal Access Token (api scope) | No (if `--backup-only`) |
+| `GITLAB_API_URL` | URL to your GitLab API (default: `http://gitlab.local/api/v4`) | No |
+
+### Command Line Arguments
+| Flag | Default | Description |
+| :--- | :--- | :--- |
+| `--watch` | False | Run continuously in a loop |
+| `--interval` | 60 | Seconds to sleep between checks in watch mode |
+| `--window` | 60 | Sync only repos pushed within the last N minutes |
+| `--backup-only` | False | Mirror locally only, do not push to GitLab |
+| `--checkout` | False | Create a visible working directory alongside the mirror |
+| `--concurrency` | 5 | Number of parallel sync threads |
+| `--storage` | `./mirror-data` | Directory to store repositories |
+| `--dry-run` | False | Print what would happen without doing it |
+| `--verbose` | False | Enable detailed debug logging |
+
+## Development
+Run tests with coverage:
+```bash
+uv run pytest --cov=src
+```

holocron_sync-0.1.0/README.md

@@ -0,0 +1,74 @@
+# Holocron
+> **The "Ultimate" Git Mirroring Tool**
+
+```
+          /\
+         /  \
+        / /\ \
+       / /  \ \
+      / /    \ \
+     /_/______\_\
+     \ \      / /
+      \ \    / /
+       \ \  / /
+        \ \/ /
+         \  /
+          \/
+```
+
+**Holocron** is a powerful Python application designed to mirror your GitHub repositories to a local directory or a self-hosted GitLab instance. It supports parallel syncing, continuous watch mode, and local-only backups (no GitLab required).
+
+## Features
+- **Parallel Syncing**: Sync multiple repositories concurrently for maximum speed.
+- **Continuous Watch Mode**: Polls for changes and syncs only when necessary (smart redundancy checks).
+- **Two-Way Mirroring**: Creates a bare mirror (`.git` folder) for safety AND an optional checkout for visibility.
+- **Dockerized**: Runs as a lightweight container.
+- **Backup Level**: 
+    - Full: GitHub -> Local -> GitLab
+    - Backup-Only: GitHub -> Local (No GitLab token needed)
+
+## Quick Start
+
+### 1. Docker (Recommended)
+```bash
+docker-compose up -d --build
+```
+This starts Holocron in watch mode.
+
+### 2. Manual Run
+```bash
+# Install dependencies
+uv sync
+
+# Run a one-time backup of all your repos locally (visible files)
+export GITHUB_TOKEN=your_token
+uv run python src/holocron.py --backup-only --checkout --concurrency 10
+```
+
+## Configuration
+Holocron uses environment variables for secrets:
+
+| Variable | Description | Required | 
+| :--- | :--- | :--- |
+| `GITHUB_TOKEN` | Your GitHub Personal Access Token (repo scope) | **Yes** |
+| `GITLAB_TOKEN` | Your GitLab Personal Access Token (api scope) | No (if `--backup-only`) |
+| `GITLAB_API_URL` | URL to your GitLab API (default: `http://gitlab.local/api/v4`) | No |
+
+### Command Line Arguments
+| Flag | Default | Description |
+| :--- | :--- | :--- |
+| `--watch` | False | Run continuously in a loop |
+| `--interval` | 60 | Seconds to sleep between checks in watch mode |
+| `--window` | 60 | Sync only repos pushed within the last N minutes |
+| `--backup-only` | False | Mirror locally only, do not push to GitLab |
+| `--checkout` | False | Create a visible working directory alongside the mirror |
+| `--concurrency` | 5 | Number of parallel sync threads |
+| `--storage` | `./mirror-data` | Directory to store repositories |
+| `--dry-run` | False | Print what would happen without doing it |
+| `--verbose` | False | Enable detailed debug logging |
+
+## Development
+Run tests with coverage:
+```bash
+uv run pytest --cov=src
+```

holocron_sync-0.1.0/agents.MD

@@ -0,0 +1,178 @@
+# AGENTS.md
+
+## 📌 Purpose
+
+This document defines contribution standards for both humans and AI agents working in this repo.
+It enforces **consistency, security, and maintainability** across the codebase.
+Think of this as our **style guide + quality contract**.
+
+---
+
+## 🔐 Security
+
+* Treat all inputs as untrusted — always validate and sanitize.
+* Avoid unsafe patterns:
+
+  * Shell exec, `eval`, unverified deserialization, raw SQL.
+* Enforce explicit schemas (Pydantic, Django models, Marshmallow).
+* Never expose secrets, API keys, or tokens (zero-trust assumption).
+* Run linters, type checks, and tests before merging.
+
+---
+
+## 🧱 Code Design Principles
+
+* **Clarity > Cleverness**: prioritize readability over conciseness.
+* **Small functions**: ≤ 20 lines (target 10–15).
+* **Single responsibility**: each function/class does one thing well.
+* **Explicit over implicit**:
+
+  * Avoid dense one-liners, hidden side effects, and tricky language features.
+  * Prefer explicit `if` over nested ternaries.
+  * Limit call chaining to max 2 steps.
+* **DRY but not obsessed**: reuse logic, but don’t abstract prematurely.
+* **Consistent naming**: descriptive, not abbreviated.
+* **Remove dead code**: no commented-out blocks or unused imports.
+* **Documentation at point of use**: comments explain *why*, not *what*.
+
+**Django specifics**:
+
+* Use `bulk_create`/`bulk_update` for efficiency.
+* Always define `UPDATE_FIELDS` constants for updates.
+* Wrap multi-step writes in transactions.
+* Cache repeated upserts in memory during a run.
+
+---
+
+## 🧰 Helpers & Utilities
+
+* Keep helpers **local** if only used once; mark them private (`_helper`).
+* Extract to `api/utils/<purpose>.py` if reused.
+* Avoid generic “misc” utils files.
+* Public helpers: type hints + docstrings; side-effect free.
+* Private helpers: `_parse_*`, `_build_*`, `_apply_*`.
+* Migration rule: start local, extract only when reused or file > 300 lines.
+
+---
+
+## 🧩 Models
+
+* **One file per model**: strict rule.
+
+  * Example:
+
+    * `api/models/ticker.py` → `Ticker`
+    * `api/models/exchange.py` → `Exchange`
+    * `api/models/job.py` → `Job`
+* **Re-export** in `api/models/__init__.py`:
+
+  ```python
+  from .ticker import Ticker
+  from .exchange import Exchange
+  from .job import Job
+
+  __all__ = ["Ticker", "Exchange", "Job"]
+  ```
+* No side effects in model files (signals, listeners go elsewhere).
+* Admin config belongs in `admin.py`.
+* Tests mirror model file structure.
+
+---
+
+## ✅ Testing
+
+* Every new feature requires tests. **No exceptions.**
+* Cover: CRUD, edge cases, abuse/misuse, success paths.
+* Tests must be deterministic and fast (<1s per unit test).
+* Use integration tests only when needed.
+* Test naming: `test_<functionality>_<expected_behavior>`.
+* Test location: `/tests/` mirroring code layout.
+
+---
+
+## 🧩 Extensibility
+
+* No hardcoded credentials/config.
+* Use base classes/interfaces for provider-specific logic.
+* Prefer loose coupling; modules should be swappable.
+
+---
+
+## 🗂️ File & Folder Conventions
+
+* Each model = its own file under `src/recipes/models/`.
+* No placeholder/empty boilerplate files.
+* Organize by domain, not by layer (avoid `misc.py`, `common.py`).
+
+---
+
+## 🧾 Documentation
+
+* All public classes and functions need docstrings.
+* Comments should clarify *why* decisions were made.
+* Update Markdown/docs when behavior changes.
+
+---
+
+## 💬 Commits & PRs
+
+* **Commit title format**: `[scope] Short summary`
+
+  * Example: `[auth] Add password reset expiry`
+* Messages must explain:
+
+  * **What** changed
+  * **Why** it’s needed
+  * **How** to test it
+* Keep commits small and focused (atomic).
+* No auto-generated files in commits.
+
+---
+
+## 🤖 Agents (AI Contribution Rules)
+
+* Respect this document before generating code.
+* Reuse existing patterns and naming.
+* Never assume frameworks/libraries not already in the repo.
+* Ensure functions are short, explicit, and modular.
+* Include docstrings and type hints.
+* Run security and readability checks before suggesting changes.
+
+---
+
+## 🔍 Code Review Checklist
+
+* [ ] Functions ≤ 20 lines; single responsibility
+* [ ] No dense inline logic, no multi-statement lines
+* [ ] Call chaining ≤ 2 steps
+* [ ] DB ops isolated; transactions around multi-step writes
+* [ ] Bulk ops with `UPDATE_FIELDS`
+* [ ] No dead code; imports trimmed
+* [ ] Comments explain *why*
+* [ ] Tests added/updated (CRUD, edge, abuse, success paths)
+* [ ] No secrets/unsafe code
+* [ ] Docs updated
+
+---
+
+## 🛠️ PR Template
+
+`.github/pull_request_template.md`:
+
+```markdown
+## Summary
+- What does this change do?
+
+## Checklist
+- [ ] Functions ≤ 20 lines
+- [ ] Single responsibility per function/class
+- [ ] No chained one-liners or dense inline ternaries
+- [ ] Intermediate variables for clarity
+- [ ] DB I/O isolated; transactions for multi-step writes
+- [ ] Narrow exceptions (no bare `except`)
+- [ ] Docstrings + type hints present
+- [ ] `UPDATE_FIELDS` constant used for bulk updates
+- [ ] Tests mirror module layout and cover edge cases
+
+## 🧪 Test & Quality Tools (Backend)
+All backend test and quality tooling is executed from the project root and targets the backend codebase in `src`. Configuration files for each tool live in the backend code root (`src`).

holocron_sync-0.1.0/docker-compose.yml

@@ -0,0 +1,11 @@
+services:
+  holocron:
+    build: .
+    container_name: holocron
+    restart: unless-stopped
+    volumes:
+      - ./mirror-data:/app/mirror-data
+    env_file:
+      - .env
+    # Optional: Override command
+    # command: ["--watch", "--interval", "30", "--backup-only"]

holocron_sync-0.1.0/pyproject.toml

@@ -0,0 +1,32 @@
+[project]
+name = "holocron-sync"
+version = "0.1.0"
+description = "Holocron: The Ultimate Git Mirroring Tool"
+readme = "README.md"
+requires-python = ">=3.9"
+dependencies = [
+    "requests>=2.28.0",
+    "python-dotenv",
+]
+
+[project.scripts]
+holocron = "holocron.__main__:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/holocron"]
+
+[tool.uv]
+dev-dependencies = [
+    "pytest>=8.0.0",
+    "pytest-mock>=3.12.0",
+    "pytest-cov>=4.1.0",
+]
+
+[tool.pytest.ini_options]
+pythonpath = "src"
+norecursedirs = ["mirror-data", ".venv", ".git"]
+testpaths = ["tests"]

holocron_sync-0.1.0/requirements.txt

@@ -0,0 +1,2 @@
+requests>=2.28.0
+python-dotenv

holocron_sync-0.1.0/src/holocron/__init__.py

@@ -0,0 +1,3 @@
+from .__main__ import main
+
+__all__ = ["main"]

holocron_sync-0.1.0/src/holocron/__main__.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+import os
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+# Import from local modules
+from .config import parse_args
+from .logger import log
+from .github_provider import get_github_repos
+from .mirror import needs_sync, sync_one_repo
+
+def main():
+    args = parse_args()
+    
+    # Load secrets
+    gh_token = os.environ.get("GITHUB_TOKEN")
+    gl_token = os.environ.get("GITLAB_TOKEN")
+    
+    # Validation logic
+    if not gh_token:
+        print("CRITICAL: Missing GITHUB_TOKEN.")
+        sys.exit(1)
+
+    if not args.backup_only and not gl_token:
+        print("CRITICAL: Missing GITLAB_TOKEN.")
+        print("Please set GITLAB_TOKEN or use --backup-only.")
+        sys.exit(1)
+
+    log("Initializing Holocron...")
+    if args.dry_run:
+        log("!!! DRY RUN MODE ACTIVE !!!")
+
+    # Track the last synced push timestamp for each repo
+    synced_pushes = {}
+
+    while True:
+        repos = get_github_repos(gh_token, args.verbose)
+        log(f"Found {len(repos)} repositories on GitHub.", is_verbose_mode=args.verbose)
+
+        sync_count = 0
+        with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
+            future_to_repo = {}
+            for repo in repos:
+                repo_name = repo['name']
+                pushed_at = repo.get('pushed_at')
+                repo_dir = os.path.join(args.storage, f"{repo_name}.git")
+
+                # If watching, use the smart filter. If running once, sync all.
+                if args.watch:
+                    # 1. Skip if we already synced this exact push
+                    if repo_name in synced_pushes and synced_pushes[repo_name] == pushed_at:
+                        continue
+                    
+                    # 2. Check time window (SKIP if old AND local repo exists)
+                    # If local repo is missing, we MUST sync (bootstrap), regardless of age.
+                    if os.path.exists(repo_dir) and not needs_sync(repo, args.window):
+                         continue
+                    
+                future = executor.submit(sync_one_repo, repo, args, gh_token, gl_token)
+                future_to_repo[future] = repo
+
+            for future in as_completed(future_to_repo):
+                repo = future_to_repo[future]
+                try:
+                    future.result()
+                    sync_count += 1
+                    # Update tracking on success
+                    if repo.get('pushed_at'):
+                        synced_pushes[repo['name']] = repo['pushed_at']
+                except Exception as exc:
+                    log(f"[{repo['name']}] generated an exception: {exc}")
+
+        if sync_count > 0:
+            log(f"Sync cycle complete. Updated {sync_count} repositories.")
+        elif args.verbose:
+            log("No changes detected in this cycle.")
+
+        if not args.watch:
+            break
+            
+        time.sleep(args.interval)
+
+if __name__ == "__main__":
+    main()

holocron_sync-0.1.0/src/holocron/config.py

@@ -0,0 +1,36 @@
+import os
+import argparse
+from dotenv import load_dotenv
+
+# Load env vars from .env file
+load_dotenv()
+
+# --- CONFIGURATION DEFAULTS ---
+# We use Environment Variables for security. 
+# Never hardcode passwords in open source code!
+GITHUB_API_URL = "https://api.github.com"
+GITLAB_API_URL = os.environ.get("GITLAB_API_URL", "http://gitlab.local/api/v4")
+
+def parse_args():
+    """
+    Sets up the command line arguments.
+    This allows the user to run: 'python g2g.py --dry-run'
+    """
+    parser = argparse.ArgumentParser(
+        description="Holocron: GitHub to GitLab/Local Mirroring Tool"
+    )
+    
+    # Flags (True/False options)
+    parser.add_argument("--dry-run", action="store_true", help="Simulate execution without making changes")
+    parser.add_argument("--watch", action="store_true", help="Run continuously in a loop (Daemon mode)")
+    parser.add_argument("--verbose", action="store_true", help="Print detailed logs")
+    
+    # value options
+    parser.add_argument("--interval", type=int, default=60, help="Seconds to wait between checks (default: 60)")
+    parser.add_argument("--window", type=int, default=10, help="Only sync repos updated in the last X minutes")
+    parser.add_argument("--storage", type=str, default="./mirror-data", help="Local path to store git repositories")
+    parser.add_argument("--concurrency", type=int, default=5, help="Number of concurrent sync threads (default: 5)")
+    parser.add_argument("--backup-only", action="store_true", help="Mirror locally only, skip pushing to GitLab")
+    parser.add_argument("--checkout", action="store_true", help="Create a checkout of the repository alongside the mirror")
+
+    return parser.parse_args()