mirror-dedupe 0.1.0__tar.gz

mirror_dedupe-0.1.0/.github/workflows/build-deb.yml

@@ -0,0 +1,44 @@
+#
+# build-deb.yml : Ubuntu mirror synchronisation with global deduplication
+#
+# Copyright (c) 2025 Tim Hosking
+# Email: tim@mungerware.com
+# Website: https://github.com/munger
+# Licence: MIT
+#
+
+name: Build Debian Package
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+  build-deb:
+    runs-on: ubuntu-latest
+    
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Install build dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y debhelper dh-python python3-all python3-setuptools python3-yaml
+    
+    - name: Build Debian package
+      run: dpkg-buildpackage -us -uc -b
+    
+    - name: Upload artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: debian-package
+        path: ../mirror-dedupe_*.deb
+    
+    - name: Upload to release
+      if: github.event_name == 'release'
+      uses: softprops/action-gh-release@v1
+      with:
+        files: ../mirror-dedupe_*.deb
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

mirror_dedupe-0.1.0/.github/workflows/publish-pypi.yml

@@ -0,0 +1,42 @@
+#
+# publish-pypi.yml : Ubuntu mirror synchronisation with global deduplication
+#
+# Copyright (c) 2025 Tim Hosking
+# Email: tim@mungerware.com
+# Website: https://github.com/munger
+# Licence: MIT
+#
+
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+    
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+    
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build twine
+    
+    - name: Build package
+      run: python -m build
+    
+    - name: Publish to PyPI
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+      run: twine upload dist/*

mirror_dedupe-0.1.0/.gitignore

@@ -0,0 +1,52 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+venv/
+ENV/
+env/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+tests/test-config/
+
+# Debian packaging
+debian/files
+debian/*.debhelper
+debian/*.log
+debian/*.substvars
+debian/mirror-dedupe/
+debian/.debhelper/
+
+# Build artifacts
+*.deb
+*.buildinfo
+*.changes

mirror_dedupe-0.1.0/MANIFEST.in

@@ -0,0 +1,7 @@
+include README.md
+include LICENSE
+include REFACTORING.md
+recursive-include config *.conf *.example
+recursive-include systemd *.service *.timer
+recursive-include debian mirror-dedupe.1
+recursive-include nginx *.conf *.html *.css

mirror_dedupe-0.1.0/PKG-INFO

@@ -0,0 +1,137 @@
+Metadata-Version: 2.4
+Name: mirror-dedupe
+Version: 0.1.0
+Summary: Ubuntu mirror synchronisation with global deduplication
+Home-page: https://github.com/munger/mirror-dedupe
+Author: Tim Hosking
+Author-email: Tim Hosking <tim@mungerware.com>
+License: MIT
+Project-URL: Homepage, https://github.com/munger/mirror-dedupe
+Project-URL: Bug Tracker, https://github.com/munger/mirror-dedupe/issues
+Project-URL: Source Code, https://github.com/munger/mirror-dedupe
+Keywords: mirror,deduplication,hardlink,ubuntu,repository,apt
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: System Administrators
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: System :: Archiving :: Mirroring
+Classifier: Topic :: System :: Systems Administration
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: PyYAML>=6.0
+Dynamic: author
+Dynamic: home-page
+Dynamic: requires-python
+
+<!--
+README.md : Ubuntu mirror synchronisation with global deduplication
+
+Copyright (c) 2025 Tim Hosking
+Email: tim@mungerware.com
+Website: https://github.com/munger
+Licence: MIT
+-->
+
+Ubuntu/Debian mirror synchronisation with intelligent deduplication using hardlinks.
+
+## Features
+
+- Downloads from upstream and hardlinks duplicate files (same SHA256 hash) to save bandwidth and disk space
+- Supports multiple mirrors with global deduplication across all mirrors
+- Uses curl for duplicate files, rsync for unique files and metadata
+- Configurable via YAML files
+- Systemd timer support for automated synchronisation
+
+## Installation
+
+### Option 1: Debian/Ubuntu Package (Recommended)
+
+Download the latest `.deb` package from [GitHub Releases](https://github.com/munger/mirror-dedupe/releases):
+
+```bash
+wget https://github.com/munger/mirror-dedupe/releases/download/v0.1.0/mirror-dedupe_0.1.0-1_all.deb
+sudo dpkg -i mirror-dedupe_0.1.0-1_all.deb
+```
+
+This includes systemd integration, man pages, and proper package management.
+
+### Option 2: PyPI (All Linux Distributions)
+
+```bash
+pip install mirror-dedupe
+```
+
+Then install systemd files manually:
+
+```bash
+sudo ./install.sh --pip
+```
+
+### Option 3: From Source
+
+```bash
+git clone https://github.com/munger/mirror-dedupe.git
+cd mirror-dedupe
+sudo ./install.sh
+```
+
+## Configuration
+
+Configuration files are located in `/etc/mirror-dedupe/`:
+
+- `mirror-dedupe.conf` - Global settings
+- `repos.d/*.conf` - Individual repository definitions
+
+See the `config/` directory for examples.
+
+## Usage
+
+```bash
+# Sync all mirrors
+mirror-dedupe
+
+# Sync specific mirror
+mirror-dedupe --mirror ubuntu
+
+# Dry run
+mirror-dedupe --dry-run
+
+# Dedupe only (no sync)
+mirror-dedupe --dedupe-only
+```
+
+## Systemd Integration
+
+If installed via Debian package, systemd is already configured. Otherwise:
+
+```bash
+sudo systemctl enable --now mirror-dedupe.timer
+sudo systemctl status mirror-dedupe.timer
+```
+
+View logs:
+
+```bash
+journalctl -u mirror-dedupe.service
+```
+
+## Nginx Configuration
+
+See `nginx/mirror.conf` for an example nginx configuration.
+
+## License
+
+MIT License - see LICENSE file for details.
+
+## Author
+
+Tim Hosking <tim@mungerware.com>
+
+https://github.com/munger/mirror-dedupe

mirror_dedupe-0.1.0/PUBLISHING.md

@@ -0,0 +1,77 @@
+<!--
+PUBLISHING.md : Ubuntu mirror synchronisation with global deduplication
+
+Copyright (c) 2025 Tim Hosking
+Email: tim@mungerware.com
+Website: https://github.com/munger
+Licence: MIT
+-->
+
+1. Create account on https://pypi.org
+2. Generate API token at https://pypi.org/manage/account/token/
+3. Add token to GitHub secrets as `PYPI_API_TOKEN`
+
+### Manual Publishing
+
+```bash
+# Install build tools
+pip install build twine
+
+# Build package
+python -m build
+
+# Upload to PyPI
+twine upload dist/*
+```
+
+### Automatic Publishing
+
+The GitHub Actions workflow will automatically publish to PyPI when you create a release.
+
+## Creating a Release
+
+1. Update version in `setup.py` and `pyproject.toml`
+2. Update `debian/changelog`:
+   ```bash
+   dch -v 0.1.1-1 "New release"
+   ```
+3. Commit changes
+4. Create and push tag:
+   ```bash
+   git tag -a v0.1.1 -m "Release v0.1.1"
+   git push origin v0.1.1
+   ```
+5. Create GitHub release from tag
+6. GitHub Actions will:
+   - Build and publish to PyPI
+   - Build `.deb` package
+   - Attach `.deb` to release
+
+## Manual Debian Package Build
+
+```bash
+dpkg-buildpackage -us -uc -b
+```
+
+The package will be created in the parent directory.
+
+## Installation Methods for Users
+
+### Debian/Ubuntu
+```bash
+wget https://github.com/munger/mirror-dedupe/releases/download/v0.1.0/mirror-dedupe_0.1.0-1_all.deb
+sudo dpkg -i mirror-dedupe_0.1.0-1_all.deb
+```
+
+### PyPI (any Linux)
+```bash
+pip install mirror-dedupe
+sudo ./install.sh --pip
+```
+
+### From Source
+```bash
+git clone https://github.com/munger/mirror-dedupe.git
+cd mirror-dedupe
+sudo ./install.sh
+```

mirror_dedupe-0.1.0/README.md

@@ -0,0 +1,105 @@
+<!--
+README.md : Ubuntu mirror synchronisation with global deduplication
+
+Copyright (c) 2025 Tim Hosking
+Email: tim@mungerware.com
+Website: https://github.com/munger
+Licence: MIT
+-->
+
+Ubuntu/Debian mirror synchronisation with intelligent deduplication using hardlinks.
+
+## Features
+
+- Downloads from upstream and hardlinks duplicate files (same SHA256 hash) to save bandwidth and disk space
+- Supports multiple mirrors with global deduplication across all mirrors
+- Uses curl for duplicate files, rsync for unique files and metadata
+- Configurable via YAML files
+- Systemd timer support for automated synchronisation
+
+## Installation
+
+### Option 1: Debian/Ubuntu Package (Recommended)
+
+Download the latest `.deb` package from [GitHub Releases](https://github.com/munger/mirror-dedupe/releases):
+
+```bash
+wget https://github.com/munger/mirror-dedupe/releases/download/v0.1.0/mirror-dedupe_0.1.0-1_all.deb
+sudo dpkg -i mirror-dedupe_0.1.0-1_all.deb
+```
+
+This includes systemd integration, man pages, and proper package management.
+
+### Option 2: PyPI (All Linux Distributions)
+
+```bash
+pip install mirror-dedupe
+```
+
+Then install systemd files manually:
+
+```bash
+sudo ./install.sh --pip
+```
+
+### Option 3: From Source
+
+```bash
+git clone https://github.com/munger/mirror-dedupe.git
+cd mirror-dedupe
+sudo ./install.sh
+```
+
+## Configuration
+
+Configuration files are located in `/etc/mirror-dedupe/`:
+
+- `mirror-dedupe.conf` - Global settings
+- `repos.d/*.conf` - Individual repository definitions
+
+See the `config/` directory for examples.
+
+## Usage
+
+```bash
+# Sync all mirrors
+mirror-dedupe
+
+# Sync specific mirror
+mirror-dedupe --mirror ubuntu
+
+# Dry run
+mirror-dedupe --dry-run
+
+# Dedupe only (no sync)
+mirror-dedupe --dedupe-only
+```
+
+## Systemd Integration
+
+If installed via Debian package, systemd is already configured. Otherwise:
+
+```bash
+sudo systemctl enable --now mirror-dedupe.timer
+sudo systemctl status mirror-dedupe.timer
+```
+
+View logs:
+
+```bash
+journalctl -u mirror-dedupe.service
+```
+
+## Nginx Configuration
+
+See `nginx/mirror.conf` for an example nginx configuration.
+
+## License
+
+MIT License - see LICENSE file for details.
+
+## Author
+
+Tim Hosking <tim@mungerware.com>
+
+https://github.com/munger/mirror-dedupe

mirror_dedupe-0.1.0/config/mirror-dedupe.conf

@@ -0,0 +1,25 @@
+#
+# mirror-dedupe.conf : Ubuntu mirror synchronisation with global deduplication
+#
+# Copyright (c) 2025 Tim Hosking
+# Email: tim@mungerware.com
+# Website: https://github.com/munger
+# Licence: MIT
+#
+
+repo_root: /srv/mirror/repos
+
+# Buffer size for SHA256 hashing (default: 1048576 / 1MB)
+buffer_size: 1048576
+
+# Number of concurrent downloads (default: 10)
+parallel_downloads: 10
+
+# Timeout in seconds for curl downloads (default: 900 / 15 min)
+curl_timeout: 900
+
+# Number of retry attempts for failed downloads (default: 3)
+max_retries: 3
+
+# Print progress every N files (default: 1000)
+progress_interval: 1000

mirror_dedupe-0.1.0/config/repos.d/README.md

@@ -0,0 +1,61 @@
+<!--
+README.md : Ubuntu mirror synchronisation with global deduplication
+
+Copyright (c) 2025 Tim Hosking
+Email: tim@mungerware.com
+Website: https://github.com/munger
+Licence: MIT
+-->
+
+Place your repository configuration files here. Each file should be named `<name>.conf`.
+
+## Example
+
+Copy `ubuntu.conf.example` to `ubuntu.conf` and customise:
+
+```bash
+cp ubuntu.conf.example ubuntu.conf
+```
+
+Then edit `ubuntu.conf` to match your requirements:
+- Change `upstream` URL if needed
+- Adjust `dest` path (relative to `repo_root`)
+- Select architectures to mirror
+- Choose components
+- Specify distributions
+
+## Configuration Format
+
+Each repository config file should contain:
+
+```yaml
+name: repository-name
+upstream: http://upstream.example.com/repo
+dest: relative/path/from/repo_root
+sync_method: rsync  # or https
+gpg_key_url: http://upstream.example.com/gpg.key
+gpg_key_path: relative/path/to/key
+architectures:
+  - amd64
+  - arm64
+components:
+  - main
+  - restricted
+distributions:
+  - noble
+```
+
+## Sync Methods
+
+- **rsync**: Uses rsync protocol (faster, recommended for official Ubuntu mirrors)
+- **https**: Uses HTTPS with curl (for repositories that don't support rsync)
+
+## Distribution Expansion
+
+By default, distributions are expanded to include variants:
+- `noble` → `noble`, `noble-updates`, `noble-security`, `noble-backports`, `noble-proposed`
+
+To disable expansion, add:
+```yaml
+expand_distributions: false
+```

mirror_dedupe-0.1.0/config/repos.d/ubuntu-ports.conf.example

@@ -0,0 +1,23 @@
+# Ubuntu ARM/RISC-V ports mirror example
+# Copy this file to ubuntu-ports.conf and customise
+
+name: ubuntu-ports
+upstream: http://ports.ubuntu.com/ubuntu-ports
+dest: ubuntu/ubuntu-ports
+sync_method: rsync
+gpg_key_url: http://ports.ubuntu.com/ubuntu-ports/project/ubuntu-archive-keyring.gpg
+gpg_key_path: project/ubuntu-archive-keyring.gpg
+architectures:
+  - arm64
+  - armhf
+  - ppc64el
+  - riscv64
+  - s390x
+components:
+  - main
+  - restricted
+  - universe
+  - multiverse
+distributions:
+  - noble
+# Each distribution auto-expands to include -updates, -security, -backports, -proposed

mirror_dedupe-0.1.0/config/repos.d/ubuntu.conf.example

@@ -0,0 +1,19 @@
+# Ubuntu amd64 mirror example
+# Copy this file to ubuntu.conf and customise
+
+name: ubuntu
+upstream: http://archive.ubuntu.com/ubuntu
+dest: ubuntu/ubuntu
+sync_method: rsync
+gpg_key_url: http://archive.ubuntu.com/ubuntu/project/ubuntu-archive-keyring.gpg
+gpg_key_path: project/ubuntu-archive-keyring.gpg
+architectures:
+  - amd64
+components:
+  - main
+  - restricted
+  - universe
+  - multiverse
+distributions:
+  - noble
+# Each distribution auto-expands to include -updates, -security, -backports, -proposed

mirror_dedupe-0.1.0/debian/changelog

@@ -0,0 +1,9 @@
+mirror-dedupe (0.1.0-1) noble; urgency=medium
+
+  * Initial release
+  * Mirror synchronisation with intelligent deduplication
+  * Support for multiple mirrors with global deduplication
+  * Configurable via YAML files
+  * Systemd timer support
+
+ -- Tim Hosking <tim@mungerware.com>  Mon, 28 Oct 2025 19:33:00 +0000

mirror_dedupe-0.1.0/debian/control

@@ -0,0 +1,28 @@
+Source: mirror-dedupe
+Section: admin
+Priority: optional
+Maintainer: Tim Hosking <tim@mungerware.com>
+Build-Depends: debhelper-compat (= 13),
+               dh-python,
+               python3-all,
+               python3-setuptools,
+               python3-yaml
+Standards-Version: 4.6.0
+Homepage: https://github.com/munger/mirror-dedupe
+
+Package: mirror-dedupe
+Architecture: all
+Depends: ${python3:Depends},
+         ${misc:Depends},
+         python3-yaml,
+         curl,
+         rsync
+Description: Mirror synchronisation with intelligent deduplication
+ Downloads from upstream and hardlinks duplicate files (same SHA256 hash)
+ to save bandwidth and disk space. Supports multiple mirrors with global
+ deduplication across all mirrors.
+ .
+ Features:
+  - Uses curl for duplicate files, rsync for unique files and metadata
+  - Configurable via YAML files
+  - Systemd timer support for automated synchronisation

mirror_dedupe-0.1.0/debian/copyright

@@ -0,0 +1,27 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: mirror-dedupe
+Upstream-Contact: Tim Hosking <tim@mungerware.com>
+Source: https://github.com/munger/mirror-dedupe
+
+Files: *
+Copyright: 2025 Tim Hosking <tim@mungerware.com>
+License: MIT
+
+License: MIT
+ Permission is hereby granted, free of charge, to any person obtaining a
+ copy of this software and associated documentation files (the "Software"),
+ to deal in the Software without restriction, including without limitation
+ the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ and/or sell copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following conditions:
+ .
+ The above copyright notice and this permission notice shall be included
+ in all copies or substantial portions of the Software.
+ .
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ DEALINGS IN THE SOFTWARE.

mirror_dedupe-0.1.0/debian/install

@@ -0,0 +1,5 @@
+config/mirror-dedupe.conf etc/mirror-dedupe/
+config/repos.d/*.example etc/mirror-dedupe/repos.d/
+systemd/mirror-dedupe.service lib/systemd/system/
+systemd/mirror-dedupe.timer lib/systemd/system/
+debian/mirror-dedupe.1 usr/share/man/man1/