mirror-url 3.1.14__tar.gz

mirror_url-3.1.14/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 BP
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

mirror_url-3.1.14/PKG-INFO

@@ -0,0 +1,171 @@
+Metadata-Version: 2.4
+Name: mirror-url
+Version: 3.1.14
+Summary: Enterprise-grade remote directory mirroring tool with adaptive concurrency, integrity checks, and SSRF-hardened transport.
+Author: BP
+License: MIT License
+        
+        Copyright (c) 2026 BP
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/bpodlipnik/mirror-url
+Project-URL: Repository, https://github.com/bpodlipnik/mirror-url
+Project-URL: Issues, https://github.com/bpodlipnik/mirror-url/issues
+Keywords: mirror,download,http,scraper,directory,async,httpx
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: System Administrators
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: System :: Archiving :: Mirroring
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: httpx>=0.24
+Requires-Dist: pydantic>=2.0
+Requires-Dist: PyYAML>=6.0
+Provides-Extra: fast
+Requires-Dist: stringzilla>=3.0; extra == "fast"
+Requires-Dist: lxml>=4.9; extra == "fast"
+Provides-Extra: progress
+Requires-Dist: tqdm>=4.65; extra == "progress"
+Provides-Extra: monitor
+Requires-Dist: psutil>=5.9; extra == "monitor"
+Provides-Extra: all
+Requires-Dist: stringzilla>=3.0; extra == "all"
+Requires-Dist: lxml>=4.9; extra == "all"
+Requires-Dist: tqdm>=4.65; extra == "all"
+Requires-Dist: psutil>=5.9; extra == "all"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21; extra == "dev"
+Requires-Dist: pytest-cov>=4.1; extra == "dev"
+Requires-Dist: ruff>=0.4; extra == "dev"
+Requires-Dist: black>=24.0; extra == "dev"
+Requires-Dist: mypy>=1.8; extra == "dev"
+Requires-Dist: types-PyYAML>=6.0; extra == "dev"
+Requires-Dist: pre-commit>=3.5; extra == "dev"
+Dynamic: license-file
+
+# MirrorURL
+
+[![CI](https://github.com/bpodlipnik/mirror-url/actions/workflows/ci.yml/badge.svg)](https://github.com/bpodlipnik/mirror-url/actions/workflows/ci.yml)
+[![Python](https://img.shields.io/badge/python-3.9%E2%80%933.12-blue)](https://www.python.org)
+[![License: MIT](https://img.shields.io/badge/license-MIT-green)](./LICENSE)
+
+Enterprise-grade remote directory mirroring tool. MirrorURL recursively
+discovers files behind an HTTP(S) directory listing and mirrors them locally
+with adaptive concurrency, resumable/partial downloads, integrity verification,
+and an SSRF-hardened transport layer.
+
+> **Status:** Refactor complete. The full implementation now lives in the
+> [`src/mirror_url/`](./src/mirror_url) package (30 modules), migrated verbatim
+> from the original single-file script per [`REFACTORING_PLAN.md`](./REFACTORING_PLAN.md).
+> The legacy [`mirror_url.py`](./mirror_url.py) is retained only as a frozen
+> reference and can be deleted once you've run the test suite against the package
+> in an environment with the runtime dependencies installed (see Development).
+
+## Features
+
+- **Recursive discovery** of remote directory trees (BFS, depth/exclude limits, cycle-safe).
+- **True parallel downloads** — multiple files and multiple chunks per file concurrently.
+- **Adaptive async concurrency** that tunes itself to server RTT, throughput, and error rate.
+- **Resumable & partial downloads** with HTTP range requests and chunk assembly.
+- **Integrity checks** — size/timestamp comparison, ETag handling, content hashing.
+- **Resilience** — per-domain circuit breakers, exponential backoff, rate limiting.
+- **Security** — path-traversal and symlink-bomb defenses, private-IP/SSRF guards, URL-scope enforcement.
+- **Operability** — metrics collection, multi-level progress, optional HTTP health-check server.
+- **Caching** — filesystem and disk-backed indexes to skip unchanged content.
+
+## Installation
+
+```bash
+# From source (editable, recommended during the refactor)
+pip install -e ".[all,dev]"
+```
+
+Python 3.9+ is required. Core dependencies: `httpx`, `pydantic` (v2), `PyYAML`.
+Optional extras: `fast` (stringzilla, lxml), `progress` (tqdm), `monitor` (psutil).
+
+## Usage
+
+Run via the console entry point or the module:
+
+```bash
+mirror-url https://example.com/files/ --output ./mirror
+# or
+python -m mirror_url https://example.com/files/ --output ./mirror
+```
+
+Run `mirror-url --help` for the full option list.
+
+Configuration can also be supplied via a YAML file (see `MirrorConfig` /
+`load_config_from_args`).
+
+📖 **Full documentation:** see the [User Guide](./docs/USER_GUIDE.md)
+([HTML version](./docs/USER_GUIDE.html)) for detailed installation, CLI
+reference, config-file format, download modes, security notes, the Python API,
+and troubleshooting.
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pre-commit install
+
+ruff check .          # lint
+black --check .       # format check
+mypy                  # type-check the new package
+pytest -m "not integration"   # fast test lane
+pytest                # full suite (includes integration)
+```
+
+Continuous integration runs lint + tests across Python 3.9–3.12 (see
+`.github/workflows/ci.yml`).
+
+## Project layout
+
+```
+src/mirror_url/          # the package (30 modules, dependency-layered)
+tests/                   # pytest suite
+mirror_url.py            # legacy monolith (frozen reference, pending removal)
+REFACTORING_PLAN.md      # module breakdown + migration roadmap
+CHANGELOG.md             # notable changes (Keep a Changelog format)
+CONTRIBUTING.md          # dev setup, checks, conventions
+pyproject.toml           # packaging, deps, tool config
+```
+
+## Contributing
+
+Contributions are welcome — see [CONTRIBUTING.md](./CONTRIBUTING.md) for the dev
+setup, the checks CI runs, and the project conventions (dependency layering,
+lint policy, behavior-preserving refactors). Notable changes are tracked in
+[CHANGELOG.md](./CHANGELOG.md).
+
+## License
+
+[MIT](./LICENSE) © BP

mirror_url-3.1.14/README.md

@@ -0,0 +1,98 @@
+# MirrorURL
+
+[![CI](https://github.com/bpodlipnik/mirror-url/actions/workflows/ci.yml/badge.svg)](https://github.com/bpodlipnik/mirror-url/actions/workflows/ci.yml)
+[![Python](https://img.shields.io/badge/python-3.9%E2%80%933.12-blue)](https://www.python.org)
+[![License: MIT](https://img.shields.io/badge/license-MIT-green)](./LICENSE)
+
+Enterprise-grade remote directory mirroring tool. MirrorURL recursively
+discovers files behind an HTTP(S) directory listing and mirrors them locally
+with adaptive concurrency, resumable/partial downloads, integrity verification,
+and an SSRF-hardened transport layer.
+
+> **Status:** Refactor complete. The full implementation now lives in the
+> [`src/mirror_url/`](./src/mirror_url) package (30 modules), migrated verbatim
+> from the original single-file script per [`REFACTORING_PLAN.md`](./REFACTORING_PLAN.md).
+> The legacy [`mirror_url.py`](./mirror_url.py) is retained only as a frozen
+> reference and can be deleted once you've run the test suite against the package
+> in an environment with the runtime dependencies installed (see Development).
+
+## Features
+
+- **Recursive discovery** of remote directory trees (BFS, depth/exclude limits, cycle-safe).
+- **True parallel downloads** — multiple files and multiple chunks per file concurrently.
+- **Adaptive async concurrency** that tunes itself to server RTT, throughput, and error rate.
+- **Resumable & partial downloads** with HTTP range requests and chunk assembly.
+- **Integrity checks** — size/timestamp comparison, ETag handling, content hashing.
+- **Resilience** — per-domain circuit breakers, exponential backoff, rate limiting.
+- **Security** — path-traversal and symlink-bomb defenses, private-IP/SSRF guards, URL-scope enforcement.
+- **Operability** — metrics collection, multi-level progress, optional HTTP health-check server.
+- **Caching** — filesystem and disk-backed indexes to skip unchanged content.
+
+## Installation
+
+```bash
+# From source (editable, recommended during the refactor)
+pip install -e ".[all,dev]"
+```
+
+Python 3.9+ is required. Core dependencies: `httpx`, `pydantic` (v2), `PyYAML`.
+Optional extras: `fast` (stringzilla, lxml), `progress` (tqdm), `monitor` (psutil).
+
+## Usage
+
+Run via the console entry point or the module:
+
+```bash
+mirror-url https://example.com/files/ --output ./mirror
+# or
+python -m mirror_url https://example.com/files/ --output ./mirror
+```
+
+Run `mirror-url --help` for the full option list.
+
+Configuration can also be supplied via a YAML file (see `MirrorConfig` /
+`load_config_from_args`).
+
+📖 **Full documentation:** see the [User Guide](./docs/USER_GUIDE.md)
+([HTML version](./docs/USER_GUIDE.html)) for detailed installation, CLI
+reference, config-file format, download modes, security notes, the Python API,
+and troubleshooting.
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pre-commit install
+
+ruff check .          # lint
+black --check .       # format check
+mypy                  # type-check the new package
+pytest -m "not integration"   # fast test lane
+pytest                # full suite (includes integration)
+```
+
+Continuous integration runs lint + tests across Python 3.9–3.12 (see
+`.github/workflows/ci.yml`).
+
+## Project layout
+
+```
+src/mirror_url/          # the package (30 modules, dependency-layered)
+tests/                   # pytest suite
+mirror_url.py            # legacy monolith (frozen reference, pending removal)
+REFACTORING_PLAN.md      # module breakdown + migration roadmap
+CHANGELOG.md             # notable changes (Keep a Changelog format)
+CONTRIBUTING.md          # dev setup, checks, conventions
+pyproject.toml           # packaging, deps, tool config
+```
+
+## Contributing
+
+Contributions are welcome — see [CONTRIBUTING.md](./CONTRIBUTING.md) for the dev
+setup, the checks CI runs, and the project conventions (dependency layering,
+lint policy, behavior-preserving refactors). Notable changes are tracked in
+[CHANGELOG.md](./CHANGELOG.md).
+
+## License
+
+[MIT](./LICENSE) © BP

mirror_url-3.1.14/pyproject.toml

@@ -0,0 +1,143 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "mirror-url"
+version = "3.1.14"
+description = "Enterprise-grade remote directory mirroring tool with adaptive concurrency, integrity checks, and SSRF-hardened transport."
+readme = "README.md"
+requires-python = ">=3.9"
+license = { file = "LICENSE" }
+authors = [{ name = "BP" }]
+keywords = ["mirror", "download", "http", "scraper", "directory", "async", "httpx"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Intended Audience :: System Administrators",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: System :: Archiving :: Mirroring",
+]
+
+# Core runtime dependencies (imported unconditionally by the package).
+dependencies = [
+    "httpx>=0.24",
+    "pydantic>=2.0",
+    "PyYAML>=6.0",
+]
+
+[project.optional-dependencies]
+# Optional accelerators / niceties — guarded by try/except in the code.
+fast = ["stringzilla>=3.0", "lxml>=4.9"]
+progress = ["tqdm>=4.65"]
+monitor = ["psutil>=5.9"]
+all = ["stringzilla>=3.0", "lxml>=4.9", "tqdm>=4.65", "psutil>=5.9"]
+
+# Development / CI toolchain.
+dev = [
+    "pytest>=7.4",
+    "pytest-asyncio>=0.21",
+    "pytest-cov>=4.1",
+    "ruff>=0.4",
+    "black>=24.0",
+    "mypy>=1.8",
+    "types-PyYAML>=6.0",  # type stubs for the yaml import (silences import-untyped)
+    "pre-commit>=3.5",
+]
+
+[project.urls]
+Homepage = "https://github.com/bpodlipnik/mirror-url"
+Repository = "https://github.com/bpodlipnik/mirror-url"
+Issues = "https://github.com/bpodlipnik/mirror-url/issues"
+
+# Console entry point: the `mirror-url` command runs mirror_url.cli:main.
+[project.scripts]
+mirror-url = "mirror_url.cli:main"
+
+[tool.setuptools]
+package-dir = { "" = "src" }
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+# ---------------------------------------------------------------------------
+# Tooling configuration
+# ---------------------------------------------------------------------------
+[tool.black]
+line-length = 100
+target-version = ["py39", "py310", "py311", "py312"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py39"
+src = ["src", "tests"]
+# The legacy monolith is a frozen reference (and read-only in some checkouts):
+# exclude it from BOTH linting and formatting. `per-file-ignores` below only
+# affects lint, so this top-level exclude is what keeps `ruff format` from
+# trying to rewrite it.
+extend-exclude = ["mirror_url.py"]
+
+[tool.ruff.lint]
+# Correctness-focused rule set. UP (pyupgrade) and SIM are intentionally
+# omitted: this package is a verbatim port of working code that supports
+# Python 3.9 with classic typing (Dict/Optional), and we don't want to churn
+# ~350 lines of audited logic to modernize syntax or rewrite control flow.
+# Bump requires-python to >=3.10 and re-enable UP if/when 3.9 is dropped.
+select = ["E", "F", "W", "I", "B", "C4"]
+ignore = [
+    "E501",  # line length is handled by the formatter
+    "B008",  # function call in argument defaults (intentional in a few places)
+    # The following bugbear rules fire only on code carried over verbatim from
+    # the original single-file script; they are deliberately preserved here so
+    # the port stays behavior-identical. Revisit during the optional core/
+    # mixin refactor (see REFACTORING_PLAN.md §4.1).
+    "B904",  # raise ... from err — only affects traceback chaining, not control flow
+    "B007",  # unused loop control variable
+    "B019",  # lru_cache on a method (MirrorURL._parse_url_cached): a real latent
+             # leak inherited from the monolith — preserved, not silently rewritten
+]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["B"]
+# The legacy monolith is excluded from lint (frozen reference, pending deletion).
+"mirror_url.py" = ["ALL"]
+
+[tool.mypy]
+# Advisory, lenient type-checking. This package is a verbatim port of working,
+# largely-untyped code, so mypy is run as a signal (CI uses continue-on-error),
+# not a gate. The settings below surface genuine type mismatches without
+# demanding annotations on every function or descending into untyped bodies.
+# Tightening (disallow_untyped_defs, check_untyped_defs, strict) is a good
+# incremental follow-up, ideally alongside the core/ mixin refactor.
+python_version = "3.9"
+files = ["src/mirror_url"]
+ignore_missing_imports = true   # optional deps (stringzilla, lxml, tqdm, psutil) have no stubs
+follow_imports = "silent"
+disallow_untyped_defs = false
+check_untyped_defs = false
+# The original code predates strict implicit-Optional enforcement and relies on
+# `param: T = None` in a few signatures. Restore the permissive behavior rather
+# than churn the verbatim port; tighten alongside a dedicated typing pass.
+no_implicit_optional = false
+warn_unused_ignores = false
+warn_redundant_casts = true
+warn_unreachable = false
+# The legacy monolith is not part of the package and is not type-checked.
+exclude = ["mirror_url\\.py$", "tests/"]
+
+[tool.pytest.ini_options]
+minversion = "7.0"
+testpaths = ["tests"]
+asyncio_mode = "auto"
+addopts = "-ra --strict-markers"
+markers = [
+    "integration: end-to-end tests that exercise real I/O (deselect with -m 'not integration')",
+]

mirror_url-3.1.14/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

mirror_url-3.1.14/src/mirror_url/__init__.py

@@ -0,0 +1,48 @@
+"""MirrorURL — enterprise-grade remote directory mirroring tool.
+
+This package is the modular successor to the single-file ``mirror_url.py``
+script. During the refactor the monolith remains the runnable source of truth;
+the submodules here are being populated incrementally (see
+``REFACTORING_PLAN.md``).
+
+Public API (intended, once migration completes)::
+
+    from mirror_url import MirrorURL, MirrorConfig, main
+
+The re-exports below are commented out until the corresponding modules are
+populated, so that ``import mirror_url`` succeeds at every step of the migration.
+"""
+
+from __future__ import annotations
+
+from ._version import __author__, __version__
+
+# --- Public API --------------------------------------------------------------
+from .cli import main
+from .config import MirrorConfig, load_config_from_args
+from .core import MirrorURL
+from .exceptions import (
+    ConfigError,
+    DownloadError,
+    MirrorConnectionError,
+    MirrorError,
+    PathTraversalError,
+    SecurityError,
+    URLScopeError,
+)
+
+__all__ = [
+    "__version__",
+    "__author__",
+    "MirrorURL",
+    "MirrorConfig",
+    "load_config_from_args",
+    "main",
+    "MirrorError",
+    "MirrorConnectionError",
+    "PathTraversalError",
+    "URLScopeError",
+    "ConfigError",
+    "SecurityError",
+    "DownloadError",
+]

mirror_url-3.1.14/src/mirror_url/__main__.py

@@ -0,0 +1,21 @@
+"""Enable ``python -m mirror_url``.
+
+Delegates to the CLI entry point once ``cli`` is populated.
+"""
+
+from __future__ import annotations
+
+
+def _run() -> None:
+    try:
+        from .cli import main
+    except ImportError as exc:  # pragma: no cover - pre-migration guard
+        raise SystemExit(
+            "mirror_url.cli is not populated yet. During the refactor, run the "
+            "original script directly: `python mirror_url.py ...`"
+        ) from exc
+    main()
+
+
+if __name__ == "__main__":
+    _run()

mirror_url-3.1.14/src/mirror_url/_core/__init__.py

@@ -0,0 +1,9 @@
+"""Private implementation subpackage for the ``MirrorURL`` orchestrator.
+
+``MirrorURL`` is composed from the mixins in this package (one responsibility
+per module) by ``mirror_url.core``. Import the public class from there:
+
+    from mirror_url.core import MirrorURL   # or: from mirror_url import MirrorURL
+
+See ``REFACTORING_PLAN.md`` §4.1 for the split rationale.
+"""