magnet-scout 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- magnet_scout-0.1.0/.github/dependabot.yml +12 -0
- magnet_scout-0.1.0/.github/workflows/ci.yml +30 -0
- magnet_scout-0.1.0/.github/workflows/release.yml +80 -0
- magnet_scout-0.1.0/.gitignore +9 -0
- magnet_scout-0.1.0/AGENTS.md +17 -0
- magnet_scout-0.1.0/CHANGELOG.md +7 -0
- magnet_scout-0.1.0/CONTRIBUTING.md +16 -0
- magnet_scout-0.1.0/LICENSE +21 -0
- magnet_scout-0.1.0/PKG-INFO +129 -0
- magnet_scout-0.1.0/README.md +88 -0
- magnet_scout-0.1.0/SECURITY.md +14 -0
- magnet_scout-0.1.0/docs/architecture.md +53 -0
- magnet_scout-0.1.0/docs/development.md +39 -0
- magnet_scout-0.1.0/docs/health-scoring.md +53 -0
- magnet_scout-0.1.0/docs/providers.md +63 -0
- magnet_scout-0.1.0/docs/releasing.md +65 -0
- magnet_scout-0.1.0/pyproject.toml +75 -0
- magnet_scout-0.1.0/scripts/__init__.py +1 -0
- magnet_scout-0.1.0/scripts/check_release.py +46 -0
- magnet_scout-0.1.0/src/magnet_scout/__init__.py +19 -0
- magnet_scout-0.1.0/src/magnet_scout/__main__.py +3 -0
- magnet_scout-0.1.0/src/magnet_scout/cache.py +118 -0
- magnet_scout-0.1.0/src/magnet_scout/cli.py +307 -0
- magnet_scout-0.1.0/src/magnet_scout/config.py +63 -0
- magnet_scout-0.1.0/src/magnet_scout/dht.py +119 -0
- magnet_scout-0.1.0/src/magnet_scout/magnets.py +70 -0
- magnet_scout-0.1.0/src/magnet_scout/metainfo.py +42 -0
- magnet_scout-0.1.0/src/magnet_scout/models.py +101 -0
- magnet_scout-0.1.0/src/magnet_scout/network.py +44 -0
- magnet_scout-0.1.0/src/magnet_scout/providers/__init__.py +1 -0
- magnet_scout-0.1.0/src/magnet_scout/providers/academic_torrents.py +165 -0
- magnet_scout-0.1.0/src/magnet_scout/providers/base.py +11 -0
- magnet_scout-0.1.0/src/magnet_scout/providers/fedora.py +95 -0
- magnet_scout-0.1.0/src/magnet_scout/providers/internet_archive.py +157 -0
- magnet_scout-0.1.0/src/magnet_scout/providers/registry.py +30 -0
- magnet_scout-0.1.0/src/magnet_scout/providers/torznab.py +146 -0
- magnet_scout-0.1.0/src/magnet_scout/py.typed +1 -0
- magnet_scout-0.1.0/src/magnet_scout/scoring.py +70 -0
- magnet_scout-0.1.0/src/magnet_scout/service.py +157 -0
- magnet_scout-0.1.0/src/magnet_scout/verification.py +452 -0
- magnet_scout-0.1.0/tests/test_academic_torrents.py +107 -0
- magnet_scout-0.1.0/tests/test_cache_cli.py +34 -0
- magnet_scout-0.1.0/tests/test_cli.py +119 -0
- magnet_scout-0.1.0/tests/test_config.py +31 -0
- magnet_scout-0.1.0/tests/test_dht.py +27 -0
- magnet_scout-0.1.0/tests/test_fedora.py +80 -0
- magnet_scout-0.1.0/tests/test_internet_archive.py +104 -0
- magnet_scout-0.1.0/tests/test_magnets.py +41 -0
- magnet_scout-0.1.0/tests/test_metainfo.py +38 -0
- magnet_scout-0.1.0/tests/test_network.py +42 -0
- magnet_scout-0.1.0/tests/test_release.py +7 -0
- magnet_scout-0.1.0/tests/test_scoring.py +81 -0
- magnet_scout-0.1.0/tests/test_service.py +113 -0
- magnet_scout-0.1.0/tests/test_torznab.py +76 -0
- magnet_scout-0.1.0/tests/test_verification.py +234 -0
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
pull_request:
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: read
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
test:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
strategy:
|
|
14
|
+
matrix:
|
|
15
|
+
python-version: ["3.11", "3.12", "3.13"]
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
|
|
18
|
+
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
|
|
19
|
+
with:
|
|
20
|
+
python-version: ${{ matrix.python-version }}
|
|
21
|
+
cache: pip
|
|
22
|
+
- run: python -m pip install -e '.[dev,dht]'
|
|
23
|
+
- run: ruff format --check .
|
|
24
|
+
- run: ruff check .
|
|
25
|
+
- run: mypy
|
|
26
|
+
- run: pytest
|
|
27
|
+
- run: python scripts/check_release.py
|
|
28
|
+
- run: bandit -r src -ll
|
|
29
|
+
- run: pip-audit . --progress-spinner off
|
|
30
|
+
- run: python -m build
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_dispatch:
|
|
5
|
+
release:
|
|
6
|
+
types: [published]
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
build:
|
|
13
|
+
name: Build and verify distributions
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
|
|
17
|
+
with:
|
|
18
|
+
persist-credentials: false
|
|
19
|
+
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
|
|
20
|
+
with:
|
|
21
|
+
python-version: "3.13"
|
|
22
|
+
cache: pip
|
|
23
|
+
- name: Install build tools
|
|
24
|
+
run: python -m pip install 'build>=1.2' 'twine>=6.1,<7'
|
|
25
|
+
- name: Validate release metadata
|
|
26
|
+
env:
|
|
27
|
+
RELEASE_TAG: ${{ github.event.release.tag_name }}
|
|
28
|
+
run: python scripts/check_release.py
|
|
29
|
+
- name: Build wheel and source distribution
|
|
30
|
+
run: python -m build
|
|
31
|
+
- name: Check distribution metadata
|
|
32
|
+
run: python -m twine check dist/*
|
|
33
|
+
- name: Upload immutable build artifact
|
|
34
|
+
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
|
35
|
+
with:
|
|
36
|
+
name: python-package-distributions
|
|
37
|
+
path: dist/
|
|
38
|
+
if-no-files-found: error
|
|
39
|
+
retention-days: 7
|
|
40
|
+
|
|
41
|
+
publish-testpypi:
|
|
42
|
+
name: Publish to TestPyPI
|
|
43
|
+
if: github.event_name == 'workflow_dispatch'
|
|
44
|
+
needs: build
|
|
45
|
+
runs-on: ubuntu-latest
|
|
46
|
+
environment:
|
|
47
|
+
name: testpypi
|
|
48
|
+
url: https://test.pypi.org/p/magnet-scout
|
|
49
|
+
permissions:
|
|
50
|
+
id-token: write
|
|
51
|
+
steps:
|
|
52
|
+
- uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6
|
|
53
|
+
with:
|
|
54
|
+
name: python-package-distributions
|
|
55
|
+
path: dist/
|
|
56
|
+
- name: Publish to TestPyPI
|
|
57
|
+
uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
|
|
58
|
+
with:
|
|
59
|
+
repository-url: https://test.pypi.org/legacy/
|
|
60
|
+
packages-dir: dist/
|
|
61
|
+
|
|
62
|
+
publish-pypi:
|
|
63
|
+
name: Publish to PyPI
|
|
64
|
+
if: github.event_name == 'release'
|
|
65
|
+
needs: build
|
|
66
|
+
runs-on: ubuntu-latest
|
|
67
|
+
environment:
|
|
68
|
+
name: pypi
|
|
69
|
+
url: https://pypi.org/p/magnet-scout
|
|
70
|
+
permissions:
|
|
71
|
+
id-token: write
|
|
72
|
+
steps:
|
|
73
|
+
- uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6
|
|
74
|
+
with:
|
|
75
|
+
name: python-package-distributions
|
|
76
|
+
path: dist/
|
|
77
|
+
- name: Publish to PyPI with attestations
|
|
78
|
+
uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
|
|
79
|
+
with:
|
|
80
|
+
packages-dir: dist/
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# MagnetScout agent instructions
|
|
2
|
+
|
|
3
|
+
- Preserve the product boundary: return torrent metadata and magnet URIs only. Never add
|
|
4
|
+
payload or piece downloading, content storage, client launching, or media-library integration.
|
|
5
|
+
- Prefer structured provider APIs and explicitly legal or public collections over generic web
|
|
6
|
+
search and brittle scraping.
|
|
7
|
+
- Keep providers behind `SearchProvider`; one provider failing must not fail an aggregate search.
|
|
8
|
+
- Treat provider seed counts as untrusted reports. Only verifier observations may populate
|
|
9
|
+
`verified`, `verified_peers`, or tracker and DHT evidence.
|
|
10
|
+
- Canonicalize and deduplicate by info hash before verification and ranking.
|
|
11
|
+
- Add fixtures and tests for every provider. Tests must not depend on the live network.
|
|
12
|
+
- Keep timeouts and concurrency bounded. Never log full peer IP addresses.
|
|
13
|
+
- DHT backends must use a hard deadline, perform peer discovery only, discard addresses after
|
|
14
|
+
counting, and never fetch metadata or announce. Prefer process isolation for blocking clients.
|
|
15
|
+
- Record architectural discoveries and scoring changes in `docs/`.
|
|
16
|
+
- Commit messages must be in English and should read like natural, human-written summaries.
|
|
17
|
+
- Run `ruff format --check .`, `ruff check .`, `mypy`, and `pytest` before handing off changes.
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.0 - 2026-10-09
|
|
4
|
+
|
|
5
|
+
- Initial CLI and Python API for searching public BitTorrent metadata providers.
|
|
6
|
+
- Magnet parsing, info-hash normalization, metadata merging, verification, and health ranking.
|
|
7
|
+
- Internet Archive, Academic Torrents, Fedora, and configurable Torznab adapters.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
Thank you for helping improve MagnetScout. Keep changes within the metadata-only boundary: no
|
|
4
|
+
payload downloads, client launching, content storage, or media-library automation.
|
|
5
|
+
|
|
6
|
+
Create a focused branch, add offline tests for behavior changes, and update the relevant document
|
|
7
|
+
when provider behavior or scoring changes. Before opening a pull request, run:
|
|
8
|
+
|
|
9
|
+
```console
|
|
10
|
+
ruff format --check .
|
|
11
|
+
ruff check .
|
|
12
|
+
mypy
|
|
13
|
+
pytest
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
Commit messages should be concise, natural English summaries of the change.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 MagnetScout contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: magnet-scout
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Discover and assess public BitTorrent metadata without downloading payloads
|
|
5
|
+
Project-URL: Homepage, https://github.com/monhoney/magnet-scout
|
|
6
|
+
Project-URL: Repository, https://github.com/monhoney/magnet-scout
|
|
7
|
+
Project-URL: Issues, https://github.com/monhoney/magnet-scout/issues
|
|
8
|
+
Author: MagnetScout contributors
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: bittorrent,infohash,magnet,metadata,torrent,tracker
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Internet
|
|
21
|
+
Requires-Python: >=3.11
|
|
22
|
+
Requires-Dist: bencode-py<5,>=4.1
|
|
23
|
+
Requires-Dist: click<8.3,>=8.2
|
|
24
|
+
Requires-Dist: defusedxml<1,>=0.7.1
|
|
25
|
+
Requires-Dist: httpx<1,>=0.27
|
|
26
|
+
Requires-Dist: torf<5,>=4.3.1
|
|
27
|
+
Requires-Dist: typer<0.21,>=0.20
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: bandit<2,>=1.9; extra == 'dev'
|
|
30
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
31
|
+
Requires-Dist: mypy>=1.15; extra == 'dev'
|
|
32
|
+
Requires-Dist: pip-audit<3,>=2.10; extra == 'dev'
|
|
33
|
+
Requires-Dist: pytest-asyncio>=0.25; extra == 'dev'
|
|
34
|
+
Requires-Dist: pytest>=8.3; extra == 'dev'
|
|
35
|
+
Requires-Dist: ruff>=0.11; extra == 'dev'
|
|
36
|
+
Requires-Dist: twine<7,>=6.1; extra == 'dev'
|
|
37
|
+
Requires-Dist: types-defusedxml>=0.7.0.20240218; extra == 'dev'
|
|
38
|
+
Provides-Extra: dht
|
|
39
|
+
Requires-Dist: pythontorrentdht<2,>=1.0.3; extra == 'dht'
|
|
40
|
+
Description-Content-Type: text/markdown
|
|
41
|
+
|
|
42
|
+
# MagnetScout
|
|
43
|
+
|
|
44
|
+
MagnetScout is a lightweight Python CLI and library for discovering BitTorrent metadata from
|
|
45
|
+
pluggable providers, normalizing magnet links, checking current swarm signals, and ranking useful
|
|
46
|
+
results. It never downloads torrent payloads or sends results to a BitTorrent client.
|
|
47
|
+
|
|
48
|
+
MagnetScout is intended for public, freely licensed, and otherwise lawfully distributed content.
|
|
49
|
+
Metadata is informative rather than a legal determination; users remain responsible for deciding
|
|
50
|
+
whether they may access a result.
|
|
51
|
+
|
|
52
|
+
## Status
|
|
53
|
+
|
|
54
|
+
The project is alpha software. Provider availability and swarm observations can change at any
|
|
55
|
+
time, and a healthy score does not guarantee a successful download.
|
|
56
|
+
|
|
57
|
+
## Install
|
|
58
|
+
|
|
59
|
+
MagnetScout requires Python 3.11 or newer.
|
|
60
|
+
|
|
61
|
+
```console
|
|
62
|
+
python -m pip install magnet-scout
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Until the first PyPI release, install from a checkout:
|
|
66
|
+
|
|
67
|
+
```console
|
|
68
|
+
python -m pip install -e .
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Optional DHT peer discovery is installed separately:
|
|
72
|
+
|
|
73
|
+
```console
|
|
74
|
+
python -m pip install 'magnet-scout[dht]'
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
## CLI
|
|
78
|
+
|
|
79
|
+
```console
|
|
80
|
+
magnet-scout search ubuntu
|
|
81
|
+
magnet-scout search ubuntu --top 10
|
|
82
|
+
magnet-scout search ubuntu --verify --top 10
|
|
83
|
+
magnet-scout search ubuntu --verify --min-verified-seeders 3
|
|
84
|
+
magnet-scout search ubuntu --json
|
|
85
|
+
magnet-scout search dataset --provider academic-torrents
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
The default providers are Internet Archive, Academic Torrents, and Fedora. Provider failures are
|
|
89
|
+
reported independently, so one unavailable source does not discard results from the others.
|
|
90
|
+
|
|
91
|
+
`--verify` performs bounded tracker and web-seed observations without requesting payload pieces.
|
|
92
|
+
`--dht` adds optional, process-isolated peer discovery. A provider's reported seeder count remains
|
|
93
|
+
separate from independently observed values.
|
|
94
|
+
|
|
95
|
+
## Python API
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
from magnet_scout import parse_magnet
|
|
99
|
+
|
|
100
|
+
magnet = parse_magnet("magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567&dn=Example")
|
|
101
|
+
print(magnet.info_hash)
|
|
102
|
+
print(magnet.canonical_uri)
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Provider integration uses the asynchronous `SearchProvider` protocol. See
|
|
106
|
+
[docs/providers.md](docs/providers.md) for a complete example.
|
|
107
|
+
|
|
108
|
+
## Evidence, not guarantees
|
|
109
|
+
|
|
110
|
+
- `reported_seeders` is untrusted metadata supplied by a provider.
|
|
111
|
+
- `verified_seeders` is a recent tracker observation when the tracker supports scraping.
|
|
112
|
+
- `dht_peers` counts peer endpoints discovered during a bounded lookup; addresses are discarded.
|
|
113
|
+
- `UNKNOWN` means there was not enough independent evidence. It does not mean dead.
|
|
114
|
+
- No check proves that a complete payload will remain available.
|
|
115
|
+
|
|
116
|
+
The exact score and ranking rules are documented in
|
|
117
|
+
[docs/health-scoring.md](docs/health-scoring.md).
|
|
118
|
+
|
|
119
|
+
## Documentation
|
|
120
|
+
|
|
121
|
+
- [Architecture](docs/architecture.md)
|
|
122
|
+
- [Providers](docs/providers.md)
|
|
123
|
+
- [Health scoring](docs/health-scoring.md)
|
|
124
|
+
- [Development](docs/development.md)
|
|
125
|
+
- [Release process](docs/releasing.md)
|
|
126
|
+
|
|
127
|
+
## License
|
|
128
|
+
|
|
129
|
+
MagnetScout is released under the MIT License.
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# MagnetScout
|
|
2
|
+
|
|
3
|
+
MagnetScout is a lightweight Python CLI and library for discovering BitTorrent metadata from
|
|
4
|
+
pluggable providers, normalizing magnet links, checking current swarm signals, and ranking useful
|
|
5
|
+
results. It never downloads torrent payloads or sends results to a BitTorrent client.
|
|
6
|
+
|
|
7
|
+
MagnetScout is intended for public, freely licensed, and otherwise lawfully distributed content.
|
|
8
|
+
Metadata is informative rather than a legal determination; users remain responsible for deciding
|
|
9
|
+
whether they may access a result.
|
|
10
|
+
|
|
11
|
+
## Status
|
|
12
|
+
|
|
13
|
+
The project is alpha software. Provider availability and swarm observations can change at any
|
|
14
|
+
time, and a healthy score does not guarantee a successful download.
|
|
15
|
+
|
|
16
|
+
## Install
|
|
17
|
+
|
|
18
|
+
MagnetScout requires Python 3.11 or newer.
|
|
19
|
+
|
|
20
|
+
```console
|
|
21
|
+
python -m pip install magnet-scout
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
Until the first PyPI release, install from a checkout:
|
|
25
|
+
|
|
26
|
+
```console
|
|
27
|
+
python -m pip install -e .
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Optional DHT peer discovery is installed separately:
|
|
31
|
+
|
|
32
|
+
```console
|
|
33
|
+
python -m pip install 'magnet-scout[dht]'
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## CLI
|
|
37
|
+
|
|
38
|
+
```console
|
|
39
|
+
magnet-scout search ubuntu
|
|
40
|
+
magnet-scout search ubuntu --top 10
|
|
41
|
+
magnet-scout search ubuntu --verify --top 10
|
|
42
|
+
magnet-scout search ubuntu --verify --min-verified-seeders 3
|
|
43
|
+
magnet-scout search ubuntu --json
|
|
44
|
+
magnet-scout search dataset --provider academic-torrents
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
The default providers are Internet Archive, Academic Torrents, and Fedora. Provider failures are
|
|
48
|
+
reported independently, so one unavailable source does not discard results from the others.
|
|
49
|
+
|
|
50
|
+
`--verify` performs bounded tracker and web-seed observations without requesting payload pieces.
|
|
51
|
+
`--dht` adds optional, process-isolated peer discovery. A provider's reported seeder count remains
|
|
52
|
+
separate from independently observed values.
|
|
53
|
+
|
|
54
|
+
## Python API
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
from magnet_scout import parse_magnet
|
|
58
|
+
|
|
59
|
+
magnet = parse_magnet("magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567&dn=Example")
|
|
60
|
+
print(magnet.info_hash)
|
|
61
|
+
print(magnet.canonical_uri)
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Provider integration uses the asynchronous `SearchProvider` protocol. See
|
|
65
|
+
[docs/providers.md](docs/providers.md) for a complete example.
|
|
66
|
+
|
|
67
|
+
## Evidence, not guarantees
|
|
68
|
+
|
|
69
|
+
- `reported_seeders` is untrusted metadata supplied by a provider.
|
|
70
|
+
- `verified_seeders` is a recent tracker observation when the tracker supports scraping.
|
|
71
|
+
- `dht_peers` counts peer endpoints discovered during a bounded lookup; addresses are discarded.
|
|
72
|
+
- `UNKNOWN` means there was not enough independent evidence. It does not mean dead.
|
|
73
|
+
- No check proves that a complete payload will remain available.
|
|
74
|
+
|
|
75
|
+
The exact score and ranking rules are documented in
|
|
76
|
+
[docs/health-scoring.md](docs/health-scoring.md).
|
|
77
|
+
|
|
78
|
+
## Documentation
|
|
79
|
+
|
|
80
|
+
- [Architecture](docs/architecture.md)
|
|
81
|
+
- [Providers](docs/providers.md)
|
|
82
|
+
- [Health scoring](docs/health-scoring.md)
|
|
83
|
+
- [Development](docs/development.md)
|
|
84
|
+
- [Release process](docs/releasing.md)
|
|
85
|
+
|
|
86
|
+
## License
|
|
87
|
+
|
|
88
|
+
MagnetScout is released under the MIT License.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# Security policy
|
|
2
|
+
|
|
3
|
+
Please report suspected vulnerabilities privately through GitHub's security advisory feature.
|
|
4
|
+
Do not include API keys, full peer addresses, or private index URLs in a public issue.
|
|
5
|
+
|
|
6
|
+
MagnetScout processes untrusted network metadata. Reports involving unsafe metainfo parsing,
|
|
7
|
+
credential exposure, unbounded network activity, SSRF, or unexpected payload retrieval are
|
|
8
|
+
especially useful. The project does not consider provider downtime or stale swarm counts a
|
|
9
|
+
security vulnerability.
|
|
10
|
+
|
|
11
|
+
Every external response is subject to a decompressed byte limit before parsing. XML entity
|
|
12
|
+
expansion and redirects are disabled, tracker and web-seed verification rejects non-public address
|
|
13
|
+
ranges, and DHT discovery runs in a disposable process. These controls are security boundaries;
|
|
14
|
+
changes to them require regression tests and updated architecture documentation.
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# Architecture
|
|
2
|
+
|
|
3
|
+
## Scope
|
|
4
|
+
|
|
5
|
+
MagnetScout returns normalized torrent metadata and magnet URIs. It does not download pieces,
|
|
6
|
+
store payload content, launch torrent clients, or manage media libraries.
|
|
7
|
+
|
|
8
|
+
## Search pipeline
|
|
9
|
+
|
|
10
|
+
```text
|
|
11
|
+
query
|
|
12
|
+
-> concurrent providers
|
|
13
|
+
-> normalized TorrentResult values
|
|
14
|
+
-> magnet validation and canonical info hashes
|
|
15
|
+
-> merge by info hash
|
|
16
|
+
-> optional bounded verification
|
|
17
|
+
-> health and confidence scoring
|
|
18
|
+
-> ranking and filters
|
|
19
|
+
-> CLI or Python result objects
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
`SearchService` owns orchestration. Each provider implements `SearchProvider`, and exceptions are
|
|
23
|
+
converted to `ProviderFailure` entries. The successful providers continue to contribute results.
|
|
24
|
+
|
|
25
|
+
## Models and normalization
|
|
26
|
+
|
|
27
|
+
`TorrentResult` keeps claims and observations distinct. Provider counts use `reported_*` fields;
|
|
28
|
+
tracker and DHT observations use `verified_*` and `dht_peers`. Missing data stays `None` rather
|
|
29
|
+
than becoming a misleading zero.
|
|
30
|
+
|
|
31
|
+
`parse_magnet` accepts hexadecimal and base32 BEP 9 `btih` values and produces a lowercase,
|
|
32
|
+
40-character hexadecimal info hash. `merge_results` uses that hash as its key, retains all source
|
|
33
|
+
providers and URLs, and merges the richest available metadata.
|
|
34
|
+
|
|
35
|
+
## Verification boundaries
|
|
36
|
+
|
|
37
|
+
The tracker verifier sends HTTP(S) or UDP scrape requests where supported. It does not announce as
|
|
38
|
+
a peer. Web-seed checks make bounded HTTP `HEAD` requests for responsiveness and do not count as
|
|
39
|
+
swarm health. Optional DHT verification runs in a separate process under a hard deadline, requests
|
|
40
|
+
peer discovery only, counts returned endpoints, and discards their addresses.
|
|
41
|
+
|
|
42
|
+
All network work uses explicit timeouts, concurrency limits, and decompressed response-size caps.
|
|
43
|
+
Redirects are disabled so credentials and bounded-request rules cannot be bypassed through a
|
|
44
|
+
redirect chain. Environment proxy variables are ignored by the CLI. Verification observations are
|
|
45
|
+
cached briefly to avoid repeatedly contacting infrastructure.
|
|
46
|
+
|
|
47
|
+
## Trust boundaries
|
|
48
|
+
|
|
49
|
+
Provider titles, descriptions, counts, links, and torrent files are untrusted input. Control
|
|
50
|
+
characters are removed in human output, XML entities are disabled, magnet complexity is bounded,
|
|
51
|
+
torrent metainfo is parsed without materializing payloads, and provider errors cannot terminate
|
|
52
|
+
aggregate search. API keys for optional Torznab providers are read from environment variables,
|
|
53
|
+
never stored directly in the configuration file, and sent only to HTTPS endpoints.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Development
|
|
2
|
+
|
|
3
|
+
```console
|
|
4
|
+
python -m pip install -e '.[dev,dht]'
|
|
5
|
+
ruff check .
|
|
6
|
+
mypy
|
|
7
|
+
pytest
|
|
8
|
+
bandit -r src -ll
|
|
9
|
+
pip-audit . --progress-spinner off
|
|
10
|
+
python -m build
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
Tests use injected HTTPX transports and fixtures and must pass offline. To make
|
|
14
|
+
a live smoke test, run `magnet-scout search ubuntu --provider internet-archive
|
|
15
|
+
--top 3`; live API failures should produce a provider warning and a nonzero exit
|
|
16
|
+
only when no provider produced results.
|
|
17
|
+
|
|
18
|
+
Structured diagnostics use Python logging and go to stderr so JSON stdout stays
|
|
19
|
+
machine-readable. Do not include peer addresses in logs or fixtures.
|
|
20
|
+
|
|
21
|
+
Tracker verification tests use HTTPX's in-process mock transport and binary UDP
|
|
22
|
+
response fixtures. They must never contact public trackers during the test
|
|
23
|
+
suite. Live verification reveals the caller's IP address to each queried
|
|
24
|
+
tracker even though scrape does not register the caller as a swarm peer.
|
|
25
|
+
|
|
26
|
+
The SQLite verification cache defaults to
|
|
27
|
+
`$XDG_CACHE_HOME/magnet-scout/verification.sqlite3` (or `~/.cache/...`). Tests
|
|
28
|
+
must use `tmp_path`; never write tests into a developer's real cache.
|
|
29
|
+
|
|
30
|
+
CI runs formatting, lint, strict typing, tests, and wheel/sdist builds on Python
|
|
31
|
+
3.11–3.13. DHT tests use a stubbed count path plus a real parent-deadline test;
|
|
32
|
+
they do not query the public DHT.
|
|
33
|
+
|
|
34
|
+
Torznab tests use XML fixtures and HTTPX mock transports. Never put real API
|
|
35
|
+
keys in fixtures, command examples, exception messages, or snapshots.
|
|
36
|
+
|
|
37
|
+
See `docs/releasing.md` for the release checklist. Do not create a version tag
|
|
38
|
+
before a release commit exists, and do not publish externally without explicit
|
|
39
|
+
authorization.
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# Health and confidence scoring
|
|
2
|
+
|
|
3
|
+
Scores are evidence summaries, never availability guarantees.
|
|
4
|
+
|
|
5
|
+
## Current formula
|
|
6
|
+
|
|
7
|
+
Health starts at 0 and is capped at 100:
|
|
8
|
+
|
|
9
|
+
- independently observed peers: `min(60, 15 * log2(peers + 1))`
|
|
10
|
+
- responsive tracker with observed peers: 15
|
|
11
|
+
- metadata observed from the swarm: 10
|
|
12
|
+
- each additional reporting provider: 5, capped at 10
|
|
13
|
+
- reported seeders: `min(5, log2(seeders + 1))`; deliberately low weight
|
|
14
|
+
- freshness: 5 when checked within 15 minutes, fading linearly to 0 at 24 hours
|
|
15
|
+
|
|
16
|
+
Classification: `EXCELLENT >= 80`, `GOOD >= 60`, `FAIR >= 40`, `POOR > 0`,
|
|
17
|
+
`DEAD = 0` only when at least one tracker returned a valid zero-peer scrape.
|
|
18
|
+
Timeouts, tracker errors, unsupported trackers, and absent trackers remain
|
|
19
|
+
`UNKNOWN` rather than being treated as evidence of death.
|
|
20
|
+
|
|
21
|
+
Confidence is separate. It starts at 10 for valid normalized metadata, adds 15
|
|
22
|
+
per provider (cap 45), 25 for a completed independent check, 10 for a responsive
|
|
23
|
+
tracker, 10 for a responsive web-seed endpoint, 5 for swarm metadata, and up to
|
|
24
|
+
10 for freshness. It is capped at 100. Web-seed responsiveness never adds
|
|
25
|
+
health points and never changes the swarm health classification.
|
|
26
|
+
|
|
27
|
+
Provider-reported counts never set `verified`. Tracker scrape values populate
|
|
28
|
+
`verified_seeders`, `verified_leechers`, and `verified_peers`. Counts from
|
|
29
|
+
different trackers are not summed because their peer sets may overlap; the
|
|
30
|
+
maximum observation is retained. A positive peer count sets `verified`, but
|
|
31
|
+
does not guarantee that a complete, reachable copy can be downloaded.
|
|
32
|
+
|
|
33
|
+
`verification_status` makes the evidence explicit:
|
|
34
|
+
|
|
35
|
+
- `NOT_CHECKED`: no verification was requested or no supported tracker exists
|
|
36
|
+
- `VERIFIED_SEEDED`: at least one tracker reported a complete peer
|
|
37
|
+
- `VERIFIED_PEERS_ONLY`: peers were reported but no complete peer was observed
|
|
38
|
+
- `TRACKER_RESPONSIVE`: a valid scrape reported zero peers
|
|
39
|
+
- `UNREACHABLE`: supported trackers were tried but none returned a valid scrape
|
|
40
|
+
|
|
41
|
+
With `--dht`, `dht_peers` is an independently discovered peer count. A positive
|
|
42
|
+
count can produce `VERIFIED_PEERS_ONLY`, but cannot prove that a complete seeder
|
|
43
|
+
exists; only a tracker complete count can produce `VERIFIED_SEEDED`. A zero DHT
|
|
44
|
+
count alone remains `UNKNOWN` because DHT observation is incomplete by nature.
|
|
45
|
+
|
|
46
|
+
## Ranking
|
|
47
|
+
|
|
48
|
+
Sort keys, in order, are: verified status, health score, verified peers,
|
|
49
|
+
provider count, responsive web seed, confidence, freshness, reported seeders,
|
|
50
|
+
then title. Missing numbers sort below known numbers. With `--verify`, `--min-seeders` uses
|
|
51
|
+
`verified_seeders`; otherwise it uses the provider-reported field. Unknown
|
|
52
|
+
values are excluded. Prefer the unambiguous `--min-reported-seeders` and
|
|
53
|
+
`--min-verified-seeders` options; `--min-seeders` exists for compatibility.
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# Providers
|
|
2
|
+
|
|
3
|
+
Providers translate a source-specific response into `TorrentResult` objects. They never rank
|
|
4
|
+
results, mark provider claims as verified, or download torrent payloads.
|
|
5
|
+
|
|
6
|
+
## Included providers
|
|
7
|
+
|
|
8
|
+
| Name | Source | Notes |
|
|
9
|
+
| --- | --- | --- |
|
|
10
|
+
| `internet-archive` | Internet Archive advanced search and metadata APIs | Public collections; optional subject and explicit-license filters |
|
|
11
|
+
| `academic-torrents` | Academic Torrents public index | Research datasets and publications; index is cached locally |
|
|
12
|
+
| `fedora` | Fedora torrent metadata | Official Fedora release torrents |
|
|
13
|
+
| configurable name | Torznab-compatible API | Optional; trust and legality depend on the configured service |
|
|
14
|
+
|
|
15
|
+
The first three are enabled by default. Torznab endpoints are opt-in because they vary in quality,
|
|
16
|
+
policy, authentication, and content.
|
|
17
|
+
|
|
18
|
+
## Interface
|
|
19
|
+
|
|
20
|
+
```python
|
|
21
|
+
from typing import Protocol
|
|
22
|
+
|
|
23
|
+
from magnet_scout.models import TorrentResult
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class SearchProvider(Protocol):
|
|
27
|
+
name: str
|
|
28
|
+
|
|
29
|
+
async def search(self, query: str, limit: int) -> list[TorrentResult]: ...
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
A provider should:
|
|
33
|
+
|
|
34
|
+
1. Use a structured, documented endpoint where possible.
|
|
35
|
+
2. Apply the supplied result limit and the shared HTTP client's timeout.
|
|
36
|
+
3. Produce a valid magnet URI and place the source name in `providers`.
|
|
37
|
+
4. Put source pages in `provider_urls`, not arbitrary download destinations.
|
|
38
|
+
5. Treat seed and leech counts as reported data only.
|
|
39
|
+
6. Leave unavailable values as `None`.
|
|
40
|
+
7. Include offline fixtures and tests for success, malformed data, and server failure.
|
|
41
|
+
|
|
42
|
+
Register a default provider in `providers/registry.py`. A provider exception is deliberately
|
|
43
|
+
isolated by `SearchService`; do not catch broad exceptions merely to return an empty list.
|
|
44
|
+
|
|
45
|
+
## Torznab configuration
|
|
46
|
+
|
|
47
|
+
Create `${XDG_CONFIG_HOME:-~/.config}/magnet-scout/config.toml`:
|
|
48
|
+
|
|
49
|
+
```toml
|
|
50
|
+
[[torznab]]
|
|
51
|
+
name = "my-index"
|
|
52
|
+
url = "https://example.invalid/api"
|
|
53
|
+
api_key_env = "MY_INDEX_API_KEY"
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Then export the key and select the configured provider:
|
|
57
|
+
|
|
58
|
+
```console
|
|
59
|
+
export MY_INDEX_API_KEY='...'
|
|
60
|
+
magnet-scout search example --provider my-index
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
MagnetScout does not endorse or determine the legality of a configured Torznab source.
|