sciop-cli 0.1.1__tar.gz → 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/PKG-INFO +1 -1
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/pyproject.toml +1 -1
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/src/sciop_cli/cli/torrent.py +22 -6
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/src/sciop_cli/const.py +1 -1
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/src/sciop_cli/data.py +1 -0
- sciop_cli-0.1.2/src/sciop_cli/exceptions.py +42 -0
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/src/sciop_cli/torrent.py +47 -2
- sciop_cli-0.1.2/tests/test_cli/test_torrent.py +122 -0
- sciop_cli-0.1.2/tests/test_data.py +26 -0
- sciop_cli-0.1.1/tests/test_cli/test_torrent.py +0 -40
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/LICENSE +0 -0
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/README.md +0 -0
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/src/sciop_cli/__init__.py +0 -0
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/src/sciop_cli/__main__.py +0 -0
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/src/sciop_cli/cli/__init__.py +0 -0
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/src/sciop_cli/cli/main.py +0 -0
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/src/sciop_cli/cli/manifest.py +0 -0
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/src/sciop_cli/file.py +0 -0
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/src/sciop_cli/models/__init__.py +0 -0
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/src/sciop_cli/schema.py +0 -0
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/src/sciop_cli/types.py +0 -0
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/tests/.DS_Store +0 -0
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/tests/__init__.py +0 -0
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/tests/conftest.py +0 -0
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/tests/data/basic/part_1/subdir/part_1 +0 -0
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/tests/data/basic/part_2/subdir/part_2 +0 -0
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/tests/data/basic.torrent +0 -0
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/tests/test_cli/__init__.py +0 -0
- {sciop_cli-0.1.1 → sciop_cli-0.1.2}/tests/test_temp.py +0 -0
|
@@ -46,10 +46,15 @@ def pack() -> None:
|
|
|
46
46
|
"If not present, use the default trackers from https://sciop.net/docs/uploading/default_trackers.txt",
|
|
47
47
|
)
|
|
48
48
|
@click.option(
|
|
49
|
-
"--no-default-trackers",
|
|
49
|
+
"--default-trackers/--no-default-trackers",
|
|
50
50
|
is_flag=True,
|
|
51
|
-
default=
|
|
52
|
-
help="
|
|
51
|
+
default=None,
|
|
52
|
+
help="If trackers are specified with --tracker, "
|
|
53
|
+
"--default-trackers appends the default trackers to that list, "
|
|
54
|
+
"otherwise just use the supplied trackers (--no-default-trackers has no effect). "
|
|
55
|
+
"If no trackers are specified, "
|
|
56
|
+
"--no-default-trackers prevents adding the default tracker list,"
|
|
57
|
+
"which is done by default (--default-trackers has no effect).",
|
|
53
58
|
)
|
|
54
59
|
@click.option(
|
|
55
60
|
"-s",
|
|
@@ -100,8 +105,8 @@ def pack() -> None:
|
|
|
100
105
|
)
|
|
101
106
|
def create(
|
|
102
107
|
path: Path,
|
|
103
|
-
tracker: list[str] | None = None,
|
|
104
|
-
|
|
108
|
+
tracker: list[str] | tuple[str] | None = None,
|
|
109
|
+
default_trackers: bool | None = None,
|
|
105
110
|
piece_size: PieceSize = 512 * (2**10),
|
|
106
111
|
comment: str | None = None,
|
|
107
112
|
creator: str = DEFAULT_TORRENT_CREATOR,
|
|
@@ -120,8 +125,19 @@ def create(
|
|
|
120
125
|
form details on fields, all input here is passed through to
|
|
121
126
|
libtorrent's creation methods.
|
|
122
127
|
"""
|
|
123
|
-
|
|
128
|
+
# recast tuple to list or none rather than tuple or empty tuple
|
|
129
|
+
tracker = list(tracker) if tracker else None
|
|
130
|
+
|
|
131
|
+
if not tracker and (default_trackers is None or default_trackers):
|
|
132
|
+
click.echo(
|
|
133
|
+
"No trackers specified, using default trackers from "
|
|
134
|
+
"sciop.net/docs/uploading/default_trackers.txt"
|
|
135
|
+
)
|
|
124
136
|
tracker = get_default_trackers()
|
|
137
|
+
elif tracker and default_trackers:
|
|
138
|
+
default_tracker_list = get_default_trackers()
|
|
139
|
+
tracker.extend(default_tracker_list)
|
|
140
|
+
|
|
125
141
|
result = create_torrent(
|
|
126
142
|
path,
|
|
127
143
|
trackers=tracker,
|
|
@@ -16,6 +16,7 @@ DEFAULT_TRACKER_SOURCE = "https://sciop.net/docs/uploading/default_trackers.txt"
|
|
|
16
16
|
|
|
17
17
|
def get_default_trackers(tracker_source: str = DEFAULT_TRACKER_SOURCE) -> list[str]:
|
|
18
18
|
if not DEFAULT_TRACKERS.exists():
|
|
19
|
+
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
|
19
20
|
res = requests.get(tracker_source)
|
|
20
21
|
res.raise_for_status()
|
|
21
22
|
with open(DEFAULT_TRACKERS, "w") as f:
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Exception and warning classes.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
# --------------------------------------------------
|
|
6
|
+
# Exceptions
|
|
7
|
+
# --------------------------------------------------
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class SciopException(Exception):
|
|
11
|
+
"""
|
|
12
|
+
Generic top-level exception class.
|
|
13
|
+
Mixin other base exception classes as needed.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class TorrentCreateException(SciopException):
|
|
18
|
+
"""Exception during torrent creation"""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class NoFilesException(TorrentCreateException, ValueError, FileNotFoundError):
|
|
22
|
+
"""No files found for a created torrent, e.g. from an empty directory"""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
# --------------------------------------------------
|
|
26
|
+
# Warnings
|
|
27
|
+
# --------------------------------------------------
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class SciopWarning(UserWarning):
|
|
31
|
+
"""
|
|
32
|
+
Generic top-level warning class.
|
|
33
|
+
Mixin other base warning classes as needed
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class TorrentCreateWarning(SciopWarning, RuntimeWarning):
|
|
38
|
+
"""A warning emitted during torrent creation"""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class NoTrackersWarning(TorrentCreateWarning):
|
|
42
|
+
"""No trackers passed while creating a torrent"""
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import warnings
|
|
1
2
|
from math import floor
|
|
2
3
|
from pathlib import Path
|
|
3
4
|
|
|
@@ -7,6 +8,7 @@ from pydantic import TypeAdapter
|
|
|
7
8
|
from tqdm import tqdm
|
|
8
9
|
|
|
9
10
|
from sciop_cli.const import DEFAULT_TORRENT_CREATOR, EXCLUDE_FILES
|
|
11
|
+
from sciop_cli.exceptions import NoTrackersWarning
|
|
10
12
|
from sciop_cli.types import PieceSize
|
|
11
13
|
|
|
12
14
|
|
|
@@ -42,6 +44,37 @@ def create_torrent(
|
|
|
42
44
|
piece_size: PieceSize = 512 * (2**10),
|
|
43
45
|
pbar: bool = False,
|
|
44
46
|
) -> dict | bytes:
|
|
47
|
+
"""
|
|
48
|
+
Create a hybrid v1/v2 torrent with libtorrent
|
|
49
|
+
|
|
50
|
+
Args:
|
|
51
|
+
path: File or directory to create a torrent for
|
|
52
|
+
trackers: list of trackers to add to the torrent.
|
|
53
|
+
Each torrent is added on a separate tier, in order,
|
|
54
|
+
so that by default clients announce to all trackers
|
|
55
|
+
comment: Additional comment string embedded in the torrent
|
|
56
|
+
creator: Annotation of what tool was used to create the torrent,
|
|
57
|
+
defaults to `sciop-cli-{version}`
|
|
58
|
+
webseeds: List of HTTP urls where the content can also be found.
|
|
59
|
+
If `path` is a directory, the files on the server
|
|
60
|
+
must match the directory structure of the torrent and their content exactly.
|
|
61
|
+
similar: Infohashes of torrents that have identical files to those in this torrent.
|
|
62
|
+
Clients can use this to deduplicate downloads.
|
|
63
|
+
bencode: If ``True`` ( default ) return the bencoded bytestring.
|
|
64
|
+
Otherwise return a python dictionary
|
|
65
|
+
piece_size: The size of data chunks to hash.
|
|
66
|
+
Choosing a piece size can be complicated, but ideally you want
|
|
67
|
+
to pick a piece size that yields 20-50k pieces, and less than <100k.
|
|
68
|
+
Smaller torrents can have fewer pieces, in that case match the piece size
|
|
69
|
+
to being slightly smaller than the median file size.
|
|
70
|
+
Once torrents start to have >200k pieces, clients suffer to efficiently
|
|
71
|
+
track which peers have what pieces, and also struggle to cache the data.
|
|
72
|
+
pbar: It ``True``, show a progress bar while hashing pieces.
|
|
73
|
+
|
|
74
|
+
Returns:
|
|
75
|
+
bytes: bencoded torrent ready for writing
|
|
76
|
+
dict: python-formatted torrent file dict
|
|
77
|
+
"""
|
|
45
78
|
path = Path(path)
|
|
46
79
|
fs = libtorrent.file_storage()
|
|
47
80
|
|
|
@@ -62,13 +95,25 @@ def create_torrent(
|
|
|
62
95
|
else:
|
|
63
96
|
fs.add_file(path.name, path.stat().st_size)
|
|
64
97
|
|
|
98
|
+
if fs.num_files() == 0:
|
|
99
|
+
raise
|
|
100
|
+
|
|
65
101
|
piece_size = TypeAdapter(PieceSize).validate_python(piece_size)
|
|
66
102
|
|
|
67
103
|
torrent = libtorrent.create_torrent(fs, piece_size=piece_size)
|
|
68
104
|
|
|
69
105
|
if trackers:
|
|
70
|
-
for tracker in trackers:
|
|
71
|
-
torrent.add_tracker(tracker)
|
|
106
|
+
for tier, tracker in enumerate(trackers):
|
|
107
|
+
torrent.add_tracker(tracker, tier)
|
|
108
|
+
else:
|
|
109
|
+
warnings.warn(
|
|
110
|
+
"No trackers passed when creating a torrent. "
|
|
111
|
+
"This torrent will likely not be able to be seeded efficiently. "
|
|
112
|
+
"Consider adding trackers, or use the default trackers "
|
|
113
|
+
"(with `sciop_cli.data.get_default_trackers()`)",
|
|
114
|
+
NoTrackersWarning,
|
|
115
|
+
stacklevel=2,
|
|
116
|
+
)
|
|
72
117
|
|
|
73
118
|
if webseeds:
|
|
74
119
|
for webseed in webseeds:
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import bencode_rs
|
|
2
|
+
import pytest
|
|
3
|
+
from click.testing import CliRunner
|
|
4
|
+
|
|
5
|
+
from sciop_cli.cli import torrent
|
|
6
|
+
from sciop_cli.data import get_default_trackers
|
|
7
|
+
from sciop_cli.exceptions import NoTrackersWarning
|
|
8
|
+
|
|
9
|
+
from ..conftest import DATA_DIR
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def test_create_torrent(tmp_path):
|
|
13
|
+
trackers = ["udp://example.com:6969", "http://example.com/announce.php"]
|
|
14
|
+
webseeds = ["https://example.com/files"]
|
|
15
|
+
output = tmp_path / "basic.torrent"
|
|
16
|
+
args = [
|
|
17
|
+
"-p",
|
|
18
|
+
str(DATA_DIR / "basic"),
|
|
19
|
+
"-o",
|
|
20
|
+
str(output),
|
|
21
|
+
"--creator",
|
|
22
|
+
"sciop-tests",
|
|
23
|
+
"--comment",
|
|
24
|
+
"test",
|
|
25
|
+
"-s",
|
|
26
|
+
16 * (2**10),
|
|
27
|
+
]
|
|
28
|
+
for tracker in trackers:
|
|
29
|
+
args.extend(["--tracker", tracker])
|
|
30
|
+
for webseed in webseeds:
|
|
31
|
+
args.extend(["--webseed", webseed])
|
|
32
|
+
|
|
33
|
+
runner = CliRunner()
|
|
34
|
+
result = runner.invoke(torrent.create, args)
|
|
35
|
+
assert result.exit_code == 0
|
|
36
|
+
expected = bencode_rs.bdecode((DATA_DIR / "basic.torrent").read_bytes())
|
|
37
|
+
created = bencode_rs.bdecode(output.read_bytes())
|
|
38
|
+
|
|
39
|
+
# remove creation_date, which can't be set from python libtorrent
|
|
40
|
+
del expected[b"creation date"]
|
|
41
|
+
del created[b"creation date"]
|
|
42
|
+
|
|
43
|
+
# qbittorrent assigns trackers to the same tier by default,
|
|
44
|
+
# while we assign each to a different tier.
|
|
45
|
+
expected[b"announce-list"] = [[t] for t in expected[b"announce-list"][0]]
|
|
46
|
+
|
|
47
|
+
assert bencode_rs.bencode(expected) == bencode_rs.bencode(created)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@pytest.mark.parametrize(
|
|
51
|
+
"tracker",
|
|
52
|
+
[
|
|
53
|
+
["udp://otherexample.com/:6969"],
|
|
54
|
+
["udp://otherexample.com/:6969", "udp://thirdexample.com/:6969"],
|
|
55
|
+
None,
|
|
56
|
+
],
|
|
57
|
+
)
|
|
58
|
+
@pytest.mark.parametrize("default_trackers", [True, False, None])
|
|
59
|
+
def test_create_torrent_with_default_trackers(tmp_path, tracker, default_trackers, capsys):
|
|
60
|
+
"""
|
|
61
|
+
- If we aren't given any trackers, and we don't specify default trackers, we add them
|
|
62
|
+
- If we aren't given any trackers, and we specify no default trackers,
|
|
63
|
+
we don't and emit a warning
|
|
64
|
+
- If we are given trackers, and we don't specify default trackers, we don't add them
|
|
65
|
+
- If we are given trackers, and we specify default trackers, we add them.
|
|
66
|
+
"""
|
|
67
|
+
output = tmp_path / "basic.torrent"
|
|
68
|
+
args = [
|
|
69
|
+
"-p",
|
|
70
|
+
str(DATA_DIR / "basic"),
|
|
71
|
+
"-o",
|
|
72
|
+
str(output),
|
|
73
|
+
]
|
|
74
|
+
if tracker:
|
|
75
|
+
for t in tracker:
|
|
76
|
+
args.extend(["--tracker", t])
|
|
77
|
+
|
|
78
|
+
if default_trackers:
|
|
79
|
+
args.append("--default-trackers")
|
|
80
|
+
elif default_trackers is False:
|
|
81
|
+
# aka it's False
|
|
82
|
+
args.append("--no-default-trackers")
|
|
83
|
+
|
|
84
|
+
runner = CliRunner()
|
|
85
|
+
if default_trackers is False and not tracker:
|
|
86
|
+
# ensure we warn when creating torrents with no trackers
|
|
87
|
+
with pytest.warns(NoTrackersWarning):
|
|
88
|
+
result = runner.invoke(torrent.create, args)
|
|
89
|
+
else:
|
|
90
|
+
result = runner.invoke(torrent.create, args)
|
|
91
|
+
|
|
92
|
+
assert result.exit_code == 0
|
|
93
|
+
|
|
94
|
+
created = bencode_rs.bdecode(output.read_bytes())
|
|
95
|
+
|
|
96
|
+
# if creating a torrent with no trackers, return early
|
|
97
|
+
if default_trackers is False and not tracker:
|
|
98
|
+
assert b"announce" not in created
|
|
99
|
+
assert b"announce-list" not in created
|
|
100
|
+
return
|
|
101
|
+
# otherwise if we have a single tracker and no defaults, there isn't an announce list
|
|
102
|
+
elif (tracker and len(tracker) == 1) and (
|
|
103
|
+
default_trackers is None or default_trackers is False
|
|
104
|
+
):
|
|
105
|
+
assert created[b"announce"] == tracker[0].encode("utf-8")
|
|
106
|
+
assert b"announce-list" not in created
|
|
107
|
+
return
|
|
108
|
+
|
|
109
|
+
default_tracker_list = get_default_trackers()
|
|
110
|
+
flat_tracker_list = []
|
|
111
|
+
for tier in created[b"announce-list"]:
|
|
112
|
+
flat_tracker_list.extend([t.decode("utf-8") for t in tier])
|
|
113
|
+
|
|
114
|
+
# check if we have default trackers (if we should)
|
|
115
|
+
if default_trackers or (not tracker and default_trackers is None):
|
|
116
|
+
assert all([t in flat_tracker_list for t in default_tracker_list])
|
|
117
|
+
else:
|
|
118
|
+
assert not any([t in flat_tracker_list for t in default_tracker_list])
|
|
119
|
+
|
|
120
|
+
# if we supplied a tracker, it should be included and it should be first
|
|
121
|
+
if tracker:
|
|
122
|
+
assert flat_tracker_list[0] == tracker[0]
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from time import sleep
|
|
2
|
+
|
|
3
|
+
from pydantic import AnyUrl, TypeAdapter
|
|
4
|
+
|
|
5
|
+
from sciop_cli.data import DEFAULT_TRACKERS, get_default_trackers
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def test_get_default_trackers():
|
|
9
|
+
"""
|
|
10
|
+
We should be able to get default trackers,
|
|
11
|
+
and only get them once if they already exist
|
|
12
|
+
"""
|
|
13
|
+
DEFAULT_TRACKERS.unlink(missing_ok=True)
|
|
14
|
+
trackers = get_default_trackers()
|
|
15
|
+
assert DEFAULT_TRACKERS.exists()
|
|
16
|
+
# we got some trackers
|
|
17
|
+
assert len(trackers) > 0
|
|
18
|
+
# they are all URIs
|
|
19
|
+
validator = TypeAdapter(AnyUrl)
|
|
20
|
+
assert all([validator.validate_python(t) for t in trackers])
|
|
21
|
+
|
|
22
|
+
# ensure we don't get it twice
|
|
23
|
+
first_mtime = DEFAULT_TRACKERS.stat().st_mtime_ns
|
|
24
|
+
sleep(0.001)
|
|
25
|
+
trackers2 = get_default_trackers()
|
|
26
|
+
assert DEFAULT_TRACKERS.stat().st_mtime_ns == first_mtime
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
import bencode_rs
|
|
2
|
-
from click.testing import CliRunner
|
|
3
|
-
|
|
4
|
-
from sciop_cli.cli import torrent
|
|
5
|
-
|
|
6
|
-
from ..conftest import DATA_DIR
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
def test_create_torrent(tmp_path):
|
|
10
|
-
trackers = ["udp://example.com:6969", "http://example.com/announce.php"]
|
|
11
|
-
webseeds = ["https://example.com/files"]
|
|
12
|
-
output = tmp_path / "basic.torrent"
|
|
13
|
-
args = [
|
|
14
|
-
"-p",
|
|
15
|
-
str(DATA_DIR / "basic"),
|
|
16
|
-
"-o",
|
|
17
|
-
str(output),
|
|
18
|
-
"--creator",
|
|
19
|
-
"sciop-tests",
|
|
20
|
-
"--comment",
|
|
21
|
-
"test",
|
|
22
|
-
"-s",
|
|
23
|
-
16 * (2**10),
|
|
24
|
-
]
|
|
25
|
-
for tracker in trackers:
|
|
26
|
-
args.extend(["--tracker", tracker])
|
|
27
|
-
for webseed in webseeds:
|
|
28
|
-
args.extend(["--webseed", webseed])
|
|
29
|
-
|
|
30
|
-
runner = CliRunner()
|
|
31
|
-
result = runner.invoke(torrent.create, args)
|
|
32
|
-
assert result.exit_code == 0
|
|
33
|
-
expected = bencode_rs.bdecode((DATA_DIR / "basic.torrent").read_bytes())
|
|
34
|
-
created = bencode_rs.bdecode(output.read_bytes())
|
|
35
|
-
|
|
36
|
-
# remove creation_date, which can't be set from python libtorrent
|
|
37
|
-
del expected[b"creation date"]
|
|
38
|
-
del created[b"creation date"]
|
|
39
|
-
|
|
40
|
-
assert bencode_rs.bencode(expected) == bencode_rs.bencode(created)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|