sciop-cli 0.1.3__tar.gz → 0.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/PKG-INFO +2 -1
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/pyproject.toml +5 -1
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/cli/torrent.py +69 -5
- sciop_cli-0.1.4/src/sciop_cli/const.py +14 -0
- sciop_cli-0.1.4/src/sciop_cli/torrent.py +574 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/types.py +12 -0
- sciop_cli-0.1.4/tests/test_torrent.py +116 -0
- sciop_cli-0.1.3/src/sciop_cli/const.py +0 -4
- sciop_cli-0.1.3/src/sciop_cli/torrent.py +0 -157
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/LICENSE +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/README.md +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/__init__.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/__main__.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/cli/__init__.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/cli/main.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/cli/manifest.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/data.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/exceptions.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/file.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/models/__init__.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/schema.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/.DS_Store +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/__init__.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/conftest.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/data/basic/part_1/subdir/part_1 +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/data/basic/part_2/subdir/part_2 +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/data/basic_hybrid.torrent +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/data/basic_v2.torrent +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/test_cli/__init__.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/test_cli/test_torrent.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/test_data.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/test_temp.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: sciop-cli
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.4
|
|
4
4
|
Summary: A CLI for sciop
|
|
5
5
|
Author-Email: cosmicexplorer <dmcC2@hypnicjerk.ai>, sneakers-the-rat <sneakers-the-rat@protonmail.com>
|
|
6
6
|
License: EUPL-1.2
|
|
@@ -15,6 +15,7 @@ Requires-Dist: platformdirs>=4.3.7
|
|
|
15
15
|
Requires-Dist: requests>=2.32.3
|
|
16
16
|
Requires-Dist: tqdm>=4.67.1
|
|
17
17
|
Requires-Dist: rich>=14.0.0
|
|
18
|
+
Requires-Dist: humanize>=4.12.2
|
|
18
19
|
Provides-Extra: test
|
|
19
20
|
Requires-Dist: pytest>=8.3.4; extra == "test"
|
|
20
21
|
Provides-Extra: mypy
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sciop-cli"
|
|
3
|
-
version = "0.1.
|
|
3
|
+
version = "0.1.4"
|
|
4
4
|
description = "A CLI for sciop"
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "cosmicexplorer", email = "dmcC2@hypnicjerk.ai" },
|
|
@@ -17,6 +17,7 @@ dependencies = [
|
|
|
17
17
|
"requests>=2.32.3",
|
|
18
18
|
"tqdm>=4.67.1",
|
|
19
19
|
"rich>=14.0.0",
|
|
20
|
+
"humanize>=4.12.2",
|
|
20
21
|
]
|
|
21
22
|
requires-python = ">=3.11"
|
|
22
23
|
readme = "README.md"
|
|
@@ -148,3 +149,6 @@ show_column_numbers = true
|
|
|
148
149
|
show_error_code_links = true
|
|
149
150
|
pretty = true
|
|
150
151
|
color_output = true
|
|
152
|
+
|
|
153
|
+
[tool.pytest.ini_options]
|
|
154
|
+
markers = " v1: tests for v1 torrents\n v2: tests for v2 torrents\n hybrid: tests for hybrid torrents\n"
|
|
@@ -1,11 +1,23 @@
|
|
|
1
1
|
from pathlib import Path
|
|
2
|
+
from typing import Literal as L
|
|
2
3
|
from typing import cast
|
|
3
4
|
|
|
4
5
|
import click
|
|
6
|
+
import humanize
|
|
7
|
+
from rich import print
|
|
8
|
+
from rich.console import Group
|
|
9
|
+
from rich.panel import Panel
|
|
10
|
+
from rich.table import Table
|
|
5
11
|
|
|
6
12
|
from sciop_cli.const import DEFAULT_TORRENT_CREATOR
|
|
7
13
|
from sciop_cli.data import get_default_trackers
|
|
8
|
-
from sciop_cli.torrent import
|
|
14
|
+
from sciop_cli.torrent import (
|
|
15
|
+
calculate_overhead,
|
|
16
|
+
calculate_total_pieces,
|
|
17
|
+
create_torrent,
|
|
18
|
+
find_optimal_piece_size,
|
|
19
|
+
iter_files,
|
|
20
|
+
)
|
|
9
21
|
from sciop_cli.types import PieceSize
|
|
10
22
|
|
|
11
23
|
|
|
@@ -59,8 +71,9 @@ def pack() -> None:
|
|
|
59
71
|
@click.option(
|
|
60
72
|
"-s",
|
|
61
73
|
"--piece-size",
|
|
62
|
-
default=
|
|
63
|
-
help="Piece size, in bytes"
|
|
74
|
+
default=None,
|
|
75
|
+
help="Piece size, in bytes. If not given, calculate piece size automatically."
|
|
76
|
+
"Use `sciop-cli torrent piece-size` to preview the ",
|
|
64
77
|
show_default=True,
|
|
65
78
|
)
|
|
66
79
|
@click.option(
|
|
@@ -114,7 +127,7 @@ def create(
|
|
|
114
127
|
path: Path,
|
|
115
128
|
tracker: list[str] | tuple[str] | None = None,
|
|
116
129
|
default_trackers: bool | None = None,
|
|
117
|
-
piece_size: PieceSize
|
|
130
|
+
piece_size: PieceSize | None = None,
|
|
118
131
|
comment: str | None = None,
|
|
119
132
|
creator: str = DEFAULT_TORRENT_CREATOR,
|
|
120
133
|
webseed: list[str] | None = None,
|
|
@@ -135,7 +148,14 @@ def create(
|
|
|
135
148
|
"""
|
|
136
149
|
# recast tuple to list or none rather than tuple or empty tuple
|
|
137
150
|
tracker = list(tracker) if tracker else None
|
|
151
|
+
version = "v2" if v2 else "hybrid"
|
|
152
|
+
version = cast(L["v2", "hybrid"], version)
|
|
153
|
+
|
|
154
|
+
if piece_size is None:
|
|
155
|
+
click.echo("No piece size specified, estimating optimal piece size")
|
|
156
|
+
piece_size = find_optimal_piece_size(path=Path(path).absolute(), version=version)
|
|
138
157
|
|
|
158
|
+
click.echo(f"Piece size estimated as {humanize.naturalsize(piece_size, binary=True)}")
|
|
139
159
|
if not tracker and (default_trackers is None or default_trackers):
|
|
140
160
|
click.echo(
|
|
141
161
|
"No trackers specified, using default trackers from "
|
|
@@ -154,7 +174,7 @@ def create(
|
|
|
154
174
|
creator=creator,
|
|
155
175
|
webseeds=webseed,
|
|
156
176
|
similar=similar,
|
|
157
|
-
version=
|
|
177
|
+
version=version,
|
|
158
178
|
pbar=progress,
|
|
159
179
|
bencode=True,
|
|
160
180
|
)
|
|
@@ -164,3 +184,47 @@ def create(
|
|
|
164
184
|
f.write(result)
|
|
165
185
|
else:
|
|
166
186
|
click.echo(result)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
@torrent.command("piece-size")
|
|
190
|
+
@click.option(
|
|
191
|
+
"-p",
|
|
192
|
+
"--path",
|
|
193
|
+
required=True,
|
|
194
|
+
help="Path to a directory or file to create .torrent from",
|
|
195
|
+
type=click.Path(exists=True),
|
|
196
|
+
)
|
|
197
|
+
def piece_size(path: Path) -> None:
|
|
198
|
+
"""
|
|
199
|
+
Show the optimal piece sizes calculated for each torrent version
|
|
200
|
+
for a given file or directory.
|
|
201
|
+
"""
|
|
202
|
+
path = Path(path)
|
|
203
|
+
paths = [path] if path.is_file() else list(iter_files(path))
|
|
204
|
+
|
|
205
|
+
sizes = [p.stat().st_size for p in paths]
|
|
206
|
+
versions = ("v1", "v2", "hybrid")
|
|
207
|
+
piece_sizes = {v: find_optimal_piece_size(path=paths, sizes=sizes, version=v) for v in versions}
|
|
208
|
+
n_pieces = {v: calculate_total_pieces(sizes, piece_sizes[v], v) for v in versions}
|
|
209
|
+
hybrid_overhead = sum(calculate_overhead(sizes, piece_sizes["hybrid"]))
|
|
210
|
+
|
|
211
|
+
summary = Table(show_header=False)
|
|
212
|
+
summary.add_column("", style="bold magenta")
|
|
213
|
+
summary.add_column("")
|
|
214
|
+
summary.add_row("N Files", humanize.number.intcomma(len(paths)))
|
|
215
|
+
summary.add_row("Total size", humanize.naturalsize(sum(sizes), binary=True))
|
|
216
|
+
|
|
217
|
+
pieces = Table(title="Piece sizes")
|
|
218
|
+
pieces.add_column("Version")
|
|
219
|
+
pieces.add_column("Piece Size")
|
|
220
|
+
pieces.add_column("N Pieces")
|
|
221
|
+
pieces.add_column("Padding Overhead")
|
|
222
|
+
for v in ("v1", "v2", "hybrid"):
|
|
223
|
+
row = [v, str(piece_sizes[v]), humanize.number.intcomma(n_pieces[v])]
|
|
224
|
+
|
|
225
|
+
if v == "hybrid":
|
|
226
|
+
row.append(humanize.naturalsize(hybrid_overhead, binary=True))
|
|
227
|
+
pieces.add_row(*row)
|
|
228
|
+
|
|
229
|
+
panel = Panel(Group(summary, pieces), title=str(path))
|
|
230
|
+
print(panel)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from importlib.metadata import version
|
|
2
|
+
|
|
3
|
+
from sciop_cli.types import Size
|
|
4
|
+
|
|
5
|
+
KiB = Size(2**10)
|
|
6
|
+
MiB = Size(2**20)
|
|
7
|
+
GiB = Size(2**30)
|
|
8
|
+
TiB = Size(2**40)
|
|
9
|
+
|
|
10
|
+
DEFAULT_TORRENT_CREATOR = f"sciop-cli {version('sciop-cli')}"
|
|
11
|
+
EXCLUDE_FILES = (".DS_Store", "Thumbs.db")
|
|
12
|
+
BLOCK_SIZE = 16 * KiB
|
|
13
|
+
PIECE_SIZES = tuple(BLOCK_SIZE * (2**exp) for exp in range(14))
|
|
14
|
+
"""16KiB to 128MiB, powers of two * block size."""
|
|
@@ -0,0 +1,574 @@
|
|
|
1
|
+
import warnings
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from math import ceil
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from statistics import median
|
|
6
|
+
from typing import Any, Generator, TypedDict, cast, overload
|
|
7
|
+
from typing import Literal as L
|
|
8
|
+
|
|
9
|
+
import bencode_rs
|
|
10
|
+
import libtorrent
|
|
11
|
+
from pydantic import TypeAdapter
|
|
12
|
+
from tqdm import tqdm
|
|
13
|
+
|
|
14
|
+
from sciop_cli.const import DEFAULT_TORRENT_CREATOR, EXCLUDE_FILES, PIECE_SIZES, GiB
|
|
15
|
+
from sciop_cli.exceptions import NoTrackersWarning
|
|
16
|
+
from sciop_cli.types import PieceSize
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class PieceParams:
|
|
21
|
+
target_pieces: int = 5000
|
|
22
|
+
"""Pick the piece size that gets us closest to this number of pieces"""
|
|
23
|
+
small_torrent_threshold: int = GiB
|
|
24
|
+
"""
|
|
25
|
+
For torrents with total sizes below this threshold (in bytes),
|
|
26
|
+
use the small_target_pieces target instead
|
|
27
|
+
"""
|
|
28
|
+
small_target_pieces: int = 250
|
|
29
|
+
"""Target pieces when the total size is beneath the small torrent threshold"""
|
|
30
|
+
target_weight: float = 1
|
|
31
|
+
"""Relative weight of the target_pieces value on piece size"""
|
|
32
|
+
|
|
33
|
+
@overload
|
|
34
|
+
@classmethod
|
|
35
|
+
def from_version(cls, version: L["v1"], **kwargs: Any) -> "V1PieceParams": ...
|
|
36
|
+
|
|
37
|
+
@overload
|
|
38
|
+
@classmethod
|
|
39
|
+
def from_version(cls, version: L["v2"], **kwargs: Any) -> "V2PieceParams": ...
|
|
40
|
+
|
|
41
|
+
@overload
|
|
42
|
+
@classmethod
|
|
43
|
+
def from_version(cls, version: L["hybrid"], **kwargs: Any) -> "HybridPieceParams": ...
|
|
44
|
+
|
|
45
|
+
@classmethod
|
|
46
|
+
def from_version(cls, version: L["v1", "v2", "hybrid"] | str, **kwargs: Any) -> "PieceParams":
|
|
47
|
+
"""Create params for specific version"""
|
|
48
|
+
if version == "v1":
|
|
49
|
+
return V1PieceParams(**kwargs)
|
|
50
|
+
elif version == "v2":
|
|
51
|
+
return V2PieceParams(**kwargs)
|
|
52
|
+
elif version == "hybrid":
|
|
53
|
+
return HybridPieceParams(**kwargs)
|
|
54
|
+
else:
|
|
55
|
+
raise ValueError(f"Unknown torrent version: {version}")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass
|
|
59
|
+
class V1PieceParams(PieceParams):
|
|
60
|
+
overhead_weight: float = 1.0e-10
|
|
61
|
+
"""
|
|
62
|
+
Relative weight of the v1 overhead induced by pad files.
|
|
63
|
+
|
|
64
|
+
Tiny tiny penalty, effectively only for breaking ties,
|
|
65
|
+
because v1 torrents don't really have a overhead per se
|
|
66
|
+
|
|
67
|
+
See :func:`.score_overhead` for details.
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass
|
|
72
|
+
class V2PieceParams(PieceParams):
|
|
73
|
+
median_weight: float = 0.05
|
|
74
|
+
"""
|
|
75
|
+
Relative weight of penalty for piece sizes that are larger than the median file size
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass
|
|
80
|
+
class HybridPieceParams(PieceParams):
|
|
81
|
+
overhead_weight: float = 1.0
|
|
82
|
+
"""
|
|
83
|
+
Relative weight of the v1 overhead induced by pad files.
|
|
84
|
+
This term is *not* normalized to force its range to 0-1 like the other penalties,
|
|
85
|
+
it is instead normalized to the total size of the torrent:
|
|
86
|
+
the absolute value of the size of the overhead matters as well as
|
|
87
|
+
the relative magnitude per each piece size.
|
|
88
|
+
"""
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class Penalty(TypedDict):
|
|
92
|
+
piece_size: int
|
|
93
|
+
type: str
|
|
94
|
+
value: float
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@overload
|
|
98
|
+
def find_optimal_piece_size(path: Path | list[Path]) -> int: ...
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@overload
|
|
102
|
+
def find_optimal_piece_size(
|
|
103
|
+
path: Path | list[Path], version: L["v1", "v2", "hybrid"] | str
|
|
104
|
+
) -> int: ...
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@overload
|
|
108
|
+
def find_optimal_piece_size(
|
|
109
|
+
path: Path | list[Path],
|
|
110
|
+
version: L["v1"],
|
|
111
|
+
sizes: list[int] | None,
|
|
112
|
+
params: V1PieceParams | None = None,
|
|
113
|
+
) -> int: ...
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@overload
|
|
117
|
+
def find_optimal_piece_size(
|
|
118
|
+
path: Path | list[Path],
|
|
119
|
+
version: L["v2"],
|
|
120
|
+
sizes: list[int] | None,
|
|
121
|
+
params: V2PieceParams | None = None,
|
|
122
|
+
) -> int: ...
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
@overload
|
|
126
|
+
def find_optimal_piece_size(
|
|
127
|
+
path: Path | list[Path],
|
|
128
|
+
version: L["hybrid"],
|
|
129
|
+
sizes: list[int] | None,
|
|
130
|
+
params: HybridPieceParams | None,
|
|
131
|
+
) -> int: ...
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@overload
|
|
135
|
+
def find_optimal_piece_size(
|
|
136
|
+
path: Path | list[Path],
|
|
137
|
+
version: L["v1", "v2", "hybrid"] | str,
|
|
138
|
+
sizes: list[int] | None,
|
|
139
|
+
) -> int: ...
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@overload
|
|
143
|
+
def find_optimal_piece_size(
|
|
144
|
+
path: Path | list[Path],
|
|
145
|
+
version: L["v1", "v2", "hybrid"] | str,
|
|
146
|
+
sizes: list[int] | None,
|
|
147
|
+
params: V1PieceParams | V2PieceParams | HybridPieceParams | dict | None,
|
|
148
|
+
return_penalties: L[True],
|
|
149
|
+
) -> tuple[int, dict[str, list[Penalty]]]: ...
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def find_optimal_piece_size(
|
|
153
|
+
path: Path | str | list[Path] | list[str],
|
|
154
|
+
version: L["v1", "v2", "hybrid"] | str = "hybrid",
|
|
155
|
+
sizes: list[int] | None = None,
|
|
156
|
+
params: V1PieceParams | V2PieceParams | HybridPieceParams | dict | None = None,
|
|
157
|
+
return_penalties: bool = False,
|
|
158
|
+
) -> int | tuple[int, dict[str, list[Penalty]]]:
|
|
159
|
+
"""
|
|
160
|
+
Estimate an optimal piece size for the given file(s).
|
|
161
|
+
|
|
162
|
+
Each version has slightly different considerations for piece size
|
|
163
|
+
- v1: Piece size determined by total size, pick a piece size that makes a
|
|
164
|
+
reasonable number of pieces (i.e. 1000 - 50000-ish)
|
|
165
|
+
- v2: Piece size determined by total size and size distribution.
|
|
166
|
+
Pick a piece size that balances a reasonable number of pieces with
|
|
167
|
+
the piece size being roughly the size of the median file.
|
|
168
|
+
- hybrid: Piece size determined by total size and padding overhead.
|
|
169
|
+
Pick a piece size that balances a reasonable number of pieces
|
|
170
|
+
without creating an enormous amount of padding overhead.
|
|
171
|
+
|
|
172
|
+
See the docstrings on the *PieceParams models for more details.
|
|
173
|
+
|
|
174
|
+
Args:
|
|
175
|
+
path (Path): File or directory of files to be added to torrent
|
|
176
|
+
version ("v1", "v2", "hybrid"): version of torrent to be created.
|
|
177
|
+
params (V1PieceParams | V2PieceParams | HybridPieceParams | dict):
|
|
178
|
+
Parameterization of piece size optimization.
|
|
179
|
+
See model docstrings for more detail.
|
|
180
|
+
return_penalties (bool): If ``True``, return the calculated penalties
|
|
181
|
+
|
|
182
|
+
Returns:
|
|
183
|
+
|
|
184
|
+
"""
|
|
185
|
+
version = cast(L["v1", "v2", "hybrid"], version)
|
|
186
|
+
if isinstance(path, list):
|
|
187
|
+
paths = [Path(p) for p in path] if isinstance(path[0], str) else path
|
|
188
|
+
else:
|
|
189
|
+
path = Path(path)
|
|
190
|
+
paths = [path] if path.is_file() else list(iter_files(path))
|
|
191
|
+
|
|
192
|
+
paths = cast(list[Path], paths)
|
|
193
|
+
|
|
194
|
+
if sizes is None:
|
|
195
|
+
sizes = [p.stat().st_size for p in paths]
|
|
196
|
+
|
|
197
|
+
assert len(sizes) == len(
|
|
198
|
+
paths
|
|
199
|
+
), "Length of provided sizes does not match length of detected paths"
|
|
200
|
+
|
|
201
|
+
if isinstance(params, dict):
|
|
202
|
+
params = PieceParams.from_version(version=version, **params)
|
|
203
|
+
elif params is None:
|
|
204
|
+
params = PieceParams.from_version(version=version)
|
|
205
|
+
|
|
206
|
+
if version == "v1":
|
|
207
|
+
params = cast(V1PieceParams | None, params)
|
|
208
|
+
return find_optimal_piece_size_v1(
|
|
209
|
+
paths=paths, sizes=sizes, params=params, return_penalties=return_penalties
|
|
210
|
+
)
|
|
211
|
+
elif version == "v2":
|
|
212
|
+
params = cast(V2PieceParams | None, params)
|
|
213
|
+
return find_optimal_piece_size_v2(
|
|
214
|
+
paths=paths, sizes=sizes, params=params, return_penalties=return_penalties
|
|
215
|
+
)
|
|
216
|
+
elif version == "hybrid":
|
|
217
|
+
params = cast(HybridPieceParams | None, params)
|
|
218
|
+
return find_optimal_piece_size_hybrid(
|
|
219
|
+
paths=paths, sizes=sizes, params=params, return_penalties=return_penalties
|
|
220
|
+
)
|
|
221
|
+
else:
|
|
222
|
+
raise ValueError(f"Unkown torrent version {version}")
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def find_optimal_piece_size_v1(
|
|
226
|
+
paths: list[Path],
|
|
227
|
+
sizes: list[int],
|
|
228
|
+
params: V1PieceParams | None = None,
|
|
229
|
+
return_penalties: bool = False,
|
|
230
|
+
) -> int | tuple[int, dict[str, list[Penalty]]]:
|
|
231
|
+
"""
|
|
232
|
+
Find a piece size that produces a reasonable number of pieces,
|
|
233
|
+
using overhead as a tiebreaker (e.g. for very small torrents where most piece sizes
|
|
234
|
+
would yield a single-piece torrent.)
|
|
235
|
+
"""
|
|
236
|
+
if params is None:
|
|
237
|
+
params = V1PieceParams()
|
|
238
|
+
target_pieces = _normalize_penalties(
|
|
239
|
+
_penalty_target_pieces(paths=paths, sizes=sizes, version="v1", params=params)
|
|
240
|
+
)
|
|
241
|
+
# mildly penalize overhead in the case of very small torrents
|
|
242
|
+
overhead = _penalty_overhead(paths=paths, sizes=sizes)
|
|
243
|
+
|
|
244
|
+
piece_size = _min_penalty(
|
|
245
|
+
penalties=[target_pieces, overhead],
|
|
246
|
+
weights=[params.target_weight, params.overhead_weight],
|
|
247
|
+
)
|
|
248
|
+
if return_penalties:
|
|
249
|
+
return piece_size, {"target": target_pieces, "overhead": overhead}
|
|
250
|
+
else:
|
|
251
|
+
return piece_size
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def find_optimal_piece_size_v2(
|
|
255
|
+
paths: list[Path],
|
|
256
|
+
sizes: list[int],
|
|
257
|
+
params: V2PieceParams | None = None,
|
|
258
|
+
return_penalties: bool = False,
|
|
259
|
+
) -> int | tuple[int, dict[str, list[Penalty]]]:
|
|
260
|
+
"""
|
|
261
|
+
Find a piece size that balances having a reasonable number of pieces
|
|
262
|
+
with the hashing overhead of having a piece size that is significantly larger
|
|
263
|
+
than the median filesize in a torrent
|
|
264
|
+
"""
|
|
265
|
+
if params is None:
|
|
266
|
+
params = V2PieceParams()
|
|
267
|
+
|
|
268
|
+
target_pieces = _normalize_penalties(
|
|
269
|
+
_penalty_target_pieces(paths=paths, sizes=sizes, version="v2", params=params)
|
|
270
|
+
)
|
|
271
|
+
med_size = _normalize_penalties(_penalty_median_size(paths=paths, sizes=sizes))
|
|
272
|
+
piece_size = _min_penalty(
|
|
273
|
+
penalties=[target_pieces, med_size],
|
|
274
|
+
weights=[params.target_weight, params.median_weight],
|
|
275
|
+
)
|
|
276
|
+
if return_penalties:
|
|
277
|
+
return piece_size, {"target": target_pieces, "median": med_size}
|
|
278
|
+
else:
|
|
279
|
+
return piece_size
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def find_optimal_piece_size_hybrid(
|
|
283
|
+
paths: list[Path],
|
|
284
|
+
sizes: list[int],
|
|
285
|
+
params: HybridPieceParams | None = None,
|
|
286
|
+
return_penalties: bool = False,
|
|
287
|
+
) -> int | tuple[int, dict[str, list[Penalty]]]:
|
|
288
|
+
"""
|
|
289
|
+
Find a piece size that minimizes padfile overhead and produces a reasonable number of pieces.
|
|
290
|
+
"""
|
|
291
|
+
if params is None:
|
|
292
|
+
params = HybridPieceParams()
|
|
293
|
+
|
|
294
|
+
# don't normalize overhead - it's normalized w.r.t. to the total size
|
|
295
|
+
# (e.g. a 100% overhead would be 1)
|
|
296
|
+
overhead = _penalty_overhead(paths=paths, sizes=sizes)
|
|
297
|
+
target_pieces = _normalize_penalties(
|
|
298
|
+
_penalty_target_pieces(paths=paths, sizes=sizes, version="hybrid", params=params)
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
piece_size = _min_penalty(
|
|
302
|
+
penalties=[overhead, target_pieces],
|
|
303
|
+
weights=[params.overhead_weight, params.target_weight],
|
|
304
|
+
)
|
|
305
|
+
if return_penalties:
|
|
306
|
+
return piece_size, {"target": target_pieces, "overhead": overhead}
|
|
307
|
+
else:
|
|
308
|
+
return piece_size
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def _min_penalty(penalties: list[list[Penalty]], weights: list[float]) -> int:
|
|
312
|
+
"""
|
|
313
|
+
Given a list of piece size penalties and weights,
|
|
314
|
+
return the piece size that minimizes the sum of the weighted penalties.
|
|
315
|
+
|
|
316
|
+
Args:
|
|
317
|
+
penalties (list[list[Penalty]]): List of penalty terms per piece size
|
|
318
|
+
weights (list[float]): list of weights, length == to number of penalty lists.
|
|
319
|
+
"""
|
|
320
|
+
assert len(penalties) == len(weights)
|
|
321
|
+
# collect penalties by piece size
|
|
322
|
+
piece_penalties = {}
|
|
323
|
+
for pens in zip(*penalties):
|
|
324
|
+
assert all(
|
|
325
|
+
[p["piece_size"] == pens[0]["piece_size"]] for p in pens
|
|
326
|
+
), "piece sizes must be sorted before scoring"
|
|
327
|
+
piece_penalties[pens[0]["piece_size"]] = sum(
|
|
328
|
+
[p["value"] * weight for p, weight in zip(pens, weights)]
|
|
329
|
+
)
|
|
330
|
+
|
|
331
|
+
# invert mapping to select piece size by penalty
|
|
332
|
+
penalty_pieces = {v: k for k, v in piece_penalties.items()}
|
|
333
|
+
return penalty_pieces[min(penalty_pieces.keys())]
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def calculate_overhead(sizes: list[int], piece_size: int) -> list[float]:
|
|
337
|
+
"""
|
|
338
|
+
Given a list of file sizes and a piece size, calculate expected v1 padfile overhead
|
|
339
|
+
"""
|
|
340
|
+
return [piece_size - (size % piece_size) if size != piece_size else 0 for size in sizes]
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def _penalty_overhead(paths: list[Path], sizes: list[int]) -> list[Penalty]:
|
|
344
|
+
"""
|
|
345
|
+
Total v1 padfile overhead for a given piece size,
|
|
346
|
+
Normalized to the total size of the given files.
|
|
347
|
+
"""
|
|
348
|
+
penalties = []
|
|
349
|
+
total_size = sum(sizes)
|
|
350
|
+
for piece_size in PIECE_SIZES:
|
|
351
|
+
overheads = calculate_overhead(sizes, piece_size)
|
|
352
|
+
penalties.append(
|
|
353
|
+
Penalty(
|
|
354
|
+
piece_size=piece_size,
|
|
355
|
+
type="overhead",
|
|
356
|
+
value=sum(overheads) / total_size,
|
|
357
|
+
)
|
|
358
|
+
)
|
|
359
|
+
return penalties
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def calculate_total_pieces(
|
|
363
|
+
sizes: list[int], piece_size: int, version: L["v1", "v2", "hybrid"] | str
|
|
364
|
+
) -> int:
|
|
365
|
+
if version == "v1":
|
|
366
|
+
total_size = sum(sizes)
|
|
367
|
+
total_pieces = ceil(total_size / piece_size)
|
|
368
|
+
elif version in ("v2", "hybrid"):
|
|
369
|
+
total_pieces = sum([ceil(size / piece_size) for size in sizes])
|
|
370
|
+
else:
|
|
371
|
+
raise ValueError(f"Unkown torrent version {version}")
|
|
372
|
+
return total_pieces
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _penalty_target_pieces(
|
|
376
|
+
paths: list[Path], sizes: list[int], version: L["v1", "v2", "hybrid"], params: PieceParams
|
|
377
|
+
) -> list[Penalty]:
|
|
378
|
+
"""
|
|
379
|
+
Absolute value of difference in actual pieces vs. target pieces, with specializations.
|
|
380
|
+
|
|
381
|
+
- If the largest possible piece size produces more pieces than the target number of pieces,
|
|
382
|
+
then we squash the penalty range so that we preserve the relative penalization of
|
|
383
|
+
the second to large pieces - i.e. we put stronger weight on making larger pieces,
|
|
384
|
+
assuming the values will be normalized in the next step.
|
|
385
|
+
- For piece sizes that make *fewer* pieces than the target piece number,
|
|
386
|
+
exponentially penalize piece sizes that make us make a very small number of pieces.
|
|
387
|
+
A torrent with a single piece is not very useful.
|
|
388
|
+
"""
|
|
389
|
+
total_size = sum(sizes)
|
|
390
|
+
if total_size < params.small_torrent_threshold:
|
|
391
|
+
target = params.small_target_pieces
|
|
392
|
+
else:
|
|
393
|
+
target = params.target_pieces
|
|
394
|
+
|
|
395
|
+
penalties = []
|
|
396
|
+
for piece_size in PIECE_SIZES:
|
|
397
|
+
total_pieces = calculate_total_pieces(sizes, piece_size, version)
|
|
398
|
+
penalty = float(abs(total_pieces - target))
|
|
399
|
+
|
|
400
|
+
# if we still can't make the target with the largest piece size,
|
|
401
|
+
# squash the range because making half as many pieces is much worse than usual
|
|
402
|
+
if (min_pieces := ceil(total_size / max(PIECE_SIZES))) > target:
|
|
403
|
+
penalty = penalty ** min(((target * 2) / min_pieces), 1)
|
|
404
|
+
# penalize going under more than going over
|
|
405
|
+
elif total_pieces < target:
|
|
406
|
+
# piece sizes that want to make us make one piece...
|
|
407
|
+
exponent = ((target - total_pieces) / target) * 2
|
|
408
|
+
penalty *= max(penalty * exponent, 1)
|
|
409
|
+
|
|
410
|
+
penalties.append(
|
|
411
|
+
Penalty(
|
|
412
|
+
piece_size=piece_size,
|
|
413
|
+
type="target",
|
|
414
|
+
value=penalty,
|
|
415
|
+
)
|
|
416
|
+
)
|
|
417
|
+
return penalties
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def _penalty_median_size(paths: list[Path], sizes: list[int]) -> list[Penalty]:
|
|
421
|
+
median_size = median(sizes)
|
|
422
|
+
|
|
423
|
+
return [
|
|
424
|
+
Penalty(piece_size=piece_size, type="median", value=max(piece_size - median_size, 0) ** 2)
|
|
425
|
+
for piece_size in PIECE_SIZES
|
|
426
|
+
]
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def _normalize_penalties(penalties: list[Penalty]) -> list[Penalty]:
|
|
430
|
+
"""Normalize the values of a single kind of penalty to be between 0 and 1"""
|
|
431
|
+
penalties = sorted(penalties, key=lambda p: p["piece_size"])
|
|
432
|
+
max_pen = max([p["value"] for p in penalties])
|
|
433
|
+
min_pen = min([p["value"] for p in penalties])
|
|
434
|
+
pen_range = max_pen - min_pen
|
|
435
|
+
for i in range(len(penalties)):
|
|
436
|
+
if pen_range == 0:
|
|
437
|
+
# all are equal
|
|
438
|
+
penalties[i]["value"] = 0
|
|
439
|
+
else:
|
|
440
|
+
penalties[i]["value"] = (penalties[i]["value"] - min_pen) / (max_pen - min_pen)
|
|
441
|
+
return penalties
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def create_torrent(
|
|
445
|
+
path: Path,
|
|
446
|
+
trackers: list[str] | None = None,
|
|
447
|
+
comment: str | None = None,
|
|
448
|
+
creator: str = DEFAULT_TORRENT_CREATOR,
|
|
449
|
+
webseeds: list[str] | None = None,
|
|
450
|
+
similar: list[str] | None = None,
|
|
451
|
+
version: L["v1", "v2", "hybrid"] = "hybrid",
|
|
452
|
+
bencode: bool = True,
|
|
453
|
+
piece_size: PieceSize = 512 * (2**10),
|
|
454
|
+
pbar: bool = False,
|
|
455
|
+
) -> dict | bytes:
|
|
456
|
+
"""
|
|
457
|
+
Create a hybrid v1/v2 torrent with libtorrent
|
|
458
|
+
|
|
459
|
+
Args:
|
|
460
|
+
path: File or directory to create a torrent for
|
|
461
|
+
trackers: list of trackers to add to the torrent.
|
|
462
|
+
Each torrent is added on a separate tier, in order,
|
|
463
|
+
so that by default clients announce to all trackers
|
|
464
|
+
comment: Additional comment string embedded in the torrent
|
|
465
|
+
creator: Annotation of what tool was used to create the torrent,
|
|
466
|
+
defaults to `sciop-cli-{version}`
|
|
467
|
+
webseeds: List of HTTP urls where the content can also be found.
|
|
468
|
+
If `path` is a directory, the files on the server
|
|
469
|
+
must match the directory structure of the torrent and their content exactly.
|
|
470
|
+
similar: Infohashes of torrents that have identical files to those in this torrent.
|
|
471
|
+
Clients can use this to deduplicate downloads.
|
|
472
|
+
version: Torrent version to create. Default ``"hybrid"`` creates v1/v2 compatible torrents.
|
|
473
|
+
v1 is *not recommended.*
|
|
474
|
+
bencode: If ``True`` ( default ) return the bencoded bytestring.
|
|
475
|
+
Otherwise return a python dictionary
|
|
476
|
+
piece_size: The size of data chunks to hash.
|
|
477
|
+
Choosing a piece size can be complicated, but ideally you want
|
|
478
|
+
to pick a piece size that yields 20-50k pieces, and less than <100k.
|
|
479
|
+
Smaller torrents can have fewer pieces, in that case match the piece size
|
|
480
|
+
to being slightly smaller than the median file size.
|
|
481
|
+
Once torrents start to have >200k pieces, clients suffer to efficiently
|
|
482
|
+
track which peers have what pieces, and also struggle to cache the data.
|
|
483
|
+
pbar: It ``True``, show a progress bar while hashing pieces.
|
|
484
|
+
|
|
485
|
+
Returns:
|
|
486
|
+
bytes: bencoded torrent ready for writing
|
|
487
|
+
dict: python-formatted torrent file dict
|
|
488
|
+
"""
|
|
489
|
+
path = Path(path)
|
|
490
|
+
fs = libtorrent.file_storage()
|
|
491
|
+
|
|
492
|
+
if path.is_dir():
|
|
493
|
+
# get paths and sort
|
|
494
|
+
paths = []
|
|
495
|
+
for _path in iter_files(path):
|
|
496
|
+
# no absolute paths in the torrent plz
|
|
497
|
+
rel_path = _path.relative_to(path)
|
|
498
|
+
# add the parent again as the root
|
|
499
|
+
rel_path = Path(path.name) / rel_path
|
|
500
|
+
paths.append((str(rel_path), _path.stat().st_size))
|
|
501
|
+
|
|
502
|
+
paths = sorted(paths, key=lambda x: x[0])
|
|
503
|
+
for p, size in paths:
|
|
504
|
+
fs.add_file(p, size)
|
|
505
|
+
|
|
506
|
+
else:
|
|
507
|
+
fs.add_file(path.name, path.stat().st_size)
|
|
508
|
+
|
|
509
|
+
if fs.num_files() == 0:
|
|
510
|
+
raise
|
|
511
|
+
|
|
512
|
+
piece_size = TypeAdapter(PieceSize).validate_python(piece_size)
|
|
513
|
+
|
|
514
|
+
flags = 0
|
|
515
|
+
if version == "v1":
|
|
516
|
+
flags = libtorrent.create_torrent.v1_only
|
|
517
|
+
elif version == "v2":
|
|
518
|
+
flags = libtorrent.create_torrent.v2_only
|
|
519
|
+
|
|
520
|
+
torrent = libtorrent.create_torrent(fs, piece_size=piece_size, flags=flags)
|
|
521
|
+
|
|
522
|
+
if trackers:
|
|
523
|
+
for tier, tracker in enumerate(trackers):
|
|
524
|
+
torrent.add_tracker(tracker, tier)
|
|
525
|
+
else:
|
|
526
|
+
warnings.warn(
|
|
527
|
+
"No trackers passed when creating a torrent. "
|
|
528
|
+
"This torrent will likely not be able to be seeded efficiently. "
|
|
529
|
+
"Consider adding trackers, or use the default trackers "
|
|
530
|
+
"(with `sciop_cli.data.get_default_trackers()`)",
|
|
531
|
+
NoTrackersWarning,
|
|
532
|
+
stacklevel=2,
|
|
533
|
+
)
|
|
534
|
+
|
|
535
|
+
if webseeds:
|
|
536
|
+
for webseed in webseeds:
|
|
537
|
+
torrent.add_url_seed(webseed)
|
|
538
|
+
|
|
539
|
+
if similar:
|
|
540
|
+
for s in similar:
|
|
541
|
+
torrent.add_similar_torrent(s)
|
|
542
|
+
|
|
543
|
+
if comment:
|
|
544
|
+
torrent.set_comment(comment)
|
|
545
|
+
|
|
546
|
+
torrent.set_creator(creator)
|
|
547
|
+
|
|
548
|
+
_pbar = None
|
|
549
|
+
if pbar:
|
|
550
|
+
_pbar = tqdm(desc="hashing pieces...", total=torrent.num_pieces())
|
|
551
|
+
|
|
552
|
+
def _pbar_callback(piece_index: int) -> None:
|
|
553
|
+
_pbar.update()
|
|
554
|
+
|
|
555
|
+
libtorrent.set_piece_hashes(torrent, str(path.parent.resolve()), _pbar_callback)
|
|
556
|
+
_pbar.close()
|
|
557
|
+
else:
|
|
558
|
+
libtorrent.set_piece_hashes(torrent, str(path.parent.resolve()))
|
|
559
|
+
|
|
560
|
+
ret = torrent.generate()
|
|
561
|
+
if bencode:
|
|
562
|
+
return bencode_rs.bencode(ret)
|
|
563
|
+
else:
|
|
564
|
+
return ret
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
def iter_files(path: Path) -> Generator[Path, None, None]:
|
|
568
|
+
"""
|
|
569
|
+
Recursively iterate through files, excluding system files
|
|
570
|
+
"""
|
|
571
|
+
for p in path.rglob("*"):
|
|
572
|
+
if p.name in EXCLUDE_FILES or p.is_dir():
|
|
573
|
+
continue
|
|
574
|
+
yield p
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from typing import Annotated
|
|
2
2
|
|
|
3
|
+
import humanize
|
|
3
4
|
from pydantic import AfterValidator
|
|
4
5
|
|
|
5
6
|
|
|
@@ -15,3 +16,14 @@ def _validate_size(size: int) -> int:
|
|
|
15
16
|
|
|
16
17
|
|
|
17
18
|
PieceSize = Annotated[int, AfterValidator(_validate_size)]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Size(int):
|
|
22
|
+
def __str__(self) -> str:
|
|
23
|
+
return humanize.naturalsize(int(self), binary=True)
|
|
24
|
+
|
|
25
|
+
def __rmul__(self, factor: int) -> "Size":
|
|
26
|
+
return Size(int(self) * factor)
|
|
27
|
+
|
|
28
|
+
def __mul__(self, other: int) -> "Size":
|
|
29
|
+
return Size(int(self) * other)
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
from random import gammavariate
|
|
2
|
+
from typing import Literal as L
|
|
3
|
+
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from sciop_cli.const import GiB, KiB, MiB, TiB
|
|
7
|
+
from sciop_cli.torrent import find_optimal_piece_size
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@pytest.mark.parametrize(
|
|
11
|
+
"version",
|
|
12
|
+
(
|
|
13
|
+
pytest.param("v1", marks=pytest.mark.v1),
|
|
14
|
+
pytest.param("v2", marks=pytest.mark.v2),
|
|
15
|
+
pytest.param("hybrid", marks=pytest.mark.hybrid),
|
|
16
|
+
),
|
|
17
|
+
)
|
|
18
|
+
@pytest.mark.parametrize(
|
|
19
|
+
"sizes,expected",
|
|
20
|
+
[
|
|
21
|
+
pytest.param([16 * KiB], 16 * KiB, id="single-block"),
|
|
22
|
+
pytest.param([KiB for _ in range(10)], 16 * KiB, id="10-10kb"),
|
|
23
|
+
pytest.param(
|
|
24
|
+
[MiB for _ in range(10)],
|
|
25
|
+
{"v1": 32 * KiB, "v2": 32 * KiB, "hybrid": 16 * KiB},
|
|
26
|
+
id="10-10mb",
|
|
27
|
+
),
|
|
28
|
+
pytest.param(
|
|
29
|
+
[10 * MiB + 1 for _ in range(10)],
|
|
30
|
+
{"v1": 256 * KiB, "v2": 256 * KiB, "hybrid": 128 * KiB},
|
|
31
|
+
id="10-100mb",
|
|
32
|
+
),
|
|
33
|
+
pytest.param(
|
|
34
|
+
[(100 * MiB) + 1 for _ in range(10)],
|
|
35
|
+
{"v1": 4 * MiB, "v2": 4 * MiB, "hybrid": 1 * MiB},
|
|
36
|
+
id="10-1gb",
|
|
37
|
+
),
|
|
38
|
+
pytest.param(
|
|
39
|
+
[10 * (2**30) + 1 for _ in range(10)],
|
|
40
|
+
{"v1": 16 * MiB, "v2": 16 * MiB, "hybrid": 8 * MiB},
|
|
41
|
+
id="10-100gb",
|
|
42
|
+
),
|
|
43
|
+
pytest.param(
|
|
44
|
+
[50 * (2**30) for _ in range(10)],
|
|
45
|
+
{"v1": 64 * MiB, "v2": 64 * MiB, "hybrid": 32 * MiB},
|
|
46
|
+
id="10-500gb",
|
|
47
|
+
),
|
|
48
|
+
pytest.param(
|
|
49
|
+
[100 * (2**30) for _ in range(10)],
|
|
50
|
+
{"v1": 128 * MiB, "v2": 128 * MiB, "hybrid": 32 * MiB},
|
|
51
|
+
id="10-1tb",
|
|
52
|
+
),
|
|
53
|
+
pytest.param(
|
|
54
|
+
[TiB for _ in range(10)],
|
|
55
|
+
128 * MiB,
|
|
56
|
+
id="10-10tb",
|
|
57
|
+
),
|
|
58
|
+
# --------------------------------------------------
|
|
59
|
+
# Weird size distributions
|
|
60
|
+
# --------------------------------------------------
|
|
61
|
+
pytest.param(
|
|
62
|
+
[100 * GiB] + [1 * KiB for _ in range(100)],
|
|
63
|
+
{"v1": 16 * MiB, "v2": 8 * MiB, "hybrid": 2 * MiB},
|
|
64
|
+
id="101-100gb-one-big-file",
|
|
65
|
+
),
|
|
66
|
+
pytest.param(
|
|
67
|
+
[100 * GiB] + [1 * KiB for _ in range(1000)],
|
|
68
|
+
{"v1": 16 * MiB, "v2": 8 * MiB, "hybrid": 1 * MiB},
|
|
69
|
+
id="1001-100gb-one-big-file",
|
|
70
|
+
),
|
|
71
|
+
pytest.param(
|
|
72
|
+
[100 * GiB] + [1 * KiB for _ in range(100000)],
|
|
73
|
+
{"v1": 16 * MiB, "v2": 16 * MiB, "hybrid": 128 * KiB},
|
|
74
|
+
id="100001-100gb-one-big-file",
|
|
75
|
+
),
|
|
76
|
+
# --------------------------------------------------
|
|
77
|
+
# Gamma distributions
|
|
78
|
+
# --------------------------------------------------
|
|
79
|
+
pytest.param(
|
|
80
|
+
[gammavariate(5, 100) * KiB for _ in range(100)],
|
|
81
|
+
{"v1": 128 * KiB, "v2": 256 * KiB, "hybrid": (32 * KiB, 64 * KiB)},
|
|
82
|
+
id="100-1kb-gamma-5-100",
|
|
83
|
+
),
|
|
84
|
+
pytest.param(
|
|
85
|
+
[gammavariate(5, 100) * KiB for _ in range(10000)],
|
|
86
|
+
{"v1": MiB, "v2": 2 * MiB, "hybrid": 128 * KiB},
|
|
87
|
+
id="10000-1kb-gamma-5-100",
|
|
88
|
+
),
|
|
89
|
+
pytest.param(
|
|
90
|
+
[gammavariate(5, 100) * MiB for _ in range(100)],
|
|
91
|
+
{"v1": 8 * MiB, "v2": 8 * MiB, "hybrid": 1 * MiB},
|
|
92
|
+
id="100-1mb-gamma-5-100",
|
|
93
|
+
),
|
|
94
|
+
pytest.param(
|
|
95
|
+
[gammavariate(5, 100) * GiB for _ in range(100)],
|
|
96
|
+
128 * MiB,
|
|
97
|
+
id="100-1gb-gamma-5-100",
|
|
98
|
+
),
|
|
99
|
+
],
|
|
100
|
+
)
|
|
101
|
+
def test_auto_piece_size(sizes: list[int], version: L["v1", "v2", "hybrid"], expected):
|
|
102
|
+
"""
|
|
103
|
+
Auto piece sizing with default params should give us the piece size we expect :)
|
|
104
|
+
"""
|
|
105
|
+
paths = [str(idx) for idx in range(len(sizes))]
|
|
106
|
+
piece_size, penalties = find_optimal_piece_size(
|
|
107
|
+
path=paths, sizes=sizes, version=version, return_penalties=True
|
|
108
|
+
)
|
|
109
|
+
if isinstance(expected, int):
|
|
110
|
+
assert piece_size == expected
|
|
111
|
+
else:
|
|
112
|
+
exp_val = expected[version]
|
|
113
|
+
if isinstance(exp_val, int):
|
|
114
|
+
assert piece_size == exp_val
|
|
115
|
+
else:
|
|
116
|
+
assert piece_size in exp_val
|
|
@@ -1,157 +0,0 @@
|
|
|
1
|
-
import warnings
|
|
2
|
-
from math import floor
|
|
3
|
-
from pathlib import Path
|
|
4
|
-
from typing import Literal as L
|
|
5
|
-
|
|
6
|
-
import bencode_rs
|
|
7
|
-
import libtorrent
|
|
8
|
-
from pydantic import TypeAdapter
|
|
9
|
-
from tqdm import tqdm
|
|
10
|
-
|
|
11
|
-
from sciop_cli.const import DEFAULT_TORRENT_CREATOR, EXCLUDE_FILES
|
|
12
|
-
from sciop_cli.exceptions import NoTrackersWarning
|
|
13
|
-
from sciop_cli.types import PieceSize
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def estimate_piece_size(
|
|
17
|
-
data_size: int, min_pieces: int = 2000, min_piece_size: int = 256 * (2**10)
|
|
18
|
-
) -> int:
|
|
19
|
-
"""
|
|
20
|
-
Estimate the best piece size given some total size (in bytes) of a torrent
|
|
21
|
-
|
|
22
|
-
Largest piece size that still gives us min_pieces.
|
|
23
|
-
Files smaller than the piece size will be packed together into archives,
|
|
24
|
-
so padding files are not much of an overhead concern here.
|
|
25
|
-
|
|
26
|
-
pieces = ceil(size / (multiplier * 16KiB))
|
|
27
|
-
pieces * (multiplier * 16KiB) = size
|
|
28
|
-
multiplier = floor(size / (pieces * 16KiB))
|
|
29
|
-
|
|
30
|
-
"""
|
|
31
|
-
block = 16 * (2**10)
|
|
32
|
-
multiplier = floor(data_size / (min_pieces * block))
|
|
33
|
-
piece_size = round(multiplier * block)
|
|
34
|
-
return max(piece_size, min_piece_size)
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
def create_torrent(
|
|
38
|
-
path: Path,
|
|
39
|
-
trackers: list[str] | None = None,
|
|
40
|
-
comment: str | None = None,
|
|
41
|
-
creator: str = DEFAULT_TORRENT_CREATOR,
|
|
42
|
-
webseeds: list[str] | None = None,
|
|
43
|
-
similar: list[str] | None = None,
|
|
44
|
-
version: L["v1", "v2", "hybrid"] = "hybrid",
|
|
45
|
-
bencode: bool = True,
|
|
46
|
-
piece_size: PieceSize = 512 * (2**10),
|
|
47
|
-
pbar: bool = False,
|
|
48
|
-
) -> dict | bytes:
|
|
49
|
-
"""
|
|
50
|
-
Create a hybrid v1/v2 torrent with libtorrent
|
|
51
|
-
|
|
52
|
-
Args:
|
|
53
|
-
path: File or directory to create a torrent for
|
|
54
|
-
trackers: list of trackers to add to the torrent.
|
|
55
|
-
Each torrent is added on a separate tier, in order,
|
|
56
|
-
so that by default clients announce to all trackers
|
|
57
|
-
comment: Additional comment string embedded in the torrent
|
|
58
|
-
creator: Annotation of what tool was used to create the torrent,
|
|
59
|
-
defaults to `sciop-cli-{version}`
|
|
60
|
-
webseeds: List of HTTP urls where the content can also be found.
|
|
61
|
-
If `path` is a directory, the files on the server
|
|
62
|
-
must match the directory structure of the torrent and their content exactly.
|
|
63
|
-
similar: Infohashes of torrents that have identical files to those in this torrent.
|
|
64
|
-
Clients can use this to deduplicate downloads.
|
|
65
|
-
version: Torrent version to create. Default ``"hybrid"`` creates v1/v2 compatible torrents.
|
|
66
|
-
v1 is *not recommended.*
|
|
67
|
-
bencode: If ``True`` ( default ) return the bencoded bytestring.
|
|
68
|
-
Otherwise return a python dictionary
|
|
69
|
-
piece_size: The size of data chunks to hash.
|
|
70
|
-
Choosing a piece size can be complicated, but ideally you want
|
|
71
|
-
to pick a piece size that yields 20-50k pieces, and less than <100k.
|
|
72
|
-
Smaller torrents can have fewer pieces, in that case match the piece size
|
|
73
|
-
to being slightly smaller than the median file size.
|
|
74
|
-
Once torrents start to have >200k pieces, clients suffer to efficiently
|
|
75
|
-
track which peers have what pieces, and also struggle to cache the data.
|
|
76
|
-
pbar: It ``True``, show a progress bar while hashing pieces.
|
|
77
|
-
|
|
78
|
-
Returns:
|
|
79
|
-
bytes: bencoded torrent ready for writing
|
|
80
|
-
dict: python-formatted torrent file dict
|
|
81
|
-
"""
|
|
82
|
-
path = Path(path)
|
|
83
|
-
fs = libtorrent.file_storage()
|
|
84
|
-
|
|
85
|
-
if path.is_dir():
|
|
86
|
-
# get paths and sort
|
|
87
|
-
paths = []
|
|
88
|
-
for _path in path.rglob("*"):
|
|
89
|
-
if _path.name not in EXCLUDE_FILES and _path.is_file():
|
|
90
|
-
# no absolute paths in the torrent plz
|
|
91
|
-
rel_path = _path.relative_to(path)
|
|
92
|
-
# add the parent again as the root
|
|
93
|
-
rel_path = Path(path.name) / rel_path
|
|
94
|
-
paths.append((str(rel_path), _path.stat().st_size))
|
|
95
|
-
paths = sorted(paths, key=lambda x: x[0])
|
|
96
|
-
for p, size in paths:
|
|
97
|
-
fs.add_file(p, size)
|
|
98
|
-
|
|
99
|
-
else:
|
|
100
|
-
fs.add_file(path.name, path.stat().st_size)
|
|
101
|
-
|
|
102
|
-
if fs.num_files() == 0:
|
|
103
|
-
raise
|
|
104
|
-
|
|
105
|
-
piece_size = TypeAdapter(PieceSize).validate_python(piece_size)
|
|
106
|
-
|
|
107
|
-
flags = 0
|
|
108
|
-
if version == "v1":
|
|
109
|
-
flags = libtorrent.create_torrent.v1_only
|
|
110
|
-
elif version == "v2":
|
|
111
|
-
flags = libtorrent.create_torrent.v2_only
|
|
112
|
-
|
|
113
|
-
torrent = libtorrent.create_torrent(fs, piece_size=piece_size, flags=flags)
|
|
114
|
-
|
|
115
|
-
if trackers:
|
|
116
|
-
for tier, tracker in enumerate(trackers):
|
|
117
|
-
torrent.add_tracker(tracker, tier)
|
|
118
|
-
else:
|
|
119
|
-
warnings.warn(
|
|
120
|
-
"No trackers passed when creating a torrent. "
|
|
121
|
-
"This torrent will likely not be able to be seeded efficiently. "
|
|
122
|
-
"Consider adding trackers, or use the default trackers "
|
|
123
|
-
"(with `sciop_cli.data.get_default_trackers()`)",
|
|
124
|
-
NoTrackersWarning,
|
|
125
|
-
stacklevel=2,
|
|
126
|
-
)
|
|
127
|
-
|
|
128
|
-
if webseeds:
|
|
129
|
-
for webseed in webseeds:
|
|
130
|
-
torrent.add_url_seed(webseed)
|
|
131
|
-
|
|
132
|
-
if similar:
|
|
133
|
-
for s in similar:
|
|
134
|
-
torrent.add_similar_torrent(s)
|
|
135
|
-
|
|
136
|
-
if comment:
|
|
137
|
-
torrent.set_comment(comment)
|
|
138
|
-
|
|
139
|
-
torrent.set_creator(creator)
|
|
140
|
-
|
|
141
|
-
_pbar = None
|
|
142
|
-
if pbar:
|
|
143
|
-
_pbar = tqdm(desc="hashing pieces...", total=torrent.num_pieces())
|
|
144
|
-
|
|
145
|
-
def _pbar_callback(piece_index: int) -> None:
|
|
146
|
-
_pbar.update()
|
|
147
|
-
|
|
148
|
-
libtorrent.set_piece_hashes(torrent, str(path.parent.resolve()), _pbar_callback)
|
|
149
|
-
_pbar.close()
|
|
150
|
-
else:
|
|
151
|
-
libtorrent.set_piece_hashes(torrent, str(path.parent.resolve()))
|
|
152
|
-
|
|
153
|
-
ret = torrent.generate()
|
|
154
|
-
if bencode:
|
|
155
|
-
return bencode_rs.bencode(ret)
|
|
156
|
-
else:
|
|
157
|
-
return ret
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|