sciop-cli 0.1.3__tar.gz → 0.1.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/PKG-INFO +2 -1
  2. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/pyproject.toml +5 -1
  3. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/cli/torrent.py +69 -5
  4. sciop_cli-0.1.4/src/sciop_cli/const.py +14 -0
  5. sciop_cli-0.1.4/src/sciop_cli/torrent.py +574 -0
  6. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/types.py +12 -0
  7. sciop_cli-0.1.4/tests/test_torrent.py +116 -0
  8. sciop_cli-0.1.3/src/sciop_cli/const.py +0 -4
  9. sciop_cli-0.1.3/src/sciop_cli/torrent.py +0 -157
  10. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/LICENSE +0 -0
  11. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/README.md +0 -0
  12. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/__init__.py +0 -0
  13. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/__main__.py +0 -0
  14. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/cli/__init__.py +0 -0
  15. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/cli/main.py +0 -0
  16. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/cli/manifest.py +0 -0
  17. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/data.py +0 -0
  18. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/exceptions.py +0 -0
  19. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/file.py +0 -0
  20. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/models/__init__.py +0 -0
  21. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/src/sciop_cli/schema.py +0 -0
  22. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/.DS_Store +0 -0
  23. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/__init__.py +0 -0
  24. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/conftest.py +0 -0
  25. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/data/basic/part_1/subdir/part_1 +0 -0
  26. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/data/basic/part_2/subdir/part_2 +0 -0
  27. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/data/basic_hybrid.torrent +0 -0
  28. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/data/basic_v2.torrent +0 -0
  29. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/test_cli/__init__.py +0 -0
  30. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/test_cli/test_torrent.py +0 -0
  31. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/test_data.py +0 -0
  32. {sciop_cli-0.1.3 → sciop_cli-0.1.4}/tests/test_temp.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: sciop-cli
3
- Version: 0.1.3
3
+ Version: 0.1.4
4
4
  Summary: A CLI for sciop
5
5
  Author-Email: cosmicexplorer <dmcC2@hypnicjerk.ai>, sneakers-the-rat <sneakers-the-rat@protonmail.com>
6
6
  License: EUPL-1.2
@@ -15,6 +15,7 @@ Requires-Dist: platformdirs>=4.3.7
15
15
  Requires-Dist: requests>=2.32.3
16
16
  Requires-Dist: tqdm>=4.67.1
17
17
  Requires-Dist: rich>=14.0.0
18
+ Requires-Dist: humanize>=4.12.2
18
19
  Provides-Extra: test
19
20
  Requires-Dist: pytest>=8.3.4; extra == "test"
20
21
  Provides-Extra: mypy
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "sciop-cli"
3
- version = "0.1.3"
3
+ version = "0.1.4"
4
4
  description = "A CLI for sciop"
5
5
  authors = [
6
6
  { name = "cosmicexplorer", email = "dmcC2@hypnicjerk.ai" },
@@ -17,6 +17,7 @@ dependencies = [
17
17
  "requests>=2.32.3",
18
18
  "tqdm>=4.67.1",
19
19
  "rich>=14.0.0",
20
+ "humanize>=4.12.2",
20
21
  ]
21
22
  requires-python = ">=3.11"
22
23
  readme = "README.md"
@@ -148,3 +149,6 @@ show_column_numbers = true
148
149
  show_error_code_links = true
149
150
  pretty = true
150
151
  color_output = true
152
+
153
+ [tool.pytest.ini_options]
154
+ markers = " v1: tests for v1 torrents\n v2: tests for v2 torrents\n hybrid: tests for hybrid torrents\n"
@@ -1,11 +1,23 @@
1
1
  from pathlib import Path
2
+ from typing import Literal as L
2
3
  from typing import cast
3
4
 
4
5
  import click
6
+ import humanize
7
+ from rich import print
8
+ from rich.console import Group
9
+ from rich.panel import Panel
10
+ from rich.table import Table
5
11
 
6
12
  from sciop_cli.const import DEFAULT_TORRENT_CREATOR
7
13
  from sciop_cli.data import get_default_trackers
8
- from sciop_cli.torrent import create_torrent
14
+ from sciop_cli.torrent import (
15
+ calculate_overhead,
16
+ calculate_total_pieces,
17
+ create_torrent,
18
+ find_optimal_piece_size,
19
+ iter_files,
20
+ )
9
21
  from sciop_cli.types import PieceSize
10
22
 
11
23
 
@@ -59,8 +71,9 @@ def pack() -> None:
59
71
  @click.option(
60
72
  "-s",
61
73
  "--piece-size",
62
- default=512 * (2**10),
63
- help="Piece size, in bytes",
74
+ default=None,
75
+ help="Piece size, in bytes. If not given, calculate piece size automatically."
76
+ "Use `sciop-cli torrent piece-size` to preview the ",
64
77
  show_default=True,
65
78
  )
66
79
  @click.option(
@@ -114,7 +127,7 @@ def create(
114
127
  path: Path,
115
128
  tracker: list[str] | tuple[str] | None = None,
116
129
  default_trackers: bool | None = None,
117
- piece_size: PieceSize = 512 * (2**10),
130
+ piece_size: PieceSize | None = None,
118
131
  comment: str | None = None,
119
132
  creator: str = DEFAULT_TORRENT_CREATOR,
120
133
  webseed: list[str] | None = None,
@@ -135,7 +148,14 @@ def create(
135
148
  """
136
149
  # recast tuple to list or none rather than tuple or empty tuple
137
150
  tracker = list(tracker) if tracker else None
151
+ version = "v2" if v2 else "hybrid"
152
+ version = cast(L["v2", "hybrid"], version)
153
+
154
+ if piece_size is None:
155
+ click.echo("No piece size specified, estimating optimal piece size")
156
+ piece_size = find_optimal_piece_size(path=Path(path).absolute(), version=version)
138
157
 
158
+ click.echo(f"Piece size estimated as {humanize.naturalsize(piece_size, binary=True)}")
139
159
  if not tracker and (default_trackers is None or default_trackers):
140
160
  click.echo(
141
161
  "No trackers specified, using default trackers from "
@@ -154,7 +174,7 @@ def create(
154
174
  creator=creator,
155
175
  webseeds=webseed,
156
176
  similar=similar,
157
- version="v2" if v2 else "hybrid",
177
+ version=version,
158
178
  pbar=progress,
159
179
  bencode=True,
160
180
  )
@@ -164,3 +184,47 @@ def create(
164
184
  f.write(result)
165
185
  else:
166
186
  click.echo(result)
187
+
188
+
189
+ @torrent.command("piece-size")
190
+ @click.option(
191
+ "-p",
192
+ "--path",
193
+ required=True,
194
+ help="Path to a directory or file to create .torrent from",
195
+ type=click.Path(exists=True),
196
+ )
197
+ def piece_size(path: Path) -> None:
198
+ """
199
+ Show the optimal piece sizes calculated for each torrent version
200
+ for a given file or directory.
201
+ """
202
+ path = Path(path)
203
+ paths = [path] if path.is_file() else list(iter_files(path))
204
+
205
+ sizes = [p.stat().st_size for p in paths]
206
+ versions = ("v1", "v2", "hybrid")
207
+ piece_sizes = {v: find_optimal_piece_size(path=paths, sizes=sizes, version=v) for v in versions}
208
+ n_pieces = {v: calculate_total_pieces(sizes, piece_sizes[v], v) for v in versions}
209
+ hybrid_overhead = sum(calculate_overhead(sizes, piece_sizes["hybrid"]))
210
+
211
+ summary = Table(show_header=False)
212
+ summary.add_column("", style="bold magenta")
213
+ summary.add_column("")
214
+ summary.add_row("N Files", humanize.number.intcomma(len(paths)))
215
+ summary.add_row("Total size", humanize.naturalsize(sum(sizes), binary=True))
216
+
217
+ pieces = Table(title="Piece sizes")
218
+ pieces.add_column("Version")
219
+ pieces.add_column("Piece Size")
220
+ pieces.add_column("N Pieces")
221
+ pieces.add_column("Padding Overhead")
222
+ for v in ("v1", "v2", "hybrid"):
223
+ row = [v, str(piece_sizes[v]), humanize.number.intcomma(n_pieces[v])]
224
+
225
+ if v == "hybrid":
226
+ row.append(humanize.naturalsize(hybrid_overhead, binary=True))
227
+ pieces.add_row(*row)
228
+
229
+ panel = Panel(Group(summary, pieces), title=str(path))
230
+ print(panel)
@@ -0,0 +1,14 @@
1
+ from importlib.metadata import version
2
+
3
+ from sciop_cli.types import Size
4
+
5
+ KiB = Size(2**10)
6
+ MiB = Size(2**20)
7
+ GiB = Size(2**30)
8
+ TiB = Size(2**40)
9
+
10
+ DEFAULT_TORRENT_CREATOR = f"sciop-cli {version('sciop-cli')}"
11
+ EXCLUDE_FILES = (".DS_Store", "Thumbs.db")
12
+ BLOCK_SIZE = 16 * KiB
13
+ PIECE_SIZES = tuple(BLOCK_SIZE * (2**exp) for exp in range(14))
14
+ """16KiB to 128MiB, powers of two * block size."""
@@ -0,0 +1,574 @@
1
+ import warnings
2
+ from dataclasses import dataclass
3
+ from math import ceil
4
+ from pathlib import Path
5
+ from statistics import median
6
+ from typing import Any, Generator, TypedDict, cast, overload
7
+ from typing import Literal as L
8
+
9
+ import bencode_rs
10
+ import libtorrent
11
+ from pydantic import TypeAdapter
12
+ from tqdm import tqdm
13
+
14
+ from sciop_cli.const import DEFAULT_TORRENT_CREATOR, EXCLUDE_FILES, PIECE_SIZES, GiB
15
+ from sciop_cli.exceptions import NoTrackersWarning
16
+ from sciop_cli.types import PieceSize
17
+
18
+
19
+ @dataclass
20
+ class PieceParams:
21
+ target_pieces: int = 5000
22
+ """Pick the piece size that gets us closest to this number of pieces"""
23
+ small_torrent_threshold: int = GiB
24
+ """
25
+ For torrents with total sizes below this threshold (in bytes),
26
+ use the small_target_pieces target instead
27
+ """
28
+ small_target_pieces: int = 250
29
+ """Target pieces when the total size is beneath the small torrent threshold"""
30
+ target_weight: float = 1
31
+ """Relative weight of the target_pieces value on piece size"""
32
+
33
+ @overload
34
+ @classmethod
35
+ def from_version(cls, version: L["v1"], **kwargs: Any) -> "V1PieceParams": ...
36
+
37
+ @overload
38
+ @classmethod
39
+ def from_version(cls, version: L["v2"], **kwargs: Any) -> "V2PieceParams": ...
40
+
41
+ @overload
42
+ @classmethod
43
+ def from_version(cls, version: L["hybrid"], **kwargs: Any) -> "HybridPieceParams": ...
44
+
45
+ @classmethod
46
+ def from_version(cls, version: L["v1", "v2", "hybrid"] | str, **kwargs: Any) -> "PieceParams":
47
+ """Create params for specific version"""
48
+ if version == "v1":
49
+ return V1PieceParams(**kwargs)
50
+ elif version == "v2":
51
+ return V2PieceParams(**kwargs)
52
+ elif version == "hybrid":
53
+ return HybridPieceParams(**kwargs)
54
+ else:
55
+ raise ValueError(f"Unknown torrent version: {version}")
56
+
57
+
58
+ @dataclass
59
+ class V1PieceParams(PieceParams):
60
+ overhead_weight: float = 1.0e-10
61
+ """
62
+ Relative weight of the v1 overhead induced by pad files.
63
+
64
+ Tiny tiny penalty, effectively only for breaking ties,
65
+ because v1 torrents don't really have a overhead per se
66
+
67
+ See :func:`.score_overhead` for details.
68
+ """
69
+
70
+
71
+ @dataclass
72
+ class V2PieceParams(PieceParams):
73
+ median_weight: float = 0.05
74
+ """
75
+ Relative weight of penalty for piece sizes that are larger than the median file size
76
+ """
77
+
78
+
79
+ @dataclass
80
+ class HybridPieceParams(PieceParams):
81
+ overhead_weight: float = 1.0
82
+ """
83
+ Relative weight of the v1 overhead induced by pad files.
84
+ This term is *not* normalized to force its range to 0-1 like the other penalties,
85
+ it is instead normalized to the total size of the torrent:
86
+ the absolute value of the size of the overhead matters as well as
87
+ the relative magnitude per each piece size.
88
+ """
89
+
90
+
91
+ class Penalty(TypedDict):
92
+ piece_size: int
93
+ type: str
94
+ value: float
95
+
96
+
97
+ @overload
98
+ def find_optimal_piece_size(path: Path | list[Path]) -> int: ...
99
+
100
+
101
+ @overload
102
+ def find_optimal_piece_size(
103
+ path: Path | list[Path], version: L["v1", "v2", "hybrid"] | str
104
+ ) -> int: ...
105
+
106
+
107
+ @overload
108
+ def find_optimal_piece_size(
109
+ path: Path | list[Path],
110
+ version: L["v1"],
111
+ sizes: list[int] | None,
112
+ params: V1PieceParams | None = None,
113
+ ) -> int: ...
114
+
115
+
116
+ @overload
117
+ def find_optimal_piece_size(
118
+ path: Path | list[Path],
119
+ version: L["v2"],
120
+ sizes: list[int] | None,
121
+ params: V2PieceParams | None = None,
122
+ ) -> int: ...
123
+
124
+
125
+ @overload
126
+ def find_optimal_piece_size(
127
+ path: Path | list[Path],
128
+ version: L["hybrid"],
129
+ sizes: list[int] | None,
130
+ params: HybridPieceParams | None,
131
+ ) -> int: ...
132
+
133
+
134
+ @overload
135
+ def find_optimal_piece_size(
136
+ path: Path | list[Path],
137
+ version: L["v1", "v2", "hybrid"] | str,
138
+ sizes: list[int] | None,
139
+ ) -> int: ...
140
+
141
+
142
+ @overload
143
+ def find_optimal_piece_size(
144
+ path: Path | list[Path],
145
+ version: L["v1", "v2", "hybrid"] | str,
146
+ sizes: list[int] | None,
147
+ params: V1PieceParams | V2PieceParams | HybridPieceParams | dict | None,
148
+ return_penalties: L[True],
149
+ ) -> tuple[int, dict[str, list[Penalty]]]: ...
150
+
151
+
152
+ def find_optimal_piece_size(
153
+ path: Path | str | list[Path] | list[str],
154
+ version: L["v1", "v2", "hybrid"] | str = "hybrid",
155
+ sizes: list[int] | None = None,
156
+ params: V1PieceParams | V2PieceParams | HybridPieceParams | dict | None = None,
157
+ return_penalties: bool = False,
158
+ ) -> int | tuple[int, dict[str, list[Penalty]]]:
159
+ """
160
+ Estimate an optimal piece size for the given file(s).
161
+
162
+ Each version has slightly different considerations for piece size
163
+ - v1: Piece size determined by total size, pick a piece size that makes a
164
+ reasonable number of pieces (i.e. 1000 - 50000-ish)
165
+ - v2: Piece size determined by total size and size distribution.
166
+ Pick a piece size that balances a reasonable number of pieces with
167
+ the piece size being roughly the size of the median file.
168
+ - hybrid: Piece size determined by total size and padding overhead.
169
+ Pick a piece size that balances a reasonable number of pieces
170
+ without creating an enormous amount of padding overhead.
171
+
172
+ See the docstrings on the *PieceParams models for more details.
173
+
174
+ Args:
175
+ path (Path): File or directory of files to be added to torrent
176
+ version ("v1", "v2", "hybrid"): version of torrent to be created.
177
+ params (V1PieceParams | V2PieceParams | HybridPieceParams | dict):
178
+ Parameterization of piece size optimization.
179
+ See model docstrings for more detail.
180
+ return_penalties (bool): If ``True``, return the calculated penalties
181
+
182
+ Returns:
183
+
184
+ """
185
+ version = cast(L["v1", "v2", "hybrid"], version)
186
+ if isinstance(path, list):
187
+ paths = [Path(p) for p in path] if isinstance(path[0], str) else path
188
+ else:
189
+ path = Path(path)
190
+ paths = [path] if path.is_file() else list(iter_files(path))
191
+
192
+ paths = cast(list[Path], paths)
193
+
194
+ if sizes is None:
195
+ sizes = [p.stat().st_size for p in paths]
196
+
197
+ assert len(sizes) == len(
198
+ paths
199
+ ), "Length of provided sizes does not match length of detected paths"
200
+
201
+ if isinstance(params, dict):
202
+ params = PieceParams.from_version(version=version, **params)
203
+ elif params is None:
204
+ params = PieceParams.from_version(version=version)
205
+
206
+ if version == "v1":
207
+ params = cast(V1PieceParams | None, params)
208
+ return find_optimal_piece_size_v1(
209
+ paths=paths, sizes=sizes, params=params, return_penalties=return_penalties
210
+ )
211
+ elif version == "v2":
212
+ params = cast(V2PieceParams | None, params)
213
+ return find_optimal_piece_size_v2(
214
+ paths=paths, sizes=sizes, params=params, return_penalties=return_penalties
215
+ )
216
+ elif version == "hybrid":
217
+ params = cast(HybridPieceParams | None, params)
218
+ return find_optimal_piece_size_hybrid(
219
+ paths=paths, sizes=sizes, params=params, return_penalties=return_penalties
220
+ )
221
+ else:
222
+ raise ValueError(f"Unkown torrent version {version}")
223
+
224
+
225
+ def find_optimal_piece_size_v1(
226
+ paths: list[Path],
227
+ sizes: list[int],
228
+ params: V1PieceParams | None = None,
229
+ return_penalties: bool = False,
230
+ ) -> int | tuple[int, dict[str, list[Penalty]]]:
231
+ """
232
+ Find a piece size that produces a reasonable number of pieces,
233
+ using overhead as a tiebreaker (e.g. for very small torrents where most piece sizes
234
+ would yield a single-piece torrent.)
235
+ """
236
+ if params is None:
237
+ params = V1PieceParams()
238
+ target_pieces = _normalize_penalties(
239
+ _penalty_target_pieces(paths=paths, sizes=sizes, version="v1", params=params)
240
+ )
241
+ # mildly penalize overhead in the case of very small torrents
242
+ overhead = _penalty_overhead(paths=paths, sizes=sizes)
243
+
244
+ piece_size = _min_penalty(
245
+ penalties=[target_pieces, overhead],
246
+ weights=[params.target_weight, params.overhead_weight],
247
+ )
248
+ if return_penalties:
249
+ return piece_size, {"target": target_pieces, "overhead": overhead}
250
+ else:
251
+ return piece_size
252
+
253
+
254
+ def find_optimal_piece_size_v2(
255
+ paths: list[Path],
256
+ sizes: list[int],
257
+ params: V2PieceParams | None = None,
258
+ return_penalties: bool = False,
259
+ ) -> int | tuple[int, dict[str, list[Penalty]]]:
260
+ """
261
+ Find a piece size that balances having a reasonable number of pieces
262
+ with the hashing overhead of having a piece size that is significantly larger
263
+ than the median filesize in a torrent
264
+ """
265
+ if params is None:
266
+ params = V2PieceParams()
267
+
268
+ target_pieces = _normalize_penalties(
269
+ _penalty_target_pieces(paths=paths, sizes=sizes, version="v2", params=params)
270
+ )
271
+ med_size = _normalize_penalties(_penalty_median_size(paths=paths, sizes=sizes))
272
+ piece_size = _min_penalty(
273
+ penalties=[target_pieces, med_size],
274
+ weights=[params.target_weight, params.median_weight],
275
+ )
276
+ if return_penalties:
277
+ return piece_size, {"target": target_pieces, "median": med_size}
278
+ else:
279
+ return piece_size
280
+
281
+
282
+ def find_optimal_piece_size_hybrid(
283
+ paths: list[Path],
284
+ sizes: list[int],
285
+ params: HybridPieceParams | None = None,
286
+ return_penalties: bool = False,
287
+ ) -> int | tuple[int, dict[str, list[Penalty]]]:
288
+ """
289
+ Find a piece size that minimizes padfile overhead and produces a reasonable number of pieces.
290
+ """
291
+ if params is None:
292
+ params = HybridPieceParams()
293
+
294
+ # don't normalize overhead - it's normalized w.r.t. to the total size
295
+ # (e.g. a 100% overhead would be 1)
296
+ overhead = _penalty_overhead(paths=paths, sizes=sizes)
297
+ target_pieces = _normalize_penalties(
298
+ _penalty_target_pieces(paths=paths, sizes=sizes, version="hybrid", params=params)
299
+ )
300
+
301
+ piece_size = _min_penalty(
302
+ penalties=[overhead, target_pieces],
303
+ weights=[params.overhead_weight, params.target_weight],
304
+ )
305
+ if return_penalties:
306
+ return piece_size, {"target": target_pieces, "overhead": overhead}
307
+ else:
308
+ return piece_size
309
+
310
+
311
+ def _min_penalty(penalties: list[list[Penalty]], weights: list[float]) -> int:
312
+ """
313
+ Given a list of piece size penalties and weights,
314
+ return the piece size that minimizes the sum of the weighted penalties.
315
+
316
+ Args:
317
+ penalties (list[list[Penalty]]): List of penalty terms per piece size
318
+ weights (list[float]): list of weights, length == to number of penalty lists.
319
+ """
320
+ assert len(penalties) == len(weights)
321
+ # collect penalties by piece size
322
+ piece_penalties = {}
323
+ for pens in zip(*penalties):
324
+ assert all(
325
+ [p["piece_size"] == pens[0]["piece_size"]] for p in pens
326
+ ), "piece sizes must be sorted before scoring"
327
+ piece_penalties[pens[0]["piece_size"]] = sum(
328
+ [p["value"] * weight for p, weight in zip(pens, weights)]
329
+ )
330
+
331
+ # invert mapping to select piece size by penalty
332
+ penalty_pieces = {v: k for k, v in piece_penalties.items()}
333
+ return penalty_pieces[min(penalty_pieces.keys())]
334
+
335
+
336
+ def calculate_overhead(sizes: list[int], piece_size: int) -> list[float]:
337
+ """
338
+ Given a list of file sizes and a piece size, calculate expected v1 padfile overhead
339
+ """
340
+ return [piece_size - (size % piece_size) if size != piece_size else 0 for size in sizes]
341
+
342
+
343
+ def _penalty_overhead(paths: list[Path], sizes: list[int]) -> list[Penalty]:
344
+ """
345
+ Total v1 padfile overhead for a given piece size,
346
+ Normalized to the total size of the given files.
347
+ """
348
+ penalties = []
349
+ total_size = sum(sizes)
350
+ for piece_size in PIECE_SIZES:
351
+ overheads = calculate_overhead(sizes, piece_size)
352
+ penalties.append(
353
+ Penalty(
354
+ piece_size=piece_size,
355
+ type="overhead",
356
+ value=sum(overheads) / total_size,
357
+ )
358
+ )
359
+ return penalties
360
+
361
+
362
+ def calculate_total_pieces(
363
+ sizes: list[int], piece_size: int, version: L["v1", "v2", "hybrid"] | str
364
+ ) -> int:
365
+ if version == "v1":
366
+ total_size = sum(sizes)
367
+ total_pieces = ceil(total_size / piece_size)
368
+ elif version in ("v2", "hybrid"):
369
+ total_pieces = sum([ceil(size / piece_size) for size in sizes])
370
+ else:
371
+ raise ValueError(f"Unkown torrent version {version}")
372
+ return total_pieces
373
+
374
+
375
+ def _penalty_target_pieces(
376
+ paths: list[Path], sizes: list[int], version: L["v1", "v2", "hybrid"], params: PieceParams
377
+ ) -> list[Penalty]:
378
+ """
379
+ Absolute value of difference in actual pieces vs. target pieces, with specializations.
380
+
381
+ - If the largest possible piece size produces more pieces than the target number of pieces,
382
+ then we squash the penalty range so that we preserve the relative penalization of
383
+ the second to large pieces - i.e. we put stronger weight on making larger pieces,
384
+ assuming the values will be normalized in the next step.
385
+ - For piece sizes that make *fewer* pieces than the target piece number,
386
+ exponentially penalize piece sizes that make us make a very small number of pieces.
387
+ A torrent with a single piece is not very useful.
388
+ """
389
+ total_size = sum(sizes)
390
+ if total_size < params.small_torrent_threshold:
391
+ target = params.small_target_pieces
392
+ else:
393
+ target = params.target_pieces
394
+
395
+ penalties = []
396
+ for piece_size in PIECE_SIZES:
397
+ total_pieces = calculate_total_pieces(sizes, piece_size, version)
398
+ penalty = float(abs(total_pieces - target))
399
+
400
+ # if we still can't make the target with the largest piece size,
401
+ # squash the range because making half as many pieces is much worse than usual
402
+ if (min_pieces := ceil(total_size / max(PIECE_SIZES))) > target:
403
+ penalty = penalty ** min(((target * 2) / min_pieces), 1)
404
+ # penalize going under more than going over
405
+ elif total_pieces < target:
406
+ # piece sizes that want to make us make one piece...
407
+ exponent = ((target - total_pieces) / target) * 2
408
+ penalty *= max(penalty * exponent, 1)
409
+
410
+ penalties.append(
411
+ Penalty(
412
+ piece_size=piece_size,
413
+ type="target",
414
+ value=penalty,
415
+ )
416
+ )
417
+ return penalties
418
+
419
+
420
+ def _penalty_median_size(paths: list[Path], sizes: list[int]) -> list[Penalty]:
421
+ median_size = median(sizes)
422
+
423
+ return [
424
+ Penalty(piece_size=piece_size, type="median", value=max(piece_size - median_size, 0) ** 2)
425
+ for piece_size in PIECE_SIZES
426
+ ]
427
+
428
+
429
+ def _normalize_penalties(penalties: list[Penalty]) -> list[Penalty]:
430
+ """Normalize the values of a single kind of penalty to be between 0 and 1"""
431
+ penalties = sorted(penalties, key=lambda p: p["piece_size"])
432
+ max_pen = max([p["value"] for p in penalties])
433
+ min_pen = min([p["value"] for p in penalties])
434
+ pen_range = max_pen - min_pen
435
+ for i in range(len(penalties)):
436
+ if pen_range == 0:
437
+ # all are equal
438
+ penalties[i]["value"] = 0
439
+ else:
440
+ penalties[i]["value"] = (penalties[i]["value"] - min_pen) / (max_pen - min_pen)
441
+ return penalties
442
+
443
+
444
+ def create_torrent(
445
+ path: Path,
446
+ trackers: list[str] | None = None,
447
+ comment: str | None = None,
448
+ creator: str = DEFAULT_TORRENT_CREATOR,
449
+ webseeds: list[str] | None = None,
450
+ similar: list[str] | None = None,
451
+ version: L["v1", "v2", "hybrid"] = "hybrid",
452
+ bencode: bool = True,
453
+ piece_size: PieceSize = 512 * (2**10),
454
+ pbar: bool = False,
455
+ ) -> dict | bytes:
456
+ """
457
+ Create a hybrid v1/v2 torrent with libtorrent
458
+
459
+ Args:
460
+ path: File or directory to create a torrent for
461
+ trackers: list of trackers to add to the torrent.
462
+ Each torrent is added on a separate tier, in order,
463
+ so that by default clients announce to all trackers
464
+ comment: Additional comment string embedded in the torrent
465
+ creator: Annotation of what tool was used to create the torrent,
466
+ defaults to `sciop-cli-{version}`
467
+ webseeds: List of HTTP urls where the content can also be found.
468
+ If `path` is a directory, the files on the server
469
+ must match the directory structure of the torrent and their content exactly.
470
+ similar: Infohashes of torrents that have identical files to those in this torrent.
471
+ Clients can use this to deduplicate downloads.
472
+ version: Torrent version to create. Default ``"hybrid"`` creates v1/v2 compatible torrents.
473
+ v1 is *not recommended.*
474
+ bencode: If ``True`` ( default ) return the bencoded bytestring.
475
+ Otherwise return a python dictionary
476
+ piece_size: The size of data chunks to hash.
477
+ Choosing a piece size can be complicated, but ideally you want
478
+ to pick a piece size that yields 20-50k pieces, and less than <100k.
479
+ Smaller torrents can have fewer pieces, in that case match the piece size
480
+ to being slightly smaller than the median file size.
481
+ Once torrents start to have >200k pieces, clients suffer to efficiently
482
+ track which peers have what pieces, and also struggle to cache the data.
483
+ pbar: It ``True``, show a progress bar while hashing pieces.
484
+
485
+ Returns:
486
+ bytes: bencoded torrent ready for writing
487
+ dict: python-formatted torrent file dict
488
+ """
489
+ path = Path(path)
490
+ fs = libtorrent.file_storage()
491
+
492
+ if path.is_dir():
493
+ # get paths and sort
494
+ paths = []
495
+ for _path in iter_files(path):
496
+ # no absolute paths in the torrent plz
497
+ rel_path = _path.relative_to(path)
498
+ # add the parent again as the root
499
+ rel_path = Path(path.name) / rel_path
500
+ paths.append((str(rel_path), _path.stat().st_size))
501
+
502
+ paths = sorted(paths, key=lambda x: x[0])
503
+ for p, size in paths:
504
+ fs.add_file(p, size)
505
+
506
+ else:
507
+ fs.add_file(path.name, path.stat().st_size)
508
+
509
+ if fs.num_files() == 0:
510
+ raise
511
+
512
+ piece_size = TypeAdapter(PieceSize).validate_python(piece_size)
513
+
514
+ flags = 0
515
+ if version == "v1":
516
+ flags = libtorrent.create_torrent.v1_only
517
+ elif version == "v2":
518
+ flags = libtorrent.create_torrent.v2_only
519
+
520
+ torrent = libtorrent.create_torrent(fs, piece_size=piece_size, flags=flags)
521
+
522
+ if trackers:
523
+ for tier, tracker in enumerate(trackers):
524
+ torrent.add_tracker(tracker, tier)
525
+ else:
526
+ warnings.warn(
527
+ "No trackers passed when creating a torrent. "
528
+ "This torrent will likely not be able to be seeded efficiently. "
529
+ "Consider adding trackers, or use the default trackers "
530
+ "(with `sciop_cli.data.get_default_trackers()`)",
531
+ NoTrackersWarning,
532
+ stacklevel=2,
533
+ )
534
+
535
+ if webseeds:
536
+ for webseed in webseeds:
537
+ torrent.add_url_seed(webseed)
538
+
539
+ if similar:
540
+ for s in similar:
541
+ torrent.add_similar_torrent(s)
542
+
543
+ if comment:
544
+ torrent.set_comment(comment)
545
+
546
+ torrent.set_creator(creator)
547
+
548
+ _pbar = None
549
+ if pbar:
550
+ _pbar = tqdm(desc="hashing pieces...", total=torrent.num_pieces())
551
+
552
+ def _pbar_callback(piece_index: int) -> None:
553
+ _pbar.update()
554
+
555
+ libtorrent.set_piece_hashes(torrent, str(path.parent.resolve()), _pbar_callback)
556
+ _pbar.close()
557
+ else:
558
+ libtorrent.set_piece_hashes(torrent, str(path.parent.resolve()))
559
+
560
+ ret = torrent.generate()
561
+ if bencode:
562
+ return bencode_rs.bencode(ret)
563
+ else:
564
+ return ret
565
+
566
+
567
+ def iter_files(path: Path) -> Generator[Path, None, None]:
568
+ """
569
+ Recursively iterate through files, excluding system files
570
+ """
571
+ for p in path.rglob("*"):
572
+ if p.name in EXCLUDE_FILES or p.is_dir():
573
+ continue
574
+ yield p
@@ -1,5 +1,6 @@
1
1
  from typing import Annotated
2
2
 
3
+ import humanize
3
4
  from pydantic import AfterValidator
4
5
 
5
6
 
@@ -15,3 +16,14 @@ def _validate_size(size: int) -> int:
15
16
 
16
17
 
17
18
  PieceSize = Annotated[int, AfterValidator(_validate_size)]
19
+
20
+
21
+ class Size(int):
22
+ def __str__(self) -> str:
23
+ return humanize.naturalsize(int(self), binary=True)
24
+
25
+ def __rmul__(self, factor: int) -> "Size":
26
+ return Size(int(self) * factor)
27
+
28
+ def __mul__(self, other: int) -> "Size":
29
+ return Size(int(self) * other)
@@ -0,0 +1,116 @@
1
+ from random import gammavariate
2
+ from typing import Literal as L
3
+
4
+ import pytest
5
+
6
+ from sciop_cli.const import GiB, KiB, MiB, TiB
7
+ from sciop_cli.torrent import find_optimal_piece_size
8
+
9
+
10
+ @pytest.mark.parametrize(
11
+ "version",
12
+ (
13
+ pytest.param("v1", marks=pytest.mark.v1),
14
+ pytest.param("v2", marks=pytest.mark.v2),
15
+ pytest.param("hybrid", marks=pytest.mark.hybrid),
16
+ ),
17
+ )
18
+ @pytest.mark.parametrize(
19
+ "sizes,expected",
20
+ [
21
+ pytest.param([16 * KiB], 16 * KiB, id="single-block"),
22
+ pytest.param([KiB for _ in range(10)], 16 * KiB, id="10-10kb"),
23
+ pytest.param(
24
+ [MiB for _ in range(10)],
25
+ {"v1": 32 * KiB, "v2": 32 * KiB, "hybrid": 16 * KiB},
26
+ id="10-10mb",
27
+ ),
28
+ pytest.param(
29
+ [10 * MiB + 1 for _ in range(10)],
30
+ {"v1": 256 * KiB, "v2": 256 * KiB, "hybrid": 128 * KiB},
31
+ id="10-100mb",
32
+ ),
33
+ pytest.param(
34
+ [(100 * MiB) + 1 for _ in range(10)],
35
+ {"v1": 4 * MiB, "v2": 4 * MiB, "hybrid": 1 * MiB},
36
+ id="10-1gb",
37
+ ),
38
+ pytest.param(
39
+ [10 * (2**30) + 1 for _ in range(10)],
40
+ {"v1": 16 * MiB, "v2": 16 * MiB, "hybrid": 8 * MiB},
41
+ id="10-100gb",
42
+ ),
43
+ pytest.param(
44
+ [50 * (2**30) for _ in range(10)],
45
+ {"v1": 64 * MiB, "v2": 64 * MiB, "hybrid": 32 * MiB},
46
+ id="10-500gb",
47
+ ),
48
+ pytest.param(
49
+ [100 * (2**30) for _ in range(10)],
50
+ {"v1": 128 * MiB, "v2": 128 * MiB, "hybrid": 32 * MiB},
51
+ id="10-1tb",
52
+ ),
53
+ pytest.param(
54
+ [TiB for _ in range(10)],
55
+ 128 * MiB,
56
+ id="10-10tb",
57
+ ),
58
+ # --------------------------------------------------
59
+ # Weird size distributions
60
+ # --------------------------------------------------
61
+ pytest.param(
62
+ [100 * GiB] + [1 * KiB for _ in range(100)],
63
+ {"v1": 16 * MiB, "v2": 8 * MiB, "hybrid": 2 * MiB},
64
+ id="101-100gb-one-big-file",
65
+ ),
66
+ pytest.param(
67
+ [100 * GiB] + [1 * KiB for _ in range(1000)],
68
+ {"v1": 16 * MiB, "v2": 8 * MiB, "hybrid": 1 * MiB},
69
+ id="1001-100gb-one-big-file",
70
+ ),
71
+ pytest.param(
72
+ [100 * GiB] + [1 * KiB for _ in range(100000)],
73
+ {"v1": 16 * MiB, "v2": 16 * MiB, "hybrid": 128 * KiB},
74
+ id="100001-100gb-one-big-file",
75
+ ),
76
+ # --------------------------------------------------
77
+ # Gamma distributions
78
+ # --------------------------------------------------
79
+ pytest.param(
80
+ [gammavariate(5, 100) * KiB for _ in range(100)],
81
+ {"v1": 128 * KiB, "v2": 256 * KiB, "hybrid": (32 * KiB, 64 * KiB)},
82
+ id="100-1kb-gamma-5-100",
83
+ ),
84
+ pytest.param(
85
+ [gammavariate(5, 100) * KiB for _ in range(10000)],
86
+ {"v1": MiB, "v2": 2 * MiB, "hybrid": 128 * KiB},
87
+ id="10000-1kb-gamma-5-100",
88
+ ),
89
+ pytest.param(
90
+ [gammavariate(5, 100) * MiB for _ in range(100)],
91
+ {"v1": 8 * MiB, "v2": 8 * MiB, "hybrid": 1 * MiB},
92
+ id="100-1mb-gamma-5-100",
93
+ ),
94
+ pytest.param(
95
+ [gammavariate(5, 100) * GiB for _ in range(100)],
96
+ 128 * MiB,
97
+ id="100-1gb-gamma-5-100",
98
+ ),
99
+ ],
100
+ )
101
+ def test_auto_piece_size(sizes: list[int], version: L["v1", "v2", "hybrid"], expected):
102
+ """
103
+ Auto piece sizing with default params should give us the piece size we expect :)
104
+ """
105
+ paths = [str(idx) for idx in range(len(sizes))]
106
+ piece_size, penalties = find_optimal_piece_size(
107
+ path=paths, sizes=sizes, version=version, return_penalties=True
108
+ )
109
+ if isinstance(expected, int):
110
+ assert piece_size == expected
111
+ else:
112
+ exp_val = expected[version]
113
+ if isinstance(exp_val, int):
114
+ assert piece_size == exp_val
115
+ else:
116
+ assert piece_size in exp_val
@@ -1,4 +0,0 @@
1
- from importlib.metadata import version
2
-
3
- DEFAULT_TORRENT_CREATOR = f"sciop-cli {version('sciop-cli')}"
4
- EXCLUDE_FILES = (".DS_Store", "Thumbs.db")
@@ -1,157 +0,0 @@
1
- import warnings
2
- from math import floor
3
- from pathlib import Path
4
- from typing import Literal as L
5
-
6
- import bencode_rs
7
- import libtorrent
8
- from pydantic import TypeAdapter
9
- from tqdm import tqdm
10
-
11
- from sciop_cli.const import DEFAULT_TORRENT_CREATOR, EXCLUDE_FILES
12
- from sciop_cli.exceptions import NoTrackersWarning
13
- from sciop_cli.types import PieceSize
14
-
15
-
16
- def estimate_piece_size(
17
- data_size: int, min_pieces: int = 2000, min_piece_size: int = 256 * (2**10)
18
- ) -> int:
19
- """
20
- Estimate the best piece size given some total size (in bytes) of a torrent
21
-
22
- Largest piece size that still gives us min_pieces.
23
- Files smaller than the piece size will be packed together into archives,
24
- so padding files are not much of an overhead concern here.
25
-
26
- pieces = ceil(size / (multiplier * 16KiB))
27
- pieces * (multiplier * 16KiB) = size
28
- multiplier = floor(size / (pieces * 16KiB))
29
-
30
- """
31
- block = 16 * (2**10)
32
- multiplier = floor(data_size / (min_pieces * block))
33
- piece_size = round(multiplier * block)
34
- return max(piece_size, min_piece_size)
35
-
36
-
37
- def create_torrent(
38
- path: Path,
39
- trackers: list[str] | None = None,
40
- comment: str | None = None,
41
- creator: str = DEFAULT_TORRENT_CREATOR,
42
- webseeds: list[str] | None = None,
43
- similar: list[str] | None = None,
44
- version: L["v1", "v2", "hybrid"] = "hybrid",
45
- bencode: bool = True,
46
- piece_size: PieceSize = 512 * (2**10),
47
- pbar: bool = False,
48
- ) -> dict | bytes:
49
- """
50
- Create a hybrid v1/v2 torrent with libtorrent
51
-
52
- Args:
53
- path: File or directory to create a torrent for
54
- trackers: list of trackers to add to the torrent.
55
- Each torrent is added on a separate tier, in order,
56
- so that by default clients announce to all trackers
57
- comment: Additional comment string embedded in the torrent
58
- creator: Annotation of what tool was used to create the torrent,
59
- defaults to `sciop-cli-{version}`
60
- webseeds: List of HTTP urls where the content can also be found.
61
- If `path` is a directory, the files on the server
62
- must match the directory structure of the torrent and their content exactly.
63
- similar: Infohashes of torrents that have identical files to those in this torrent.
64
- Clients can use this to deduplicate downloads.
65
- version: Torrent version to create. Default ``"hybrid"`` creates v1/v2 compatible torrents.
66
- v1 is *not recommended.*
67
- bencode: If ``True`` ( default ) return the bencoded bytestring.
68
- Otherwise return a python dictionary
69
- piece_size: The size of data chunks to hash.
70
- Choosing a piece size can be complicated, but ideally you want
71
- to pick a piece size that yields 20-50k pieces, and less than <100k.
72
- Smaller torrents can have fewer pieces, in that case match the piece size
73
- to being slightly smaller than the median file size.
74
- Once torrents start to have >200k pieces, clients suffer to efficiently
75
- track which peers have what pieces, and also struggle to cache the data.
76
- pbar: It ``True``, show a progress bar while hashing pieces.
77
-
78
- Returns:
79
- bytes: bencoded torrent ready for writing
80
- dict: python-formatted torrent file dict
81
- """
82
- path = Path(path)
83
- fs = libtorrent.file_storage()
84
-
85
- if path.is_dir():
86
- # get paths and sort
87
- paths = []
88
- for _path in path.rglob("*"):
89
- if _path.name not in EXCLUDE_FILES and _path.is_file():
90
- # no absolute paths in the torrent plz
91
- rel_path = _path.relative_to(path)
92
- # add the parent again as the root
93
- rel_path = Path(path.name) / rel_path
94
- paths.append((str(rel_path), _path.stat().st_size))
95
- paths = sorted(paths, key=lambda x: x[0])
96
- for p, size in paths:
97
- fs.add_file(p, size)
98
-
99
- else:
100
- fs.add_file(path.name, path.stat().st_size)
101
-
102
- if fs.num_files() == 0:
103
- raise
104
-
105
- piece_size = TypeAdapter(PieceSize).validate_python(piece_size)
106
-
107
- flags = 0
108
- if version == "v1":
109
- flags = libtorrent.create_torrent.v1_only
110
- elif version == "v2":
111
- flags = libtorrent.create_torrent.v2_only
112
-
113
- torrent = libtorrent.create_torrent(fs, piece_size=piece_size, flags=flags)
114
-
115
- if trackers:
116
- for tier, tracker in enumerate(trackers):
117
- torrent.add_tracker(tracker, tier)
118
- else:
119
- warnings.warn(
120
- "No trackers passed when creating a torrent. "
121
- "This torrent will likely not be able to be seeded efficiently. "
122
- "Consider adding trackers, or use the default trackers "
123
- "(with `sciop_cli.data.get_default_trackers()`)",
124
- NoTrackersWarning,
125
- stacklevel=2,
126
- )
127
-
128
- if webseeds:
129
- for webseed in webseeds:
130
- torrent.add_url_seed(webseed)
131
-
132
- if similar:
133
- for s in similar:
134
- torrent.add_similar_torrent(s)
135
-
136
- if comment:
137
- torrent.set_comment(comment)
138
-
139
- torrent.set_creator(creator)
140
-
141
- _pbar = None
142
- if pbar:
143
- _pbar = tqdm(desc="hashing pieces...", total=torrent.num_pieces())
144
-
145
- def _pbar_callback(piece_index: int) -> None:
146
- _pbar.update()
147
-
148
- libtorrent.set_piece_hashes(torrent, str(path.parent.resolve()), _pbar_callback)
149
- _pbar.close()
150
- else:
151
- libtorrent.set_piece_hashes(torrent, str(path.parent.resolve()))
152
-
153
- ret = torrent.generate()
154
- if bencode:
155
- return bencode_rs.bencode(ret)
156
- else:
157
- return ret
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes