sciop-cli 0.1.5__tar.gz → 0.1.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/PKG-INFO +1 -1
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/pyproject.toml +1 -1
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/api.py +13 -5
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/torrent.py +24 -10
- sciop_cli-0.1.6/tests/data/distributions/nbu_arwen_ver01.json +1 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/test_torrent.py +22 -11
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/LICENSE +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/README.md +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/__init__.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/__main__.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/cli/__init__.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/cli/api.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/cli/main.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/cli/manifest.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/cli/torrent.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/config.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/const.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/data.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/exceptions.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/models/__init__.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/models/sciop.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/schema.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/types.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/__init__.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/conftest.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/data/basic/part_1/subdir/part_1 +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/data/basic/part_2/subdir/part_2 +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/data/basic_hybrid.torrent +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/data/basic_v2.torrent +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/fixtures/__init__.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/fixtures/base.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/fixtures/fabricators.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/test_api.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/test_cli/__init__.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/test_cli/test_api.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/test_cli/test_torrent.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/test_config.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/test_data.py +0 -0
- {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/test_temp.py +0 -0
|
@@ -46,6 +46,7 @@ def api_call(
|
|
|
46
46
|
instance_url: str | None = None,
|
|
47
47
|
api_prefix: Literal["/api/v1", False] = "/api/v1",
|
|
48
48
|
autorefresh_token: bool = True,
|
|
49
|
+
timeout: float | None = None,
|
|
49
50
|
do_raise: bool = True,
|
|
50
51
|
) -> requests.Response:
|
|
51
52
|
"""
|
|
@@ -70,6 +71,7 @@ def api_call(
|
|
|
70
71
|
autorefresh_token (bool): If `True` (default), try and get a token if we have credentials.
|
|
71
72
|
Made `False` e.g. to avoid infinite recursion while logging in,
|
|
72
73
|
or if it is otherwise desirable to use only the preconfigured token.
|
|
74
|
+
timeout (float): Explicit request timeout. If `None`, use config.request_timeout
|
|
73
75
|
do_raise (bool): If `True` (default), raise on status codes >=400
|
|
74
76
|
"""
|
|
75
77
|
config = get_config()
|
|
@@ -84,6 +86,7 @@ def api_call(
|
|
|
84
86
|
instance_url=instance_url,
|
|
85
87
|
api_prefix=api_prefix,
|
|
86
88
|
autorefresh_token=autorefresh_token,
|
|
89
|
+
timeout=timeout,
|
|
87
90
|
)
|
|
88
91
|
|
|
89
92
|
res = requests.request(**request_kwargs)
|
|
@@ -109,6 +112,7 @@ def _api_call_kwargs(
|
|
|
109
112
|
instance_url: str | None = None,
|
|
110
113
|
api_prefix: Literal["/api/v1", False] = "/api/v1",
|
|
111
114
|
autorefresh_token: bool = True,
|
|
115
|
+
timeout: float | None = None,
|
|
112
116
|
) -> dict:
|
|
113
117
|
default = default_headers(
|
|
114
118
|
instance_url=instance_url, headers=headers, autorefresh_token=autorefresh_token
|
|
@@ -126,7 +130,7 @@ def _api_call_kwargs(
|
|
|
126
130
|
"method": method,
|
|
127
131
|
"url": url,
|
|
128
132
|
"headers": merged_headers,
|
|
129
|
-
"timeout": config.request_timeout,
|
|
133
|
+
"timeout": timeout if timeout else config.request_timeout,
|
|
130
134
|
"files": files,
|
|
131
135
|
}
|
|
132
136
|
if data:
|
|
@@ -217,9 +221,9 @@ def claim_next(dataset: str) -> DatasetClaim | None:
|
|
|
217
221
|
return DatasetClaim(**claim)
|
|
218
222
|
|
|
219
223
|
|
|
220
|
-
def upload_torrent(path: Path) -> TorrentFile:
|
|
224
|
+
def upload_torrent(path: Path, timeout: float | None = 60.0) -> TorrentFile:
|
|
221
225
|
with open(path, "rb") as f:
|
|
222
|
-
response = api_call("torrents", files={"file": f})
|
|
226
|
+
response = api_call("torrents", files={"file": f}, timeout=timeout)
|
|
223
227
|
tf = cast(TorrentFile, response.json())
|
|
224
228
|
return TorrentFile(**tf)
|
|
225
229
|
|
|
@@ -231,6 +235,7 @@ def create_upload(
|
|
|
231
235
|
infohash: str | None = None,
|
|
232
236
|
method: str | Path | None = None,
|
|
233
237
|
description: str | Path | None = None,
|
|
238
|
+
timeout: float | None = 60.0,
|
|
234
239
|
) -> Upload:
|
|
235
240
|
"""
|
|
236
241
|
Create an upload for a dataset or dataset parts.
|
|
@@ -248,11 +253,14 @@ def create_upload(
|
|
|
248
253
|
description (str | Path | None): Any description of the contents of the upload
|
|
249
254
|
that are not contained within the dataset description, e.g. additional structure,
|
|
250
255
|
if this is some sub-component of a dataset, and so on
|
|
256
|
+
timeout (float | None): Timeout in seconds to wait for the upload to complete.
|
|
257
|
+
Defaults to 60 seconds since processing uploads can take some time,
|
|
258
|
+
explicitly pass `None` to use `config.request_timeout`
|
|
251
259
|
"""
|
|
252
260
|
if torrent_path and infohash or not (torrent_path or infohash):
|
|
253
261
|
raise ValueError("Must pass torrent path OR infohash")
|
|
254
262
|
elif torrent_path:
|
|
255
|
-
tf = upload_torrent(path=torrent_path)
|
|
263
|
+
tf = upload_torrent(path=torrent_path, timeout=timeout)
|
|
256
264
|
infohash = tf["v1_infohash"] if tf["version"] == "v1" else tf["v2_infohash"]
|
|
257
265
|
|
|
258
266
|
kwargs: dict[str, str | list[str] | None] = {
|
|
@@ -267,6 +275,6 @@ def create_upload(
|
|
|
267
275
|
if dataset_parts:
|
|
268
276
|
kwargs["part_slugs"] = dataset_parts
|
|
269
277
|
|
|
270
|
-
res = api_call("uploads", data=kwargs)
|
|
278
|
+
res = api_call("uploads", data=kwargs, timeout=timeout)
|
|
271
279
|
ul = cast(Upload, res.json())
|
|
272
280
|
return Upload(**ul)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import warnings
|
|
2
2
|
from dataclasses import dataclass
|
|
3
|
-
from math import ceil
|
|
3
|
+
from math import ceil, sqrt
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
from statistics import median
|
|
6
6
|
from typing import Any, Generator, TypedDict, cast, overload
|
|
@@ -70,9 +70,17 @@ class V1PieceParams(PieceParams):
|
|
|
70
70
|
|
|
71
71
|
@dataclass
|
|
72
72
|
class V2PieceParams(PieceParams):
|
|
73
|
-
median_weight: float = 0.05
|
|
74
73
|
"""
|
|
75
|
-
|
|
74
|
+
Piece size is less important in v2-only torrents because the root hash ends up the same.
|
|
75
|
+
|
|
76
|
+
By default we only care about the number of pieces and use overhead weight as a tiebreaker,
|
|
77
|
+
but if you want to optimize a torrent for streaming (making pieces smaller),
|
|
78
|
+
increase the overhead weight.
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
overhead_weight: float = 1.0e-10
|
|
82
|
+
"""
|
|
83
|
+
Tiny overhead penalty that favors smaller piece sizes to break ties.
|
|
76
84
|
"""
|
|
77
85
|
|
|
78
86
|
|
|
@@ -268,13 +276,13 @@ def find_optimal_piece_size_v2(
|
|
|
268
276
|
target_pieces = _normalize_penalties(
|
|
269
277
|
_penalty_target_pieces(paths=paths, sizes=sizes, version="v2", params=params)
|
|
270
278
|
)
|
|
271
|
-
|
|
279
|
+
overhead = _penalty_overhead(paths=paths, sizes=sizes)
|
|
272
280
|
piece_size = _min_penalty(
|
|
273
|
-
penalties=[target_pieces,
|
|
274
|
-
weights=[params.target_weight, params.
|
|
281
|
+
penalties=[target_pieces, overhead],
|
|
282
|
+
weights=[params.target_weight, params.overhead_weight],
|
|
275
283
|
)
|
|
276
284
|
if return_penalties:
|
|
277
|
-
return piece_size, {"target": target_pieces, "
|
|
285
|
+
return piece_size, {"target": target_pieces, "overhead": overhead}
|
|
278
286
|
else:
|
|
279
287
|
return piece_size
|
|
280
288
|
|
|
@@ -404,8 +412,13 @@ def _penalty_target_pieces(
|
|
|
404
412
|
# penalize going under more than going over
|
|
405
413
|
elif total_pieces < target:
|
|
406
414
|
# piece sizes that want to make us make one piece...
|
|
407
|
-
|
|
408
|
-
|
|
415
|
+
# take the fraction of the target and total so e.g. making 1 piece is 0.999...
|
|
416
|
+
# and making target-1 is 0.00001...
|
|
417
|
+
# then square to make larger misses (like making one piece) matter more
|
|
418
|
+
# then double so that we increase the penalty by up to 2 in the worst cases,
|
|
419
|
+
# and otherwise leave unchangesd
|
|
420
|
+
exponent = (((target - total_pieces) / target) ** 2) * 2
|
|
421
|
+
penalty *= max(exponent, 1)
|
|
409
422
|
|
|
410
423
|
penalties.append(
|
|
411
424
|
Penalty(
|
|
@@ -437,7 +450,8 @@ def _normalize_penalties(penalties: list[Penalty]) -> list[Penalty]:
|
|
|
437
450
|
# all are equal
|
|
438
451
|
penalties[i]["value"] = 0
|
|
439
452
|
else:
|
|
440
|
-
|
|
453
|
+
# take the sqrt of the value after normalizing 0-1 to enlarge small errors around 0
|
|
454
|
+
penalties[i]["value"] = sqrt((penalties[i]["value"] - min_pen) / (max_pen - min_pen))
|
|
441
455
|
return penalties
|
|
442
456
|
|
|
443
457
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
[142, 55, 195, 6148, 191830, 3, 192, 16513, 128, 17914, 6148, 6148, 4512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 6060, 512, 512, 1310508, 512, 512, 275465, 512, 512, 512, 58409, 512, 10638, 512, 1269377, 512, 512, 109911, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 24399, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 5033285, 512, 512, 512, 11353603, 512, 512, 512, 512, 512, 512, 512, 512, 512, 11544899, 512, 512, 512, 1901725, 1945492, 1955557, 1874709, 33105, 260189, 509809, 495830, 303552, 308683, 444971, 443337, 371743, 8349, 1878755, 1898601, 320317, 306840, 446374, 478163, 463701, 547313, 443998, 278714, 33105, 8349, 1881934, 1864659, 8349, 416648, 290765, 216853, 399117, 365746, 206528, 256766, 406015, 1869933, 1876975, 1876987, 1830248, 33105, 427310, 446033, 242199, 274446, 408555, 500628, 366985, 286959, 1889706, 1859539, 8345, 1874340, 1854280, 33101, 436491, 368594, 143366, 276756, 1875136, 1880121, 1880042, 1871658, 8350, 257003, 173057, 376548, 470500, 33106, 463253, 523912, 338963, 311692, 1892197, 1830859, 1891273, 1864130, 365898, 290187, 446908, 561226, 8345, 33101, 8345, 1794466, 1889825, 1820382, 1917371, 33101, 284689, 299686, 471822, 391197, 293781, 371346, 535766, 431745, 347509, 262494, 516028, 441766, 1801154, 1869516, 1814251, 1880349, 509719, 406124, 33105, 234776, 318335, 8349, 358668, 272613, 446052, 457075, 1864795, 8350, 1874028, 1879853, 1865526, 33106, 385992, 403737, 234838, 181158, 1845567, 1891217, 33099, 1837532, 1899007, 236326, 311762, 515692, 424030, 279375, 85849, 333031, 458212, 8344, 430968, 377389, 158271, 33106, 240906, 1865495, 1886842, 1869138, 1891869, 275006, 168209, 400369, 467310, 8350, 432931, 181239, 282538, 364338, 493749, 164145, 259433, 373668, 8350, 33106, 1869274, 1861958, 1871731, 1878147, 8345, 1871708, 1901984, 1817425, 1923937, 33101, 495861, 450840, 313146, 309774, 548836, 408178, 252151, 364001, 410595, 8349, 516822, 312135, 233276, 33105, 1784054, 1854448, 1903390, 1809789, 497555, 414593, 315259, 253979, 33101, 1867496, 1810989, 1803207, 1852131, 8345, 295527, 402417, 451820, 411886, 278357, 529504, 409643, 285588, 449760, 416318, 240894, 242645, 8350, 1875947, 33106, 1867159, 1893165, 1881516, 221893, 156341, 375829, 399718, 1910053, 1881674, 33105, 8349, 483980, 519878, 373012, 288542, 228042, 271904, 468076, 435471, 1925391, 1880157, 1836958, 8349, 1848238, 33105, 460790, 410158, 224846, 351076, 448316, 409488, 300914, 347655, 1876773, 1846609, 1862167, 1844259, 284694, 221990, 401848, 393580, 33106, 410386, 417072, 264002, 234633, 8350, 1853657, 1859028, 245664, 401354, 512565, 430899, 1859053, 33105, 1841977, 1832934, 1848791, 307341, 319338, 460520, 450490, 8349, 1860161, 1894775, 307405, 237505, 420528, 476044, 33105, 493455, 456943, 279902, 382741, 1903739, 1904116, 8349, 33105, 8349, 1881814, 1846434, 1859948, 1893917, 225869, 476423, 480403, 309260, 282243, 456342, 412398, 357130, 33105, 349287, 459313, 476348, 8349, 260377, 314866, 461332, 486409, 253597, 1895650, 1947841, 1913248, 1862330, 33105, 8349, 1887226, 1893500, 1896669, 1837811, 483236, 421204, 281700, 490445, 457459, 331452, 302927, 560119, 406496, 346597, 276854, 447130, 483781, 293832, 232523, 462174, 33105, 8349, 1929508, 1838821, 1871933, 1850646, 1898820, 1874026, 8344, 490876, 505398, 310132, 33099, 228009, 497279, 334463, 84611, 291780, 1859946, 1892842, 8349, 430381, 274417, 420810, 418924, 33105, 523269, 229911, 204381, 487716, 1844793, 1869084, 1856308, 1855776, 1871422, 1873612, 8350, 199207, 226120, 404899, 391420, 33106, 253309, 257949, 436352, 433062, 1869957, 1868077, 285102, 388014, 509931, 446026, 33105, 329050, 258098, 8349, 441639, 492510, 1863546, 1911421, 1883020, 1873061, 1871558, 1898990, 8349, 495630, 481886, 297537, 380928, 312149, 234756, 435560, 33105, 482493, 1887035, 1879822, 1955858, 1889982, 1861465, 1919151, 312082, 466664, 550082, 82873, 316172, 358902, 507891, 221527, 8344, 33099, 444744, 503505, 457558, 262106, 417573, 407712, 228639, 202598, 1844989, 1867954, 33105, 1852085, 1911113, 8349, 1866204, 1808362, 8349, 1831114, 1806760, 529167, 479278, 302496, 472182, 33105, 397486, 376443, 201194, 238488, 407219, 8349, 288618, 319756, 504143, 465933, 393603, 263407, 433745, 1901193, 1876488, 1857970, 1898233, 33105, 8349, 1954694, 1865961, 1874623, 1929630, 312348, 472401, 469403, 238427, 358465, 436080, 442540, 294551, 33105, 1863880, 1853704, 1884441, 1847406, 8349, 454100, 459881, 33105, 261191, 341118, 397854, 437700, 280917, 218130, 1831803, 1868514, 1878160, 1848858, 33105, 539047, 314219, 321119, 446332, 463207, 266027, 424737, 484994, 8349, 447600, 470300, 315997, 243682, 1909671, 1953429, 1886749, 1917888, 33105, 8349, 281878, 352982, 512053, 506680, 8350, 1877685, 1876534, 1886742, 1885725, 404793, 374482, 156308, 221966, 375345, 33106, 450935, 238289, 159172, 1838828, 1865251, 1875581, 1818682, 33105, 8349, 502046, 302403, 255425, 417433, 414984, 304432, 300555, 492154, 1854035, 1901916, 1905159, 1850441, 33101, 8345, 297898, 576065, 418794, 307406, 231504, 454679, 491901, 380035, 8350, 1903416, 1870310, 1867869, 1849923, 457240, 184342, 33106, 226198, 437355, 395981, 264823, 252934, 388914, 1854344, 1882928, 459402, 269703, 465947, 8349, 513509, 457184, 422658, 243250, 307522, 33105, 1869578, 1874349, 1905018, 2014293, 278410, 431263, 533973, 515611, 244982, 199276, 450736, 436302, 33105, 8349, 2006445, 1883983, 33105, 302522, 496186, 437955, 361899, 240841, 484546, 477981, 328909, 8349, 1977509, 1897066, 1890105, 1913932, 33105, 1838235, 1847426, 1852930, 1855845, 8349, 444463, 464612, 347533, 299344, 527431, 438781, 249820, 463395, 1920128, 1841268, 346338, 33099, 322160, 491351, 505230, 399159, 8344, 328471, 87572, 203707, 1899467, 1864498, 350477, 291938, 518515, 471904, 228781, 322381, 469846, 419095, 33105, 8349, 1816564, 1918038, 1824382, 1870270, 8350, 428245, 33106, 373842, 170337, 265056, 1891065, 1859226, 1860748, 1856610, 458880, 412811, 262346, 280595, 33105, 1818844, 1887214, 206374, 246989, 8349, 404676, 398824, 369450, 275384, 467317, 479815, 1866755, 1815400, 33105, 8349, 1880449, 1859795, 463693, 426755, 228174, 308206, 385522, 299416, 448967, 509823, 1876627, 1860933, 8349, 477948, 520699, 287835, 360595, 1878841, 1889276, 33105, 1934672, 1864999, 471055, 460363, 233550, 316333, 275745, 410660, 542509, 466849, 1859627, 33105, 1921395, 8349, 1856187, 1882758, 470239, 483411, 312635, 293540, 356945, 304923, 463340, 484545, 33105, 1921599, 1891423, 8349, 1913723, 1842596, 451171, 437478, 248202, 310284, 258963, 151193, 384856, 436767, 239110, 238010, 33106, 450794, 445773, 1887091, 1893551, 1912843, 1874310, 8350, 1866309, 1865268, 294390, 236782, 423641, 448835, 8349, 33105, 367679, 299723, 486560, 488852, 1839046, 1889513, 551942, 437744, 293434, 418251, 8345, 1894880, 1825082, 1812205, 1879545, 33101, 420492, 480149, 297685, 295392, 8349, 1909486, 1878261, 1879588, 1871245, 506734, 307272, 396618, 448968, 518848, 240081, 339876, 485394, 33105, 1884934, 1867273, 1885975, 1857558, 406106, 226347, 282434, 441593, 409701, 256299, 215243, 414670, 33106, 8350, 1883403, 1781948, 1845711, 1895457, 8349, 424954, 219376, 306551, 508999, 503569, 280450, 321252, 408090, 33105, 1890115, 1863118, 1898196, 1860396, 8350, 392864, 258227, 369740, 422225, 476383, 223289, 187201, 33106, 467290, 1831679, 1889352, 1898141, 1852248, 33100, 88517, 513028, 8345, 414362, 272425, 230453, 461368, 322586, 295279, 1837190, 1879491, 1889076, 1826661, 33105, 482360, 270566, 350930, 397191, 476365, 238104, 323477, 8349, 502733, 8350, 155401, 219986, 400104, 388778, 1852289, 1838773, 1908492, 1894927, 262557, 254535, 417819, 33106, 427046, 292765, 417054, 542395, 290156, 392456, 472974, 405642, 207103, 8345, 1874580, 1856296, 33101, 1812825, 1886567, 366034, 287688, 514598, 513783, 227380, 313129, 455855, 449593, 8349, 1944834, 1873201, 1937256, 1835718, 33105, 1846333, 1827859, 8349, 1855656, 1840049, 272405, 257262, 429438, 33105, 460357, 276018, 223901, 404880, 415752, 1909888, 1859930, 296130, 365214, 499710, 493975, 242012, 290103, 436573, 430431, 8349, 33105, 1878065, 1915330, 1911139, 1866703, 1875784, 1860795, 451730, 252534, 440751, 462151, 509910, 314849, 319804, 445476, 8349, 33105, 33105, 440564, 444744, 285224, 258190, 1863697, 1856303, 1854015, 1830545, 424182, 283952, 460672, 538153, 8349, 33105, 325692, 459545, 489327, 237584, 354356, 424036, 480275, 319740, 1896042, 1880326, 8349, 1893329, 1897594, 8349, 1855094, 1904922, 1844035, 1811537, 33105, 407084, 282735, 424726, 420745, 468940, 253433, 208451, 442128, 33106, 8350, 1886216, 1868022, 1876482, 1868928, 394182, 230193, 155446, 453130, 359086, 257240, 237728, 386384, 289915, 323370, 482624, 384613, 1899332, 33101, 1788819, 1784346, 8345, 1889337, 401613, 260199, 408915, 533808, 400080, 474546, 315021, 215995, 1893185, 33105, 1820861, 1864498, 1840219, 8349, 270469, 330447, 458513, 494049, 33105, 448507, 351443, 247546, 501406, 432775, 380551, 298583, 493356, 8349, 1856493, 1856230, 1857671, 1875102, 8342, 33097, 1892846, 1927954, 346580, 482780, 206535, 84247, 302245, 302113, 529048, 507197, 1903539, 1903866, 8349, 33105, 496956, 204119, 245984, 497769, 429445, 402865, 264880, 431625, 1818135, 1935507, 1915262, 1869695, 406677, 395434, 236043, 33106, 281843, 413686, 427599, 333606, 263555, 8350, 1874146, 1850191, 1839335, 1868603, 8349, 1860027, 1915139, 33105, 1828370, 1873338, 297716, 319594, 474086, 482682, 317293, 233110, 431468, 468502, 448958, 512213, 324052, 251929, 8349, 476290, 564475, 411919, 288281, 33105, 1917215, 1840317, 1907020, 1881718, 263300, 414368, 483903, 334993, 308512, 517030, 381273, 269149, 8345, 33101, 1871671, 1843765, 1796234, 1927651, 364942, 451523, 259435, 142417, 1879143, 1866283, 1873000, 1876670, 33106, 372113, 8350, 389973, 214305, 157561, 471133, 462742, 328353, 275686, 1901271, 1915331, 8349, 1902962, 1915500, 33105, 239191, 283959, 458454, 421252, 8349, 1919695, 1874610, 1880034, 1919755, 30932, 300930, 473477, 447773, 378232, 263392, 508104, 491162, 312240, 1901408, 1863519, 1800080, 1888390, 8349, 418733, 351946, 286027, 482714, 508872, 315724, 231216, 390378, 33105, 1860745, 1870564, 1865675, 1872187, 33106, 422139, 287507, 255950, 437077, 459177, 284830, 224658, 465885, 8350, 1887348, 1875360, 1875759, 1879618, 8349, 500514, 335740, 240836, 534678, 449638, 374365, 312043, 522900, 33105, 1874873, 1875973, 403118, 451805, 307922, 232898, 8349, 444309, 478583, 330309, 251626, 1866296, 33105, 1849817, 1900560, 1901142, 1849632, 1875866, 352495, 310930, 33105, 8349, 454163, 468602, 253371, 333548, 490417, 419272, 1868630, 1891626, 1915412, 1885891, 369937, 8349, 488933, 491361, 310149, 312405, 525828, 525298, 234427, 33105, 8350, 1867596, 1868824, 1857064, 1865962, 33106, 186045, 252011, 419062, 368699, 274116, 201524, 391763, 439987, 8345, 278887, 283643, 407772, 465154, 2017098, 1814078, 33101, 1891141, 1799335, 431565, 582493, 404790, 291055, 1885267, 1814658, 1887668, 1784128, 8349, 252440, 313570, 441120, 494941, 33105, 323616, 247269, 396598, 507430, 254895, 157897, 387943, 417006, 255140, 245906, 448243, 416565, 1934952, 1914901, 1886188, 1912949, 33106, 8350, 153855, 426018, 369345, 248825, 158078, 452636, 388453, 240174, 8350, 33106, 1870595, 1883454, 1888801, 1866961, 285723, 348280, 529633, 448514, 1819875, 1883392, 1855717, 1923897, 33101, 8345, 471528, 428288, 291008, 315889, 33106, 1871548, 8350, 1877470, 1944947, 1933518, 398232, 149616, 249980, 370072, 450785, 159797, 244019, 378852, 33101, 1925193, 1849366, 1799905, 1890119, 307096, 412447, 546539, 319687, 402764, 8345, 509941, 403920, 263071, 251689, 346572, 506964, 393486, 8345, 33101, 1891585, 1841290, 1826299, 1883893, 240294, 368358, 439946, 441592, 8350, 393092, 282931, 155510, 491434, 360156, 298235, 200390, 432443, 33106, 1873974, 1876335, 1863615, 1869720, 33106, 269235, 441245, 469061, 155973, 254590, 384146, 443136, 291909, 1874054, 1901636, 1907282, 1876784, 8350, 33100, 1898443, 1852952, 339034, 467536, 286008, 86555, 8345, 512756, 473609, 226467, 317548, 1809267, 1875785, 474609, 33105, 495301, 259958, 337420, 1874098, 1947974, 8349, 1869777, 1841827, 491258, 429538, 242169, 304793, 479619, 503235, 379487, 295235, 462104, 416961, 33105, 226879, 308962, 8349, 1917351, 1871968, 1931207, 1894408, 1854259, 1844232, 1862527, 1858378, 33105, 467889, 334896, 8349, 290315, 503662, 457264, 434063, 270590, 475067, 33105, 422106, 230999, 336441, 395894, 460384, 223831, 298705, 396673, 8349, 1862517, 1866710, 1865426, 1883487, 33105, 470854, 294379, 415890, 451654, 8349, 523016, 234528, 336213, 457506, 1856525, 1819203, 1847914, 1859364, 1880983, 1936256, 1881686, 1920547, 33105, 8349, 490827, 479179, 295394, 374344, 428416, 461821, 303685, 236274, 1847606, 33105, 1877737, 1872628, 1843493, 8349, 444377, 203054, 249295, 457831, 408943, 329499, 258237, 402322, 33105, 266919, 320798, 495253, 451744, 1922369, 1845425, 8349, 1880436, 1905402, 404949, 280808, 452261, 564338, 453744, 227394, 319430, 496620, 461565, 275384, 353465, 404536, 33105, 1854849, 1797376, 1827856, 1904536, 8349, 426567, 463232, 304740, 225619, 33105, 1879569, 1847730, 8349, 1876018, 1903848, 490455, 486201, 352022, 290025, 8349, 33105, 1906946, 1856682, 1872448, 1905509, 450899, 405305, 469998, 247694, 325450, 466041, 524787, 218076, 1874272, 1882075, 8349, 475154, 545584, 33105, 386855, 302155, 452662, 518468, 323187, 232650, 1890661, 1845183, 1867771, 1871735, 8350, 1863630, 1883510, 33106, 410887, 389744, 200852, 224672, 510276, 437077, 269008, 462739, 337885, 33099, 297702, 503803, 515209, 8344, 85400, 252860, 490742, 358283, 1922669, 1917267, 1898030, 1868615, 33105, 1888075, 1914397, 248847, 322669, 468322, 428663, 289365, 387086, 505322, 450203, 1887020, 1878688, 8349, 33106, 1869526, 1852290, 259710, 222889, 398988, 416647, 227749, 289096, 402537, 399224, 8350, 1859900, 1870744, 85950, 557873, 501027, 276739, 332555, 489565, 349144, 301662, 8340, 33095, 1896997, 1919689, 1913507, 1876341, 1810019, 1925133, 452203, 460437, 455176, 8349, 268490, 234836, 199675, 431795, 391226, 33105, 1794546, 1889427, 1876546, 1865839, 436489, 415342, 208194, 248961, 217963, 294520, 428576, 417962, 1862112, 33106, 1860953, 8350, 1914065, 1843138, 197302, 232827, 431191, 418541, 495622, 8349, 490867, 267013, 365516, 1908054, 1879180, 33105, 1894854, 1855574, 8349, 1907903, 1886572, 482072, 482842, 370623, 291713, 478642, 33105, 417024, 234585, 324243, 1901366, 1854943, 1881348, 1890103, 288054, 508091, 367388, 259602, 92782, 506947, 508303, 8344, 295836, 33099, 302093, 239237, 412195, 444610, 8349, 1828280, 1882332, 1870444, 1823085, 33105, 343581, 275752, 484796, 464796, 252076, 506340, 508354, 315402, 283046, 498539, 470026, 342746, 1885400, 1917179, 1937767, 1904399, 8349, 33105, 402028, 307262, 409578, 439155, 542950, 310094, 259966, 393100, 8349, 1888276, 1812548, 1804313, 1877468, 33105]
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import json
|
|
1
2
|
from random import gammavariate
|
|
2
3
|
from typing import Literal as L
|
|
3
4
|
|
|
@@ -6,6 +7,11 @@ import pytest
|
|
|
6
7
|
from sciop_cli.const import GiB, KiB, MiB, TiB
|
|
7
8
|
from sciop_cli.torrent import find_optimal_piece_size
|
|
8
9
|
|
|
10
|
+
from .conftest import DATA_DIR
|
|
11
|
+
|
|
12
|
+
with open(DATA_DIR / "distributions" / "nbu_arwen_ver01.json") as f:
|
|
13
|
+
arwen_dist = json.load(f)
|
|
14
|
+
|
|
9
15
|
|
|
10
16
|
@pytest.mark.parametrize(
|
|
11
17
|
"version",
|
|
@@ -22,32 +28,32 @@ from sciop_cli.torrent import find_optimal_piece_size
|
|
|
22
28
|
pytest.param([KiB for _ in range(10)], 16 * KiB, id="10-10kb"),
|
|
23
29
|
pytest.param(
|
|
24
30
|
[MiB for _ in range(10)],
|
|
25
|
-
|
|
31
|
+
32 * KiB,
|
|
26
32
|
id="10-10mb",
|
|
27
33
|
),
|
|
28
34
|
pytest.param(
|
|
29
35
|
[10 * MiB + 1 for _ in range(10)],
|
|
30
|
-
|
|
36
|
+
512 * KiB,
|
|
31
37
|
id="10-100mb",
|
|
32
38
|
),
|
|
33
39
|
pytest.param(
|
|
34
40
|
[(100 * MiB) + 1 for _ in range(10)],
|
|
35
|
-
|
|
41
|
+
4 * MiB,
|
|
36
42
|
id="10-1gb",
|
|
37
43
|
),
|
|
38
44
|
pytest.param(
|
|
39
45
|
[10 * (2**30) + 1 for _ in range(10)],
|
|
40
|
-
|
|
46
|
+
16 * MiB,
|
|
41
47
|
id="10-100gb",
|
|
42
48
|
),
|
|
43
49
|
pytest.param(
|
|
44
50
|
[50 * (2**30) for _ in range(10)],
|
|
45
|
-
|
|
51
|
+
128 * MiB,
|
|
46
52
|
id="10-500gb",
|
|
47
53
|
),
|
|
48
54
|
pytest.param(
|
|
49
55
|
[100 * (2**30) for _ in range(10)],
|
|
50
|
-
|
|
56
|
+
128 * MiB,
|
|
51
57
|
id="10-1tb",
|
|
52
58
|
),
|
|
53
59
|
pytest.param(
|
|
@@ -60,17 +66,17 @@ from sciop_cli.torrent import find_optimal_piece_size
|
|
|
60
66
|
# --------------------------------------------------
|
|
61
67
|
pytest.param(
|
|
62
68
|
[100 * GiB] + [1 * KiB for _ in range(100)],
|
|
63
|
-
|
|
69
|
+
16 * MiB,
|
|
64
70
|
id="101-100gb-one-big-file",
|
|
65
71
|
),
|
|
66
72
|
pytest.param(
|
|
67
73
|
[100 * GiB] + [1 * KiB for _ in range(1000)],
|
|
68
|
-
{"v1": 16 * MiB, "v2":
|
|
74
|
+
{"v1": 16 * MiB, "v2": 32 * MiB, "hybrid": 4 * MiB},
|
|
69
75
|
id="1001-100gb-one-big-file",
|
|
70
76
|
),
|
|
71
77
|
pytest.param(
|
|
72
78
|
[100 * GiB] + [1 * KiB for _ in range(100000)],
|
|
73
|
-
{"v1": 16 * MiB, "v2":
|
|
79
|
+
{"v1": 16 * MiB, "v2": 128 * MiB, "hybrid": 128 * KiB},
|
|
74
80
|
id="100001-100gb-one-big-file",
|
|
75
81
|
),
|
|
76
82
|
# --------------------------------------------------
|
|
@@ -78,7 +84,7 @@ from sciop_cli.torrent import find_optimal_piece_size
|
|
|
78
84
|
# --------------------------------------------------
|
|
79
85
|
pytest.param(
|
|
80
86
|
[gammavariate(5, 100) * KiB for _ in range(100)],
|
|
81
|
-
{"v1":
|
|
87
|
+
{"v1": 256 * KiB, "v2": 256 * KiB, "hybrid": (128 * KiB, 256 * KiB)},
|
|
82
88
|
id="100-1kb-gamma-5-100",
|
|
83
89
|
),
|
|
84
90
|
pytest.param(
|
|
@@ -88,7 +94,7 @@ from sciop_cli.torrent import find_optimal_piece_size
|
|
|
88
94
|
),
|
|
89
95
|
pytest.param(
|
|
90
96
|
[gammavariate(5, 100) * MiB for _ in range(100)],
|
|
91
|
-
|
|
97
|
+
(8 * MiB, 16 * MiB),
|
|
92
98
|
id="100-1mb-gamma-5-100",
|
|
93
99
|
),
|
|
94
100
|
pytest.param(
|
|
@@ -96,6 +102,9 @@ from sciop_cli.torrent import find_optimal_piece_size
|
|
|
96
102
|
128 * MiB,
|
|
97
103
|
id="100-1gb-gamma-5-100",
|
|
98
104
|
),
|
|
105
|
+
pytest.param(
|
|
106
|
+
arwen_dist, {"v1": 256 * KiB, "v2": 512 * KiB, "hybrid": 256 * KiB}, id="chronam-arwen"
|
|
107
|
+
),
|
|
99
108
|
],
|
|
100
109
|
)
|
|
101
110
|
def test_auto_piece_size(sizes: list[int], version: L["v1", "v2", "hybrid"], expected):
|
|
@@ -108,6 +117,8 @@ def test_auto_piece_size(sizes: list[int], version: L["v1", "v2", "hybrid"], exp
|
|
|
108
117
|
)
|
|
109
118
|
if isinstance(expected, int):
|
|
110
119
|
assert piece_size == expected
|
|
120
|
+
elif isinstance(expected, tuple):
|
|
121
|
+
assert piece_size in expected
|
|
111
122
|
else:
|
|
112
123
|
exp_val = expected[version]
|
|
113
124
|
if isinstance(exp_val, int):
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|