sciop-cli 0.1.5__tar.gz → 0.1.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/PKG-INFO +1 -1
  2. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/pyproject.toml +1 -1
  3. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/api.py +13 -5
  4. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/torrent.py +24 -10
  5. sciop_cli-0.1.6/tests/data/distributions/nbu_arwen_ver01.json +1 -0
  6. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/test_torrent.py +22 -11
  7. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/LICENSE +0 -0
  8. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/README.md +0 -0
  9. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/__init__.py +0 -0
  10. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/__main__.py +0 -0
  11. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/cli/__init__.py +0 -0
  12. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/cli/api.py +0 -0
  13. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/cli/main.py +0 -0
  14. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/cli/manifest.py +0 -0
  15. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/cli/torrent.py +0 -0
  16. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/config.py +0 -0
  17. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/const.py +0 -0
  18. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/data.py +0 -0
  19. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/exceptions.py +0 -0
  20. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/models/__init__.py +0 -0
  21. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/models/sciop.py +0 -0
  22. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/schema.py +0 -0
  23. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/src/sciop_cli/types.py +0 -0
  24. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/__init__.py +0 -0
  25. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/conftest.py +0 -0
  26. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/data/basic/part_1/subdir/part_1 +0 -0
  27. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/data/basic/part_2/subdir/part_2 +0 -0
  28. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/data/basic_hybrid.torrent +0 -0
  29. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/data/basic_v2.torrent +0 -0
  30. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/fixtures/__init__.py +0 -0
  31. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/fixtures/base.py +0 -0
  32. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/fixtures/fabricators.py +0 -0
  33. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/test_api.py +0 -0
  34. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/test_cli/__init__.py +0 -0
  35. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/test_cli/test_api.py +0 -0
  36. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/test_cli/test_torrent.py +0 -0
  37. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/test_config.py +0 -0
  38. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/test_data.py +0 -0
  39. {sciop_cli-0.1.5 → sciop_cli-0.1.6}/tests/test_temp.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: sciop-cli
3
- Version: 0.1.5
3
+ Version: 0.1.6
4
4
  Summary: A CLI for sciop
5
5
  Author-Email: cosmicexplorer <dmcC2@hypnicjerk.ai>, sneakers-the-rat <sneakers-the-rat@protonmail.com>
6
6
  License: EUPL-1.2
@@ -23,7 +23,7 @@ dependencies = [
23
23
  ]
24
24
  requires-python = ">=3.11"
25
25
  readme = "README.md"
26
- version = "0.1.5"
26
+ version = "0.1.6"
27
27
 
28
28
  [project.license]
29
29
  text = "EUPL-1.2"
@@ -46,6 +46,7 @@ def api_call(
46
46
  instance_url: str | None = None,
47
47
  api_prefix: Literal["/api/v1", False] = "/api/v1",
48
48
  autorefresh_token: bool = True,
49
+ timeout: float | None = None,
49
50
  do_raise: bool = True,
50
51
  ) -> requests.Response:
51
52
  """
@@ -70,6 +71,7 @@ def api_call(
70
71
  autorefresh_token (bool): If `True` (default), try and get a token if we have credentials.
71
72
  Made `False` e.g. to avoid infinite recursion while logging in,
72
73
  or if it is otherwise desirable to use only the preconfigured token.
74
+ timeout (float): Explicit request timeout. If `None`, use config.request_timeout
73
75
  do_raise (bool): If `True` (default), raise on status codes >=400
74
76
  """
75
77
  config = get_config()
@@ -84,6 +86,7 @@ def api_call(
84
86
  instance_url=instance_url,
85
87
  api_prefix=api_prefix,
86
88
  autorefresh_token=autorefresh_token,
89
+ timeout=timeout,
87
90
  )
88
91
 
89
92
  res = requests.request(**request_kwargs)
@@ -109,6 +112,7 @@ def _api_call_kwargs(
109
112
  instance_url: str | None = None,
110
113
  api_prefix: Literal["/api/v1", False] = "/api/v1",
111
114
  autorefresh_token: bool = True,
115
+ timeout: float | None = None,
112
116
  ) -> dict:
113
117
  default = default_headers(
114
118
  instance_url=instance_url, headers=headers, autorefresh_token=autorefresh_token
@@ -126,7 +130,7 @@ def _api_call_kwargs(
126
130
  "method": method,
127
131
  "url": url,
128
132
  "headers": merged_headers,
129
- "timeout": config.request_timeout,
133
+ "timeout": timeout if timeout else config.request_timeout,
130
134
  "files": files,
131
135
  }
132
136
  if data:
@@ -217,9 +221,9 @@ def claim_next(dataset: str) -> DatasetClaim | None:
217
221
  return DatasetClaim(**claim)
218
222
 
219
223
 
220
- def upload_torrent(path: Path) -> TorrentFile:
224
+ def upload_torrent(path: Path, timeout: float | None = 60.0) -> TorrentFile:
221
225
  with open(path, "rb") as f:
222
- response = api_call("torrents", files={"file": f})
226
+ response = api_call("torrents", files={"file": f}, timeout=timeout)
223
227
  tf = cast(TorrentFile, response.json())
224
228
  return TorrentFile(**tf)
225
229
 
@@ -231,6 +235,7 @@ def create_upload(
231
235
  infohash: str | None = None,
232
236
  method: str | Path | None = None,
233
237
  description: str | Path | None = None,
238
+ timeout: float | None = 60.0,
234
239
  ) -> Upload:
235
240
  """
236
241
  Create an upload for a dataset or dataset parts.
@@ -248,11 +253,14 @@ def create_upload(
248
253
  description (str | Path | None): Any description of the contents of the upload
249
254
  that are not contained within the dataset description, e.g. additional structure,
250
255
  if this is some sub-component of a dataset, and so on
256
+ timeout (float | None): Timeout in seconds to wait for the upload to complete.
257
+ Defaults to 60 seconds since processing uploads can take some time,
258
+ explicitly pass `None` to use `config.request_timeout`
251
259
  """
252
260
  if torrent_path and infohash or not (torrent_path or infohash):
253
261
  raise ValueError("Must pass torrent path OR infohash")
254
262
  elif torrent_path:
255
- tf = upload_torrent(path=torrent_path)
263
+ tf = upload_torrent(path=torrent_path, timeout=timeout)
256
264
  infohash = tf["v1_infohash"] if tf["version"] == "v1" else tf["v2_infohash"]
257
265
 
258
266
  kwargs: dict[str, str | list[str] | None] = {
@@ -267,6 +275,6 @@ def create_upload(
267
275
  if dataset_parts:
268
276
  kwargs["part_slugs"] = dataset_parts
269
277
 
270
- res = api_call("uploads", data=kwargs)
278
+ res = api_call("uploads", data=kwargs, timeout=timeout)
271
279
  ul = cast(Upload, res.json())
272
280
  return Upload(**ul)
@@ -1,6 +1,6 @@
1
1
  import warnings
2
2
  from dataclasses import dataclass
3
- from math import ceil
3
+ from math import ceil, sqrt
4
4
  from pathlib import Path
5
5
  from statistics import median
6
6
  from typing import Any, Generator, TypedDict, cast, overload
@@ -70,9 +70,17 @@ class V1PieceParams(PieceParams):
70
70
 
71
71
  @dataclass
72
72
  class V2PieceParams(PieceParams):
73
- median_weight: float = 0.05
74
73
  """
75
- Relative weight of penalty for piece sizes that are larger than the median file size
74
+ Piece size is less important in v2-only torrents because the root hash ends up the same.
75
+
76
+ By default we only care about the number of pieces and use overhead weight as a tiebreaker,
77
+ but if you want to optimize a torrent for streaming (making pieces smaller),
78
+ increase the overhead weight.
79
+ """
80
+
81
+ overhead_weight: float = 1.0e-10
82
+ """
83
+ Tiny overhead penalty that favors smaller piece sizes to break ties.
76
84
  """
77
85
 
78
86
 
@@ -268,13 +276,13 @@ def find_optimal_piece_size_v2(
268
276
  target_pieces = _normalize_penalties(
269
277
  _penalty_target_pieces(paths=paths, sizes=sizes, version="v2", params=params)
270
278
  )
271
- med_size = _normalize_penalties(_penalty_median_size(paths=paths, sizes=sizes))
279
+ overhead = _penalty_overhead(paths=paths, sizes=sizes)
272
280
  piece_size = _min_penalty(
273
- penalties=[target_pieces, med_size],
274
- weights=[params.target_weight, params.median_weight],
281
+ penalties=[target_pieces, overhead],
282
+ weights=[params.target_weight, params.overhead_weight],
275
283
  )
276
284
  if return_penalties:
277
- return piece_size, {"target": target_pieces, "median": med_size}
285
+ return piece_size, {"target": target_pieces, "overhead": overhead}
278
286
  else:
279
287
  return piece_size
280
288
 
@@ -404,8 +412,13 @@ def _penalty_target_pieces(
404
412
  # penalize going under more than going over
405
413
  elif total_pieces < target:
406
414
  # piece sizes that want to make us make one piece...
407
- exponent = ((target - total_pieces) / target) * 2
408
- penalty *= max(penalty * exponent, 1)
415
+ # take the fraction of the target and total so e.g. making 1 piece is 0.999...
416
+ # and making target-1 is 0.00001...
417
+ # then square to make larger misses (like making one piece) matter more
418
+ # then double so that we increase the penalty by up to 2 in the worst cases,
419
+ # and otherwise leave unchangesd
420
+ exponent = (((target - total_pieces) / target) ** 2) * 2
421
+ penalty *= max(exponent, 1)
409
422
 
410
423
  penalties.append(
411
424
  Penalty(
@@ -437,7 +450,8 @@ def _normalize_penalties(penalties: list[Penalty]) -> list[Penalty]:
437
450
  # all are equal
438
451
  penalties[i]["value"] = 0
439
452
  else:
440
- penalties[i]["value"] = (penalties[i]["value"] - min_pen) / (max_pen - min_pen)
453
+ # take the sqrt of the value after normalizing 0-1 to enlarge small errors around 0
454
+ penalties[i]["value"] = sqrt((penalties[i]["value"] - min_pen) / (max_pen - min_pen))
441
455
  return penalties
442
456
 
443
457
 
@@ -0,0 +1 @@
1
+ [142, 55, 195, 6148, 191830, 3, 192, 16513, 128, 17914, 6148, 6148, 4512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 6060, 512, 512, 1310508, 512, 512, 275465, 512, 512, 512, 58409, 512, 10638, 512, 1269377, 512, 512, 109911, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 24399, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 5033285, 512, 512, 512, 11353603, 512, 512, 512, 512, 512, 512, 512, 512, 512, 11544899, 512, 512, 512, 1901725, 1945492, 1955557, 1874709, 33105, 260189, 509809, 495830, 303552, 308683, 444971, 443337, 371743, 8349, 1878755, 1898601, 320317, 306840, 446374, 478163, 463701, 547313, 443998, 278714, 33105, 8349, 1881934, 1864659, 8349, 416648, 290765, 216853, 399117, 365746, 206528, 256766, 406015, 1869933, 1876975, 1876987, 1830248, 33105, 427310, 446033, 242199, 274446, 408555, 500628, 366985, 286959, 1889706, 1859539, 8345, 1874340, 1854280, 33101, 436491, 368594, 143366, 276756, 1875136, 1880121, 1880042, 1871658, 8350, 257003, 173057, 376548, 470500, 33106, 463253, 523912, 338963, 311692, 1892197, 1830859, 1891273, 1864130, 365898, 290187, 446908, 561226, 8345, 33101, 8345, 1794466, 1889825, 1820382, 1917371, 33101, 284689, 299686, 471822, 391197, 293781, 371346, 535766, 431745, 347509, 262494, 516028, 441766, 1801154, 1869516, 1814251, 1880349, 509719, 406124, 33105, 234776, 318335, 8349, 358668, 272613, 446052, 457075, 1864795, 8350, 1874028, 1879853, 1865526, 33106, 385992, 403737, 234838, 181158, 1845567, 1891217, 33099, 1837532, 1899007, 236326, 311762, 515692, 424030, 279375, 85849, 333031, 458212, 8344, 430968, 377389, 158271, 33106, 240906, 1865495, 1886842, 1869138, 1891869, 275006, 168209, 400369, 467310, 8350, 432931, 181239, 282538, 364338, 493749, 164145, 259433, 373668, 8350, 33106, 1869274, 1861958, 1871731, 1878147, 8345, 1871708, 1901984, 1817425, 1923937, 33101, 495861, 450840, 313146, 309774, 548836, 408178, 252151, 364001, 410595, 8349, 516822, 312135, 233276, 33105, 1784054, 1854448, 1903390, 1809789, 497555, 414593, 315259, 253979, 33101, 1867496, 1810989, 1803207, 1852131, 8345, 295527, 402417, 451820, 411886, 278357, 529504, 409643, 285588, 449760, 416318, 240894, 242645, 8350, 1875947, 33106, 1867159, 1893165, 1881516, 221893, 156341, 375829, 399718, 1910053, 1881674, 33105, 8349, 483980, 519878, 373012, 288542, 228042, 271904, 468076, 435471, 1925391, 1880157, 1836958, 8349, 1848238, 33105, 460790, 410158, 224846, 351076, 448316, 409488, 300914, 347655, 1876773, 1846609, 1862167, 1844259, 284694, 221990, 401848, 393580, 33106, 410386, 417072, 264002, 234633, 8350, 1853657, 1859028, 245664, 401354, 512565, 430899, 1859053, 33105, 1841977, 1832934, 1848791, 307341, 319338, 460520, 450490, 8349, 1860161, 1894775, 307405, 237505, 420528, 476044, 33105, 493455, 456943, 279902, 382741, 1903739, 1904116, 8349, 33105, 8349, 1881814, 1846434, 1859948, 1893917, 225869, 476423, 480403, 309260, 282243, 456342, 412398, 357130, 33105, 349287, 459313, 476348, 8349, 260377, 314866, 461332, 486409, 253597, 1895650, 1947841, 1913248, 1862330, 33105, 8349, 1887226, 1893500, 1896669, 1837811, 483236, 421204, 281700, 490445, 457459, 331452, 302927, 560119, 406496, 346597, 276854, 447130, 483781, 293832, 232523, 462174, 33105, 8349, 1929508, 1838821, 1871933, 1850646, 1898820, 1874026, 8344, 490876, 505398, 310132, 33099, 228009, 497279, 334463, 84611, 291780, 1859946, 1892842, 8349, 430381, 274417, 420810, 418924, 33105, 523269, 229911, 204381, 487716, 1844793, 1869084, 1856308, 1855776, 1871422, 1873612, 8350, 199207, 226120, 404899, 391420, 33106, 253309, 257949, 436352, 433062, 1869957, 1868077, 285102, 388014, 509931, 446026, 33105, 329050, 258098, 8349, 441639, 492510, 1863546, 1911421, 1883020, 1873061, 1871558, 1898990, 8349, 495630, 481886, 297537, 380928, 312149, 234756, 435560, 33105, 482493, 1887035, 1879822, 1955858, 1889982, 1861465, 1919151, 312082, 466664, 550082, 82873, 316172, 358902, 507891, 221527, 8344, 33099, 444744, 503505, 457558, 262106, 417573, 407712, 228639, 202598, 1844989, 1867954, 33105, 1852085, 1911113, 8349, 1866204, 1808362, 8349, 1831114, 1806760, 529167, 479278, 302496, 472182, 33105, 397486, 376443, 201194, 238488, 407219, 8349, 288618, 319756, 504143, 465933, 393603, 263407, 433745, 1901193, 1876488, 1857970, 1898233, 33105, 8349, 1954694, 1865961, 1874623, 1929630, 312348, 472401, 469403, 238427, 358465, 436080, 442540, 294551, 33105, 1863880, 1853704, 1884441, 1847406, 8349, 454100, 459881, 33105, 261191, 341118, 397854, 437700, 280917, 218130, 1831803, 1868514, 1878160, 1848858, 33105, 539047, 314219, 321119, 446332, 463207, 266027, 424737, 484994, 8349, 447600, 470300, 315997, 243682, 1909671, 1953429, 1886749, 1917888, 33105, 8349, 281878, 352982, 512053, 506680, 8350, 1877685, 1876534, 1886742, 1885725, 404793, 374482, 156308, 221966, 375345, 33106, 450935, 238289, 159172, 1838828, 1865251, 1875581, 1818682, 33105, 8349, 502046, 302403, 255425, 417433, 414984, 304432, 300555, 492154, 1854035, 1901916, 1905159, 1850441, 33101, 8345, 297898, 576065, 418794, 307406, 231504, 454679, 491901, 380035, 8350, 1903416, 1870310, 1867869, 1849923, 457240, 184342, 33106, 226198, 437355, 395981, 264823, 252934, 388914, 1854344, 1882928, 459402, 269703, 465947, 8349, 513509, 457184, 422658, 243250, 307522, 33105, 1869578, 1874349, 1905018, 2014293, 278410, 431263, 533973, 515611, 244982, 199276, 450736, 436302, 33105, 8349, 2006445, 1883983, 33105, 302522, 496186, 437955, 361899, 240841, 484546, 477981, 328909, 8349, 1977509, 1897066, 1890105, 1913932, 33105, 1838235, 1847426, 1852930, 1855845, 8349, 444463, 464612, 347533, 299344, 527431, 438781, 249820, 463395, 1920128, 1841268, 346338, 33099, 322160, 491351, 505230, 399159, 8344, 328471, 87572, 203707, 1899467, 1864498, 350477, 291938, 518515, 471904, 228781, 322381, 469846, 419095, 33105, 8349, 1816564, 1918038, 1824382, 1870270, 8350, 428245, 33106, 373842, 170337, 265056, 1891065, 1859226, 1860748, 1856610, 458880, 412811, 262346, 280595, 33105, 1818844, 1887214, 206374, 246989, 8349, 404676, 398824, 369450, 275384, 467317, 479815, 1866755, 1815400, 33105, 8349, 1880449, 1859795, 463693, 426755, 228174, 308206, 385522, 299416, 448967, 509823, 1876627, 1860933, 8349, 477948, 520699, 287835, 360595, 1878841, 1889276, 33105, 1934672, 1864999, 471055, 460363, 233550, 316333, 275745, 410660, 542509, 466849, 1859627, 33105, 1921395, 8349, 1856187, 1882758, 470239, 483411, 312635, 293540, 356945, 304923, 463340, 484545, 33105, 1921599, 1891423, 8349, 1913723, 1842596, 451171, 437478, 248202, 310284, 258963, 151193, 384856, 436767, 239110, 238010, 33106, 450794, 445773, 1887091, 1893551, 1912843, 1874310, 8350, 1866309, 1865268, 294390, 236782, 423641, 448835, 8349, 33105, 367679, 299723, 486560, 488852, 1839046, 1889513, 551942, 437744, 293434, 418251, 8345, 1894880, 1825082, 1812205, 1879545, 33101, 420492, 480149, 297685, 295392, 8349, 1909486, 1878261, 1879588, 1871245, 506734, 307272, 396618, 448968, 518848, 240081, 339876, 485394, 33105, 1884934, 1867273, 1885975, 1857558, 406106, 226347, 282434, 441593, 409701, 256299, 215243, 414670, 33106, 8350, 1883403, 1781948, 1845711, 1895457, 8349, 424954, 219376, 306551, 508999, 503569, 280450, 321252, 408090, 33105, 1890115, 1863118, 1898196, 1860396, 8350, 392864, 258227, 369740, 422225, 476383, 223289, 187201, 33106, 467290, 1831679, 1889352, 1898141, 1852248, 33100, 88517, 513028, 8345, 414362, 272425, 230453, 461368, 322586, 295279, 1837190, 1879491, 1889076, 1826661, 33105, 482360, 270566, 350930, 397191, 476365, 238104, 323477, 8349, 502733, 8350, 155401, 219986, 400104, 388778, 1852289, 1838773, 1908492, 1894927, 262557, 254535, 417819, 33106, 427046, 292765, 417054, 542395, 290156, 392456, 472974, 405642, 207103, 8345, 1874580, 1856296, 33101, 1812825, 1886567, 366034, 287688, 514598, 513783, 227380, 313129, 455855, 449593, 8349, 1944834, 1873201, 1937256, 1835718, 33105, 1846333, 1827859, 8349, 1855656, 1840049, 272405, 257262, 429438, 33105, 460357, 276018, 223901, 404880, 415752, 1909888, 1859930, 296130, 365214, 499710, 493975, 242012, 290103, 436573, 430431, 8349, 33105, 1878065, 1915330, 1911139, 1866703, 1875784, 1860795, 451730, 252534, 440751, 462151, 509910, 314849, 319804, 445476, 8349, 33105, 33105, 440564, 444744, 285224, 258190, 1863697, 1856303, 1854015, 1830545, 424182, 283952, 460672, 538153, 8349, 33105, 325692, 459545, 489327, 237584, 354356, 424036, 480275, 319740, 1896042, 1880326, 8349, 1893329, 1897594, 8349, 1855094, 1904922, 1844035, 1811537, 33105, 407084, 282735, 424726, 420745, 468940, 253433, 208451, 442128, 33106, 8350, 1886216, 1868022, 1876482, 1868928, 394182, 230193, 155446, 453130, 359086, 257240, 237728, 386384, 289915, 323370, 482624, 384613, 1899332, 33101, 1788819, 1784346, 8345, 1889337, 401613, 260199, 408915, 533808, 400080, 474546, 315021, 215995, 1893185, 33105, 1820861, 1864498, 1840219, 8349, 270469, 330447, 458513, 494049, 33105, 448507, 351443, 247546, 501406, 432775, 380551, 298583, 493356, 8349, 1856493, 1856230, 1857671, 1875102, 8342, 33097, 1892846, 1927954, 346580, 482780, 206535, 84247, 302245, 302113, 529048, 507197, 1903539, 1903866, 8349, 33105, 496956, 204119, 245984, 497769, 429445, 402865, 264880, 431625, 1818135, 1935507, 1915262, 1869695, 406677, 395434, 236043, 33106, 281843, 413686, 427599, 333606, 263555, 8350, 1874146, 1850191, 1839335, 1868603, 8349, 1860027, 1915139, 33105, 1828370, 1873338, 297716, 319594, 474086, 482682, 317293, 233110, 431468, 468502, 448958, 512213, 324052, 251929, 8349, 476290, 564475, 411919, 288281, 33105, 1917215, 1840317, 1907020, 1881718, 263300, 414368, 483903, 334993, 308512, 517030, 381273, 269149, 8345, 33101, 1871671, 1843765, 1796234, 1927651, 364942, 451523, 259435, 142417, 1879143, 1866283, 1873000, 1876670, 33106, 372113, 8350, 389973, 214305, 157561, 471133, 462742, 328353, 275686, 1901271, 1915331, 8349, 1902962, 1915500, 33105, 239191, 283959, 458454, 421252, 8349, 1919695, 1874610, 1880034, 1919755, 30932, 300930, 473477, 447773, 378232, 263392, 508104, 491162, 312240, 1901408, 1863519, 1800080, 1888390, 8349, 418733, 351946, 286027, 482714, 508872, 315724, 231216, 390378, 33105, 1860745, 1870564, 1865675, 1872187, 33106, 422139, 287507, 255950, 437077, 459177, 284830, 224658, 465885, 8350, 1887348, 1875360, 1875759, 1879618, 8349, 500514, 335740, 240836, 534678, 449638, 374365, 312043, 522900, 33105, 1874873, 1875973, 403118, 451805, 307922, 232898, 8349, 444309, 478583, 330309, 251626, 1866296, 33105, 1849817, 1900560, 1901142, 1849632, 1875866, 352495, 310930, 33105, 8349, 454163, 468602, 253371, 333548, 490417, 419272, 1868630, 1891626, 1915412, 1885891, 369937, 8349, 488933, 491361, 310149, 312405, 525828, 525298, 234427, 33105, 8350, 1867596, 1868824, 1857064, 1865962, 33106, 186045, 252011, 419062, 368699, 274116, 201524, 391763, 439987, 8345, 278887, 283643, 407772, 465154, 2017098, 1814078, 33101, 1891141, 1799335, 431565, 582493, 404790, 291055, 1885267, 1814658, 1887668, 1784128, 8349, 252440, 313570, 441120, 494941, 33105, 323616, 247269, 396598, 507430, 254895, 157897, 387943, 417006, 255140, 245906, 448243, 416565, 1934952, 1914901, 1886188, 1912949, 33106, 8350, 153855, 426018, 369345, 248825, 158078, 452636, 388453, 240174, 8350, 33106, 1870595, 1883454, 1888801, 1866961, 285723, 348280, 529633, 448514, 1819875, 1883392, 1855717, 1923897, 33101, 8345, 471528, 428288, 291008, 315889, 33106, 1871548, 8350, 1877470, 1944947, 1933518, 398232, 149616, 249980, 370072, 450785, 159797, 244019, 378852, 33101, 1925193, 1849366, 1799905, 1890119, 307096, 412447, 546539, 319687, 402764, 8345, 509941, 403920, 263071, 251689, 346572, 506964, 393486, 8345, 33101, 1891585, 1841290, 1826299, 1883893, 240294, 368358, 439946, 441592, 8350, 393092, 282931, 155510, 491434, 360156, 298235, 200390, 432443, 33106, 1873974, 1876335, 1863615, 1869720, 33106, 269235, 441245, 469061, 155973, 254590, 384146, 443136, 291909, 1874054, 1901636, 1907282, 1876784, 8350, 33100, 1898443, 1852952, 339034, 467536, 286008, 86555, 8345, 512756, 473609, 226467, 317548, 1809267, 1875785, 474609, 33105, 495301, 259958, 337420, 1874098, 1947974, 8349, 1869777, 1841827, 491258, 429538, 242169, 304793, 479619, 503235, 379487, 295235, 462104, 416961, 33105, 226879, 308962, 8349, 1917351, 1871968, 1931207, 1894408, 1854259, 1844232, 1862527, 1858378, 33105, 467889, 334896, 8349, 290315, 503662, 457264, 434063, 270590, 475067, 33105, 422106, 230999, 336441, 395894, 460384, 223831, 298705, 396673, 8349, 1862517, 1866710, 1865426, 1883487, 33105, 470854, 294379, 415890, 451654, 8349, 523016, 234528, 336213, 457506, 1856525, 1819203, 1847914, 1859364, 1880983, 1936256, 1881686, 1920547, 33105, 8349, 490827, 479179, 295394, 374344, 428416, 461821, 303685, 236274, 1847606, 33105, 1877737, 1872628, 1843493, 8349, 444377, 203054, 249295, 457831, 408943, 329499, 258237, 402322, 33105, 266919, 320798, 495253, 451744, 1922369, 1845425, 8349, 1880436, 1905402, 404949, 280808, 452261, 564338, 453744, 227394, 319430, 496620, 461565, 275384, 353465, 404536, 33105, 1854849, 1797376, 1827856, 1904536, 8349, 426567, 463232, 304740, 225619, 33105, 1879569, 1847730, 8349, 1876018, 1903848, 490455, 486201, 352022, 290025, 8349, 33105, 1906946, 1856682, 1872448, 1905509, 450899, 405305, 469998, 247694, 325450, 466041, 524787, 218076, 1874272, 1882075, 8349, 475154, 545584, 33105, 386855, 302155, 452662, 518468, 323187, 232650, 1890661, 1845183, 1867771, 1871735, 8350, 1863630, 1883510, 33106, 410887, 389744, 200852, 224672, 510276, 437077, 269008, 462739, 337885, 33099, 297702, 503803, 515209, 8344, 85400, 252860, 490742, 358283, 1922669, 1917267, 1898030, 1868615, 33105, 1888075, 1914397, 248847, 322669, 468322, 428663, 289365, 387086, 505322, 450203, 1887020, 1878688, 8349, 33106, 1869526, 1852290, 259710, 222889, 398988, 416647, 227749, 289096, 402537, 399224, 8350, 1859900, 1870744, 85950, 557873, 501027, 276739, 332555, 489565, 349144, 301662, 8340, 33095, 1896997, 1919689, 1913507, 1876341, 1810019, 1925133, 452203, 460437, 455176, 8349, 268490, 234836, 199675, 431795, 391226, 33105, 1794546, 1889427, 1876546, 1865839, 436489, 415342, 208194, 248961, 217963, 294520, 428576, 417962, 1862112, 33106, 1860953, 8350, 1914065, 1843138, 197302, 232827, 431191, 418541, 495622, 8349, 490867, 267013, 365516, 1908054, 1879180, 33105, 1894854, 1855574, 8349, 1907903, 1886572, 482072, 482842, 370623, 291713, 478642, 33105, 417024, 234585, 324243, 1901366, 1854943, 1881348, 1890103, 288054, 508091, 367388, 259602, 92782, 506947, 508303, 8344, 295836, 33099, 302093, 239237, 412195, 444610, 8349, 1828280, 1882332, 1870444, 1823085, 33105, 343581, 275752, 484796, 464796, 252076, 506340, 508354, 315402, 283046, 498539, 470026, 342746, 1885400, 1917179, 1937767, 1904399, 8349, 33105, 402028, 307262, 409578, 439155, 542950, 310094, 259966, 393100, 8349, 1888276, 1812548, 1804313, 1877468, 33105]
@@ -1,3 +1,4 @@
1
+ import json
1
2
  from random import gammavariate
2
3
  from typing import Literal as L
3
4
 
@@ -6,6 +7,11 @@ import pytest
6
7
  from sciop_cli.const import GiB, KiB, MiB, TiB
7
8
  from sciop_cli.torrent import find_optimal_piece_size
8
9
 
10
+ from .conftest import DATA_DIR
11
+
12
+ with open(DATA_DIR / "distributions" / "nbu_arwen_ver01.json") as f:
13
+ arwen_dist = json.load(f)
14
+
9
15
 
10
16
  @pytest.mark.parametrize(
11
17
  "version",
@@ -22,32 +28,32 @@ from sciop_cli.torrent import find_optimal_piece_size
22
28
  pytest.param([KiB for _ in range(10)], 16 * KiB, id="10-10kb"),
23
29
  pytest.param(
24
30
  [MiB for _ in range(10)],
25
- {"v1": 32 * KiB, "v2": 32 * KiB, "hybrid": 16 * KiB},
31
+ 32 * KiB,
26
32
  id="10-10mb",
27
33
  ),
28
34
  pytest.param(
29
35
  [10 * MiB + 1 for _ in range(10)],
30
- {"v1": 256 * KiB, "v2": 256 * KiB, "hybrid": 128 * KiB},
36
+ 512 * KiB,
31
37
  id="10-100mb",
32
38
  ),
33
39
  pytest.param(
34
40
  [(100 * MiB) + 1 for _ in range(10)],
35
- {"v1": 4 * MiB, "v2": 4 * MiB, "hybrid": 1 * MiB},
41
+ 4 * MiB,
36
42
  id="10-1gb",
37
43
  ),
38
44
  pytest.param(
39
45
  [10 * (2**30) + 1 for _ in range(10)],
40
- {"v1": 16 * MiB, "v2": 16 * MiB, "hybrid": 8 * MiB},
46
+ 16 * MiB,
41
47
  id="10-100gb",
42
48
  ),
43
49
  pytest.param(
44
50
  [50 * (2**30) for _ in range(10)],
45
- {"v1": 64 * MiB, "v2": 64 * MiB, "hybrid": 32 * MiB},
51
+ 128 * MiB,
46
52
  id="10-500gb",
47
53
  ),
48
54
  pytest.param(
49
55
  [100 * (2**30) for _ in range(10)],
50
- {"v1": 128 * MiB, "v2": 128 * MiB, "hybrid": 32 * MiB},
56
+ 128 * MiB,
51
57
  id="10-1tb",
52
58
  ),
53
59
  pytest.param(
@@ -60,17 +66,17 @@ from sciop_cli.torrent import find_optimal_piece_size
60
66
  # --------------------------------------------------
61
67
  pytest.param(
62
68
  [100 * GiB] + [1 * KiB for _ in range(100)],
63
- {"v1": 16 * MiB, "v2": 8 * MiB, "hybrid": 2 * MiB},
69
+ 16 * MiB,
64
70
  id="101-100gb-one-big-file",
65
71
  ),
66
72
  pytest.param(
67
73
  [100 * GiB] + [1 * KiB for _ in range(1000)],
68
- {"v1": 16 * MiB, "v2": 8 * MiB, "hybrid": 1 * MiB},
74
+ {"v1": 16 * MiB, "v2": 32 * MiB, "hybrid": 4 * MiB},
69
75
  id="1001-100gb-one-big-file",
70
76
  ),
71
77
  pytest.param(
72
78
  [100 * GiB] + [1 * KiB for _ in range(100000)],
73
- {"v1": 16 * MiB, "v2": 16 * MiB, "hybrid": 128 * KiB},
79
+ {"v1": 16 * MiB, "v2": 128 * MiB, "hybrid": 128 * KiB},
74
80
  id="100001-100gb-one-big-file",
75
81
  ),
76
82
  # --------------------------------------------------
@@ -78,7 +84,7 @@ from sciop_cli.torrent import find_optimal_piece_size
78
84
  # --------------------------------------------------
79
85
  pytest.param(
80
86
  [gammavariate(5, 100) * KiB for _ in range(100)],
81
- {"v1": 128 * KiB, "v2": 256 * KiB, "hybrid": (32 * KiB, 64 * KiB)},
87
+ {"v1": 256 * KiB, "v2": 256 * KiB, "hybrid": (128 * KiB, 256 * KiB)},
82
88
  id="100-1kb-gamma-5-100",
83
89
  ),
84
90
  pytest.param(
@@ -88,7 +94,7 @@ from sciop_cli.torrent import find_optimal_piece_size
88
94
  ),
89
95
  pytest.param(
90
96
  [gammavariate(5, 100) * MiB for _ in range(100)],
91
- {"v1": 8 * MiB, "v2": 8 * MiB, "hybrid": 1 * MiB},
97
+ (8 * MiB, 16 * MiB),
92
98
  id="100-1mb-gamma-5-100",
93
99
  ),
94
100
  pytest.param(
@@ -96,6 +102,9 @@ from sciop_cli.torrent import find_optimal_piece_size
96
102
  128 * MiB,
97
103
  id="100-1gb-gamma-5-100",
98
104
  ),
105
+ pytest.param(
106
+ arwen_dist, {"v1": 256 * KiB, "v2": 512 * KiB, "hybrid": 256 * KiB}, id="chronam-arwen"
107
+ ),
99
108
  ],
100
109
  )
101
110
  def test_auto_piece_size(sizes: list[int], version: L["v1", "v2", "hybrid"], expected):
@@ -108,6 +117,8 @@ def test_auto_piece_size(sizes: list[int], version: L["v1", "v2", "hybrid"], exp
108
117
  )
109
118
  if isinstance(expected, int):
110
119
  assert piece_size == expected
120
+ elif isinstance(expected, tuple):
121
+ assert piece_size in expected
111
122
  else:
112
123
  exp_val = expected[version]
113
124
  if isinstance(exp_val, int):
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes