sciop-cli 0.3.0__tar.gz → 0.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/PKG-INFO +1 -1
  2. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/pyproject.toml +1 -1
  3. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/cli/pack.py +8 -0
  4. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/pack.py +45 -18
  5. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/schema.py +8 -16
  6. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/torrent.py +0 -2
  7. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/test_api.py +3 -3
  8. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/test_pack.py +22 -17
  9. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/LICENSE +0 -0
  10. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/README.md +0 -0
  11. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/__init__.py +0 -0
  12. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/__main__.py +0 -0
  13. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/api.py +0 -0
  14. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/cli/__init__.py +0 -0
  15. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/cli/api.py +0 -0
  16. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/cli/client.py +0 -0
  17. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/cli/main.py +0 -0
  18. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/cli/manifest.py +0 -0
  19. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/cli/torrent.py +0 -0
  20. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/clients/__init__.py +0 -0
  21. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/clients/base.py +0 -0
  22. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/clients/qbt.py +0 -0
  23. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/config.py +0 -0
  24. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/const.py +0 -0
  25. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/data.py +0 -0
  26. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/exceptions.py +0 -0
  27. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/models/__init__.py +0 -0
  28. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/models/pack.py +0 -0
  29. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/models/sciop.py +0 -0
  30. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/src/sciop_cli/types.py +0 -0
  31. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/__init__.py +0 -0
  32. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/conftest.py +0 -0
  33. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/data/basic/part_1/subdir/part_1 +0 -0
  34. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/data/basic/part_2/subdir/part_2 +0 -0
  35. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/data/basic_hybrid.torrent +0 -0
  36. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/data/basic_v2.torrent +0 -0
  37. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/data/distributions/dlc_chrysler_ver02.json +0 -0
  38. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/data/distributions/nbu_arwen_ver01.json +0 -0
  39. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/fixtures/__init__.py +0 -0
  40. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/fixtures/base.py +0 -0
  41. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/fixtures/fabricators.py +0 -0
  42. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/test_cli/__init__.py +0 -0
  43. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/test_cli/test_api.py +0 -0
  44. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/test_cli/test_torrent.py +0 -0
  45. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/test_client/__init__.py +0 -0
  46. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/test_client/test_client_config.py +0 -0
  47. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/test_config.py +0 -0
  48. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/test_data.py +0 -0
  49. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/test_temp.py +0 -0
  50. {sciop_cli-0.3.0 → sciop_cli-0.3.1}/tests/test_torrent.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: sciop-cli
3
- Version: 0.3.0
3
+ Version: 0.3.1
4
4
  Summary: A CLI for sciop
5
5
  Author-Email: cosmicexplorer <dmcC2@hypnicjerk.ai>, sneakers-the-rat <sneakers-the-rat@protonmail.com>
6
6
  License: EUPL-1.2
@@ -26,7 +26,7 @@ dependencies = [
26
26
  ]
27
27
  requires-python = ">=3.11"
28
28
  readme = "README.md"
29
- version = "0.3.0"
29
+ version = "0.3.1"
30
30
 
31
31
  [project.license]
32
32
  text = "EUPL-1.2"
@@ -52,6 +52,12 @@ from sciop_cli.pack import ALGO_MAP, PACK_MODES, PackArgs, PackManager, run_pack
52
52
  show_default=True,
53
53
  help="Which checksum algorithm to write into manifest-<algo>.txt",
54
54
  )
55
+ @click.option(
56
+ "--skip-hash",
57
+ is_flag=True,
58
+ default=False,
59
+ help="Skip checksum file creation",
60
+ )
55
61
  def pack(
56
62
  path: str,
57
63
  mode: PACK_MODES = "heuristic",
@@ -62,6 +68,7 @@ def pack(
62
68
  min_total_mib: int | None = None,
63
69
  top_n: int | None = None,
64
70
  hash_algo: str = "sha512",
71
+ skip_hash: bool = False,
65
72
  ) -> None:
66
73
  args = PackArgs.from_kwargs(
67
74
  mode=mode,
@@ -73,6 +80,7 @@ def pack(
73
80
  min_total_mib=min_total_mib,
74
81
  top_n=top_n,
75
82
  hash_algo=hash_algo,
83
+ skip_hash=skip_hash,
76
84
  )
77
85
 
78
86
  try:
@@ -39,6 +39,7 @@ class PackArgs(BaseModel):
39
39
  mode: str
40
40
  base_dir: DirectoryPath
41
41
  hash_algo: HASH_ALGOS = "sha512"
42
+ skip_hash: bool = False
42
43
 
43
44
  @overload
44
45
  @classmethod
@@ -70,9 +71,9 @@ class HeuristicWeights(BaseModel):
70
71
 
71
72
  class HeuristicArgs(PackArgs):
72
73
  mode: Literal["heuristic"]
73
- min_filecount: int = 0
74
- max_mean_kib: int = 0
75
- min_total_mib: int = 0
74
+ min_filecount: int | None = None
75
+ max_mean_kib: int | None = None
76
+ min_total_mib: int | None = None
76
77
  weights: HeuristicWeights = Field(default_factory=HeuristicWeights)
77
78
  top_n: Annotated[int, Gt(0)] | None = None
78
79
 
@@ -84,7 +85,15 @@ class DepthArgs(PackArgs):
84
85
 
85
86
  class LeafArgs(PackArgs):
86
87
  mode: Literal["leaf"]
87
- skip_threshold: float
88
+ skip_threshold: float = 0.80
89
+
90
+
91
+ def _scrub_tarinfo(ti: tarfile.TarInfo) -> tarfile.TarInfo:
92
+ ti.uid = 0
93
+ ti.gid = 0
94
+ ti.uname = "root"
95
+ ti.gname = "root"
96
+ return ti
88
97
 
89
98
 
90
99
  def _dir_stats(path: Path) -> tuple[int, int]:
@@ -166,7 +175,7 @@ def compress_folder(folder_path: str) -> str | None:
166
175
  continue
167
176
  src = os.path.join(root, fname)
168
177
  rel = os.path.relpath(src, folder_path)
169
- tar.add(src, arcname=rel, recursive=False)
178
+ tar.add(src, arcname=rel, recursive=False, filter=_scrub_tarinfo)
170
179
 
171
180
  # ensure all buffers are flushed to disk
172
181
  fout.flush()
@@ -252,6 +261,13 @@ class Compressor(ABC):
252
261
  class HeuristicCompressor(Compressor):
253
262
  """
254
263
  Heuristic directory selector based directly on the fields in `PackArgs`.
264
+
265
+ For each weighting criterion, get a rank-ordered list of directories,
266
+ with higher-scoring, more likely to be packed items at the front.
267
+ Return the top_n candidates (or, if top_n is `None`, all that match the filtering criteria).
268
+
269
+ Orderings should be consistent across runs,
270
+ as each ranking is sorted by the relevant score and then the path name in the case of ties.
255
271
  """
256
272
 
257
273
  args_cls = HeuristicArgs
@@ -269,7 +285,7 @@ class HeuristicCompressor(Compressor):
269
285
  counts[d] = tot
270
286
  if tot >= limit and not any(os.path.join(d, x) in chosen for x in ds):
271
287
  chosen.add(d)
272
- return sorted(chosen, key=lambda p: counts[p], reverse=True)
288
+ return sorted(chosen, key=lambda p: (-counts[p], p))
273
289
 
274
290
  def _by_mean(self) -> list[str]:
275
291
  byte_limit = (self.args.max_mean_kib or 0) * 1024
@@ -283,13 +299,10 @@ class HeuristicCompressor(Compressor):
283
299
  total += b
284
300
  cnt += c
285
301
  stats[d] = (total, cnt)
286
- if (
287
- cnt
288
- and total / cnt <= byte_limit
289
- and not any(os.path.join(d, x) in chosen for x in ds)
290
- ):
302
+ allowed = not byte_limit or (total / cnt <= byte_limit)
303
+ if cnt and allowed and not any(os.path.join(d, x) in chosen for x in ds):
291
304
  chosen.add(d)
292
- return sorted(chosen, key=lambda p: stats[p][0] / stats[p][1])
305
+ return sorted(chosen, key=lambda p: (stats[p][0] / stats[p][1], p))
293
306
 
294
307
  def _by_total(self) -> list[str]:
295
308
  byte_limit = (self.args.min_total_mib or 0) * 1_048_576
@@ -302,7 +315,7 @@ class HeuristicCompressor(Compressor):
302
315
  stats[d] = total
303
316
  if total >= byte_limit and not any(os.path.join(d, x) in chosen for x in ds):
304
317
  chosen.add(d)
305
- return sorted(chosen, key=lambda p: stats[p], reverse=True)
318
+ return sorted(chosen, key=lambda p: (-stats[p], p))
306
319
 
307
320
  def _rank_sum(self, cand: dict[str, Sequence[str]]) -> list[str]:
308
321
  scores: dict[str, int] = {}
@@ -320,11 +333,22 @@ class HeuristicCompressor(Compressor):
320
333
  ]
321
334
 
322
335
  def select_targets(self) -> list[str]:
336
+ counts = self._by_filecount()
337
+ means = self._by_mean()
338
+ totals = self._by_total()
339
+
340
+ # each method excludes directories that are not allowed by the hard filters,
341
+ # so filter by the set intersection of directories
342
+ allowed = set(counts) & set(means) & set(totals)
343
+ counts = [x for x in counts if x in allowed]
344
+ means = [x for x in means if x in allowed]
345
+ totals = [x for x in totals if x in allowed]
346
+
323
347
  return self._rank_sum(
324
348
  {
325
- "file_count": self._by_filecount(),
326
- "mean_size": self._by_mean(),
327
- "total_size": self._by_total(),
349
+ "file_count": counts,
350
+ "mean_size": means,
351
+ "total_size": totals,
328
352
  }
329
353
  )
330
354
 
@@ -518,8 +542,11 @@ def run_packing_pipeline(args: PackArgs) -> None:
518
542
  base = manager.base
519
543
 
520
544
  # create manifest
521
- manifest_path = manager.write_sums()
522
- logger.info(f"Wrote checksum manifest → {manifest_path.relative_to(base)}")
545
+ if manager.args.skip_hash:
546
+ logger.info("Skipping checksum hash")
547
+ else:
548
+ manifest_path = manager.write_sums()
549
+ logger.info(f"Wrote checksum manifest → {manifest_path.relative_to(base)}")
523
550
 
524
551
  # build compressor and choose targets
525
552
  compressor = Compressor.from_args(args)
@@ -70,18 +70,14 @@ class FileVerificationFailed(VerificationFailed):
70
70
  def __init__(self, prev: SingleFileManifest, cur: SingleFileManifest) -> None:
71
71
  self.prev = prev
72
72
  self.cur = cur
73
- super().__init__(
74
- dedent(
75
- """\
73
+ super().__init__(dedent("""\
76
74
  file manifest failed to match file input:
77
75
  manifest:
78
76
  {prev}
79
77
 
80
78
  input:
81
79
  {cur}
82
- """
83
- ).format(prev=prev.model_dump_json(indent=2), cur=cur.model_dump_json(indent=2))
84
- )
80
+ """).format(prev=prev.model_dump_json(indent=2), cur=cur.model_dump_json(indent=2)))
85
81
 
86
82
 
87
83
  class SingleFileManifest(BaseModel):
@@ -266,18 +262,14 @@ class ExactDirectoryVerificationFailed(VerificationFailed):
266
262
  def __init__(self, prev: DirectoryManifest, cur: DirectoryManifest):
267
263
  self.prev = prev
268
264
  self.cur = cur
269
- super().__init__(
270
- dedent(
271
- """\
265
+ super().__init__(dedent("""\
272
266
  directory manifest failed to exactly match directory input:
273
267
  manifest:
274
268
  {prev}
275
269
 
276
270
  input:
277
271
  {cur}
278
- """
279
- ).format(prev=prev.model_dump_json(indent=2), cur=cur.model_dump_json(indent=2))
280
- )
272
+ """).format(prev=prev.model_dump_json(indent=2), cur=cur.model_dump_json(indent=2)))
281
273
 
282
274
 
283
275
  class InexactDirectoryVerificationFailed(VerificationFailed):
@@ -286,16 +278,16 @@ class InexactDirectoryVerificationFailed(VerificationFailed):
286
278
  self.entry_path = entry_path
287
279
  self.reason = reason
288
280
  super().__init__(
289
- dedent(
290
- """\
281
+ dedent("""\
291
282
  directory manifest failed to match directory input:
292
283
  manifest:
293
284
  {prev}
294
285
 
295
286
  failed entry was: {entry_path}
296
287
  reason: {reason}
297
- """
298
- ).format(prev=prev.model_dump_json(indent=2), entry_path=entry_path, reason=reason)
288
+ """).format(
289
+ prev=prev.model_dump_json(indent=2), entry_path=entry_path, reason=reason
290
+ )
299
291
  )
300
292
 
301
293
 
@@ -207,8 +207,6 @@ def find_optimal_piece_size(
207
207
  path = Path(path)
208
208
  paths = [path] if path.is_file() else list(iter_files(path))
209
209
 
210
- paths = cast(list[Path], paths)
211
-
212
210
  if sizes is None:
213
211
  sizes = [p.stat().st_size for p in paths]
214
212
 
@@ -14,7 +14,7 @@ from sciop_cli.models.sciop import TorrentFile, Upload
14
14
 
15
15
  if TYPE_CHECKING:
16
16
  from sciop.models import Account
17
- from torf import Torrent
17
+ from torrent_models import Torrent
18
18
 
19
19
 
20
20
  @pytest.mark.asyncio
@@ -156,7 +156,7 @@ async def test_upload_torrent(as_admin: "Account", run_server_sync, torrent, tmp
156
156
 
157
157
  adapter = TypeAdapter(TorrentFile)
158
158
  adapter.validate_python(res)
159
- assert res["v1_infohash"] == torrent.infohash
159
+ assert res["v1_infohash"] == torrent.v1_infohash
160
160
  assert res["file_name"] == "test.torrent"
161
161
 
162
162
 
@@ -182,7 +182,7 @@ async def test_create_upload(
182
182
 
183
183
  if mode == "infohash":
184
184
  upload_torrent(path)
185
- ul = create_upload(dataset=slug, infohash=torrent.infohash)
185
+ ul = create_upload(dataset=slug, infohash=torrent.v1_infohash)
186
186
  elif mode == "path":
187
187
  ul = create_upload(dataset=slug, torrent_path=path)
188
188
  else:
@@ -1,7 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import json
4
- import os
5
4
  from pathlib import Path
6
5
  from typing import Literal
7
6
 
@@ -178,36 +177,42 @@ def _heuristic_fixture(tmp_path: Path) -> Path:
178
177
  large_mean/ (3 x 2 KiB files) - mean 2 KiB
179
178
  low_total/ (10 x 50 KiB ≈ 0.5 MiB) - under 1 MiB
180
179
  high_total/ (2 x 1 MiB ≈ 2 MiB) - over 1 MiB
180
+
181
+ Each of the cases should have one match that ties it to test ordering:
182
+ e.g. for n file ranking, there should be two dirs with the same number of files, etc.
181
183
  """
182
184
  root = tmp_path / "hsel"
183
185
  spec = {
184
186
  "few": (1, 1),
185
- "many": (3, 1),
187
+ "many": (20, 1),
186
188
  "small_mean": (3, 512),
187
189
  "large_mean": (3, 2048),
188
190
  "low_total": (10, 50 * 1024),
189
191
  "high_total": (2, 1 * 1024 * 1024),
192
+ "high_total_2": (4, 512 * 1024),
190
193
  }
191
194
  for dirname, (n, size) in spec.items():
192
195
  d = root / dirname
193
196
  d.mkdir(parents=True, exist_ok=True)
194
197
  for i in range(n):
195
- (d / f"f{i}.bin").write_bytes(os.urandom(size))
198
+ (d / f"f{i}.bin").write_bytes(bytes(size))
196
199
  return root
197
200
 
198
201
 
199
202
  HEURISTIC_RULES = [
200
203
  # rule, kwargs, must_have, must_not
201
- ("min_filecount", {"min_filecount": 2}, {"many"}, {"few"}),
202
- ("max_mean_kib", {"max_mean_kib": 1}, {"small_mean"}, {"large_mean"}),
203
- ("min_total_mib", {"min_total_mib": 1}, {"high_total"}, {"low_total"}),
204
+ (
205
+ "min_filecount",
206
+ {"min_filecount": 3},
207
+ ["many", "low_total", "high_total_2", "large_mean", "small_mean"],
208
+ ),
209
+ ("max_mean_kib", {"max_mean_kib": 1}, ["few", "many", "small_mean"]),
210
+ ("min_total_mib", {"min_total_mib": 1}, ["high_total", "high_total_2"]),
204
211
  ]
205
212
 
206
213
 
207
- @pytest.mark.parametrize("rule, kwargs, must_have, must_not", HEURISTIC_RULES)
208
- def test_heuristic_rules(
209
- tmp_path: Path, rule: str, kwargs: dict, must_have: set[str], must_not: set[str]
210
- ):
214
+ @pytest.mark.parametrize("rule, kwargs, expected", HEURISTIC_RULES)
215
+ def test_heuristic_rules(tmp_path: Path, rule: str, kwargs: dict, expected: list[str]):
211
216
 
212
217
  base = _heuristic_fixture(tmp_path)
213
218
  weights = {"file_count": 1, "mean_size": 1, "total_size": 1}
@@ -221,18 +226,18 @@ def test_heuristic_rules(
221
226
  weights["total_size"] = 1
222
227
  weights["file_count"] = weights["mean_size"] = 0
223
228
 
224
- _rule_top_n = {"min_filecount": 2, "max_mean_kib": 3, "min_total_mib": 1}
225
- kw = dict(kwargs, top_n=_rule_top_n[rule])
226
-
227
229
  args = PackArgs.from_kwargs(
228
230
  mode="heuristic",
229
231
  base_dir=str(base),
230
- **kw,
232
+ **kwargs,
231
233
  weights=weights,
232
234
  )
233
- selected = set(Compressor.from_args(args).select_targets())
234
- assert must_have <= selected
235
- assert selected.isdisjoint(must_not)
235
+
236
+ # repeat several times to ensure consistency
237
+ repeat_n = 20
238
+ for _ in range(repeat_n):
239
+ selected = Compressor.from_args(args).select_targets()
240
+ assert selected == expected
236
241
 
237
242
 
238
243
  def test_cli_smoke(tmp_path: Path):
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes