sciop-cli 0.2.3__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/PKG-INFO +2 -1
  2. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/pyproject.toml +3 -5
  3. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/cli/client.py +2 -1
  4. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/cli/main.py +3 -0
  5. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/cli/manifest.py +2 -1
  6. sciop_cli-0.3.0/src/sciop_cli/cli/pack.py +100 -0
  7. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/cli/torrent.py +0 -12
  8. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/clients/base.py +2 -1
  9. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/const.py +3 -0
  10. sciop_cli-0.3.0/src/sciop_cli/models/pack.py +95 -0
  11. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/models/sciop.py +3 -3
  12. sciop_cli-0.3.0/src/sciop_cli/pack.py +550 -0
  13. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/schema.py +3 -2
  14. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/torrent.py +3 -2
  15. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/types.py +14 -1
  16. sciop_cli-0.3.0/tests/test_pack.py +259 -0
  17. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/test_torrent.py +5 -5
  18. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/LICENSE +0 -0
  19. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/README.md +0 -0
  20. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/__init__.py +0 -0
  21. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/__main__.py +0 -0
  22. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/api.py +0 -0
  23. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/cli/__init__.py +0 -0
  24. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/cli/api.py +0 -0
  25. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/clients/__init__.py +0 -0
  26. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/clients/qbt.py +0 -0
  27. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/config.py +0 -0
  28. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/data.py +0 -0
  29. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/exceptions.py +0 -0
  30. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/models/__init__.py +0 -0
  31. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/__init__.py +0 -0
  32. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/conftest.py +0 -0
  33. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/data/basic/part_1/subdir/part_1 +0 -0
  34. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/data/basic/part_2/subdir/part_2 +0 -0
  35. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/data/basic_hybrid.torrent +0 -0
  36. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/data/basic_v2.torrent +0 -0
  37. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/data/distributions/dlc_chrysler_ver02.json +0 -0
  38. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/data/distributions/nbu_arwen_ver01.json +0 -0
  39. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/fixtures/__init__.py +0 -0
  40. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/fixtures/base.py +0 -0
  41. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/fixtures/fabricators.py +0 -0
  42. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/test_api.py +0 -0
  43. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/test_cli/__init__.py +0 -0
  44. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/test_cli/test_api.py +0 -0
  45. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/test_cli/test_torrent.py +0 -0
  46. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/test_client/__init__.py +0 -0
  47. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/test_client/test_client_config.py +0 -0
  48. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/test_config.py +0 -0
  49. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/test_data.py +0 -0
  50. {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/test_temp.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: sciop-cli
3
- Version: 0.2.3
3
+ Version: 0.3.0
4
4
  Summary: A CLI for sciop
5
5
  Author-Email: cosmicexplorer <dmcC2@hypnicjerk.ai>, sneakers-the-rat <sneakers-the-rat@protonmail.com>
6
6
  License: EUPL-1.2
@@ -20,6 +20,7 @@ Requires-Dist: humanize>=4.12.2
20
20
  Requires-Dist: pydantic-settings>=2.9.1
21
21
  Requires-Dist: keyring>=25.6.0
22
22
  Requires-Dist: pyyaml>=6.0.2
23
+ Requires-Dist: zstandard>=0.22
23
24
  Requires-Dist: qbittorrent-api>=2025.5.0
24
25
  Requires-Dist: typing-extensions>=4.10.0; python_version < "3.12"
25
26
  Provides-Extra: mypy
@@ -20,12 +20,13 @@ dependencies = [
20
20
  "pydantic-settings>=2.9.1",
21
21
  "keyring>=25.6.0",
22
22
  "pyyaml>=6.0.2",
23
+ "zstandard>=0.22",
23
24
  "qbittorrent-api>=2025.5.0",
24
25
  "typing-extensions>=4.10.0;python_version<\"3.12\"",
25
26
  ]
26
27
  requires-python = ">=3.11"
27
28
  readme = "README.md"
28
- version = "0.2.3"
29
+ version = "0.3.0"
29
30
 
30
31
  [project.license]
31
32
  text = "EUPL-1.2"
@@ -124,12 +125,9 @@ ignore = [
124
125
  "ANN401",
125
126
  "ANN204",
126
127
  "B905",
127
- "UP006",
128
- "UP035",
128
+ "D107",
129
129
  "UP007",
130
130
  "UP038",
131
- "D107",
132
- "E712",
133
131
  ]
134
132
  fixable = [
135
133
  "ALL",
@@ -1,5 +1,6 @@
1
+ from collections.abc import Callable
1
2
  from pathlib import Path
2
- from typing import Any, Callable
3
+ from typing import Any
3
4
 
4
5
  import click
5
6
  from pydantic import TypeAdapter
@@ -5,6 +5,7 @@ import click
5
5
  from sciop_cli.cli.api import claims, login, upload
6
6
  from sciop_cli.cli.client import client
7
7
  from sciop_cli.cli.manifest import manifest
8
+ from sciop_cli.cli.pack import pack, unpack
8
9
  from sciop_cli.cli.torrent import torrent
9
10
 
10
11
 
@@ -25,6 +26,8 @@ def cli(debug: bool) -> None:
25
26
  cli.add_command(login)
26
27
  cli.add_command(claims)
27
28
  cli.add_command(upload)
29
+ cli.add_command(pack)
30
+ cli.add_command(unpack)
28
31
 
29
32
  # command groups
30
33
  cli.add_command(client)
@@ -2,8 +2,9 @@ from __future__ import annotations
2
2
 
3
3
  import logging
4
4
  import warnings
5
+ from collections.abc import Iterable
5
6
  from pathlib import Path
6
- from typing import BinaryIO, Iterable, TextIO
7
+ from typing import BinaryIO, TextIO
7
8
 
8
9
  import click
9
10
  from click_option_group import RequiredMutuallyExclusiveOptionGroup, optgroup
@@ -0,0 +1,100 @@
1
+ import click
2
+
3
+ from sciop_cli.pack import ALGO_MAP, PACK_MODES, PackArgs, PackManager, run_packing_pipeline
4
+
5
+
6
+ @click.command("pack")
7
+ @click.option(
8
+ "--path",
9
+ "-p",
10
+ required=True,
11
+ type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True),
12
+ help="Directory to pack (walked recursively).",
13
+ )
14
+ @click.option(
15
+ "-m",
16
+ "--mode",
17
+ type=click.Choice(["heuristic", "leaf", "depth"], case_sensitive=False),
18
+ default="heuristic",
19
+ show_default=True,
20
+ help=(
21
+ "Directory-selection strategy:\n"
22
+ " - heuristic → score each sub-directory using --min-filecount, "
23
+ "--max-mean-kib, and --min-total-mib (optionally --top-n or --weights); "
24
+ "compress the top matches.\n"
25
+ " - leaf → inspect only leaf dirs (contain files but no subfolders); "
26
+ "skip a leaf if >= --skip-threshold of its files are already compressed.\n"
27
+ " - depth → compress those dirs exactly --depth levels below base "
28
+ "(depth 1 = children, 2 = grandchildren, etc.)."
29
+ ),
30
+ )
31
+ @click.option("--depth", type=int, default=None, help="Depth level for depth mode.")
32
+ @click.option(
33
+ "--skip-threshold",
34
+ type=float,
35
+ default=None,
36
+ help="Leaf mode: fraction of present compressed files in a leaf that causes skipping.",
37
+ )
38
+ @click.option("--min-filecount", type=int, default=None, help="Heuristic: minimum # files.")
39
+ @click.option("--max-mean-kib", type=int, default=None, help="Heuristic: max mean file size (KiB).")
40
+ @click.option("--min-total-mib", type=int, default=None, help="Heuristic: min total size (MiB).")
41
+ @click.option(
42
+ "--top-n",
43
+ type=int,
44
+ default=None,
45
+ help="Heuristic: cap number of directories to compress.",
46
+ )
47
+ @click.option(
48
+ "--hash-algo",
49
+ "-a",
50
+ type=click.Choice(list(ALGO_MAP.keys()), case_sensitive=False),
51
+ default="sha512",
52
+ show_default=True,
53
+ help="Which checksum algorithm to write into manifest-<algo>.txt",
54
+ )
55
+ def pack(
56
+ path: str,
57
+ mode: PACK_MODES = "heuristic",
58
+ depth: int | None = None,
59
+ skip_threshold: float | None = None,
60
+ min_filecount: int | None = None,
61
+ max_mean_kib: int | None = None,
62
+ min_total_mib: int | None = None,
63
+ top_n: int | None = None,
64
+ hash_algo: str = "sha512",
65
+ ) -> None:
66
+ args = PackArgs.from_kwargs(
67
+ mode=mode,
68
+ base_dir=path,
69
+ depth=depth,
70
+ skip_threshold=skip_threshold,
71
+ min_filecount=min_filecount,
72
+ max_mean_kib=max_mean_kib,
73
+ min_total_mib=min_total_mib,
74
+ top_n=top_n,
75
+ hash_algo=hash_algo,
76
+ )
77
+
78
+ try:
79
+ run_packing_pipeline(args)
80
+ except Exception as e:
81
+ click.echo(click.style(f"Packing failed: {e}", fg="red"), err=True)
82
+ raise SystemExit(1) from e
83
+
84
+
85
+ @click.command("unpack")
86
+ @click.option(
87
+ "-p",
88
+ "--base-dir",
89
+ required=True,
90
+ type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True),
91
+ help="Directory that contains <base>.packmap.json and the .tar.zst archives.",
92
+ )
93
+ def unpack(base_dir: str) -> None:
94
+ try:
95
+ pm = PackManager(PackArgs.from_kwargs(mode="leaf", base_dir=base_dir))
96
+ restored = pm.restore_directory()
97
+ click.echo(f"Restored {len(restored)} archive(s).")
98
+ except Exception as e:
99
+ click.echo(click.style(f"Restore failed: {e}", fg="red"), err=True)
100
+ raise SystemExit(1) from e
@@ -32,18 +32,6 @@ def torrent() -> None:
32
32
  """
33
33
 
34
34
 
35
- @torrent.command()
36
- def pack() -> None:
37
- """
38
- Pack a directory to prepare it for torrent creation
39
-
40
- - Generate a manifest for the directory
41
- - Archive small files
42
- - Emit a .packmap.json description of the packing operation
43
- """
44
- raise NotImplementedError()
45
-
46
-
47
35
  @torrent.command()
48
36
  @click.option(
49
37
  "-p",
@@ -1,7 +1,8 @@
1
1
  from abc import abstractmethod
2
+ from collections.abc import Sequence
2
3
  from dataclasses import dataclass
3
4
  from pathlib import Path
4
- from typing import Any, ClassVar, Sequence
5
+ from typing import Any, ClassVar
5
6
 
6
7
  from pydantic import BaseModel
7
8
 
@@ -1,3 +1,5 @@
1
+ from __future__ import annotations
2
+
1
3
  from importlib.metadata import version
2
4
 
3
5
  from sciop_cli.types import Size
@@ -12,3 +14,4 @@ EXCLUDE_FILES = (".DS_Store", "Thumbs.db")
12
14
  BLOCK_SIZE = 16 * KiB
13
15
  PIECE_SIZES = tuple(BLOCK_SIZE * (2**exp) for exp in range(14))
14
16
  """16KiB to 128MiB, powers of two * block size."""
17
+ CHECKSUM_CHUNK = 4 << 20 # 4 MiB
@@ -0,0 +1,95 @@
1
+ import hashlib
2
+ import uuid
3
+ from datetime import UTC, datetime
4
+ from importlib.metadata import version
5
+ from pathlib import Path
6
+ from typing import Literal, Self
7
+
8
+ from pydantic import BaseModel, Field, model_validator
9
+ from tqdm import tqdm
10
+
11
+ from sciop_cli.const import CHECKSUM_CHUNK, DEFAULT_TORRENT_CREATOR
12
+ from sciop_cli.types import PosixPath, UTCDateTime
13
+
14
+ HASH_ALGOS = Literal["sha512", "sha256"]
15
+
16
+
17
+ class PackedDirectory(BaseModel):
18
+ """A single packed directory within a packmap"""
19
+
20
+ src_dir: PosixPath
21
+ archive: PosixPath
22
+ pre_archive_bytes: int
23
+ post_archive_bytes: int
24
+ file_count: int
25
+
26
+
27
+ class PackMap(BaseModel):
28
+ """
29
+ Metadata file containing information about files before being compressed/packed,
30
+ as well as the parameterization of the packing.
31
+ """
32
+
33
+ packmap_version: int = 1
34
+ sciop_cli_version: str = Field(default_factory=lambda: version("sciop-cli"))
35
+ run_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
36
+ timestamp: UTCDateTime = Field(default_factory=lambda: datetime.now(UTC))
37
+ packed_by: str = DEFAULT_TORRENT_CREATOR
38
+ archives: list[PackedDirectory] = Field(default_factory=list)
39
+ total_size_before: int
40
+ total_size_after: int
41
+ total_file_count: int
42
+
43
+
44
+ class FileHash(BaseModel):
45
+ algo: HASH_ALGOS
46
+ path: PosixPath
47
+ digest: str
48
+ """hexdigest of hash"""
49
+
50
+ @classmethod
51
+ def from_path(
52
+ cls,
53
+ path: Path,
54
+ algo: HASH_ALGOS,
55
+ relative_to: Path,
56
+ pbar: tqdm | None = None,
57
+ chunk_size: int = CHECKSUM_CHUNK,
58
+ ) -> "FileHash":
59
+ h = hashlib.new(algo)
60
+ with path.open("rb") as f:
61
+ while chunk := f.read(chunk_size):
62
+ h.update(chunk)
63
+ if pbar:
64
+ pbar.update(len(chunk))
65
+ return FileHash(algo=algo, path=path.relative_to(relative_to), digest=h.hexdigest())
66
+
67
+
68
+ class HashManifest(BaseModel):
69
+ """Manifest of hashes and relative paths"""
70
+
71
+ algo: HASH_ALGOS
72
+ hashes: list[FileHash]
73
+
74
+ @model_validator(mode="after")
75
+ def matching_algos(self) -> Self:
76
+ """all file hashes have the same algo as the manifest"""
77
+ assert all(
78
+ [hash.algo == self.algo for hash in self.hashes]
79
+ ), "All hashes must be computed with the manifest's hashing algo"
80
+ return self
81
+
82
+ def write(self, path: Path) -> Path:
83
+ """Write to a manifest-{algo}.txt file.
84
+
85
+ Args:
86
+ path (Path): The *directory* to write the manifest into
87
+ (filename is fixed according to the algo).
88
+ """
89
+ # invoke serialization
90
+ dumped = self.model_dump()
91
+ manifest_lines = [" ".join([h["digest"], h["path"]]) for h in dumped["hashes"]]
92
+ manifest = "\n".join(manifest_lines)
93
+ output = path / f"manifest-{dumped['algo']}.txt"
94
+ output.write_text(manifest)
95
+ return output
@@ -5,12 +5,12 @@ until we can split them off into a separate package and single-source them.
5
5
 
6
6
  import sys
7
7
  from datetime import datetime
8
- from typing import Literal
8
+ from typing import Literal, NotRequired
9
9
 
10
10
  if sys.version_info >= (3, 12):
11
- from typing import NotRequired, TypedDict
11
+ from typing import TypedDict
12
12
  else:
13
- from typing_extensions import NotRequired, TypedDict
13
+ from typing_extensions import TypedDict
14
14
 
15
15
 
16
16
  class Token(TypedDict):