sciop-cli 0.2.3__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/PKG-INFO +2 -1
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/pyproject.toml +3 -5
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/cli/client.py +2 -1
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/cli/main.py +3 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/cli/manifest.py +2 -1
- sciop_cli-0.3.0/src/sciop_cli/cli/pack.py +100 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/cli/torrent.py +0 -12
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/clients/base.py +2 -1
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/const.py +3 -0
- sciop_cli-0.3.0/src/sciop_cli/models/pack.py +95 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/models/sciop.py +3 -3
- sciop_cli-0.3.0/src/sciop_cli/pack.py +550 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/schema.py +3 -2
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/torrent.py +3 -2
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/types.py +14 -1
- sciop_cli-0.3.0/tests/test_pack.py +259 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/test_torrent.py +5 -5
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/LICENSE +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/README.md +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/__init__.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/__main__.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/api.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/cli/__init__.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/cli/api.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/clients/__init__.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/clients/qbt.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/config.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/data.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/exceptions.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/src/sciop_cli/models/__init__.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/__init__.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/conftest.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/data/basic/part_1/subdir/part_1 +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/data/basic/part_2/subdir/part_2 +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/data/basic_hybrid.torrent +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/data/basic_v2.torrent +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/data/distributions/dlc_chrysler_ver02.json +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/data/distributions/nbu_arwen_ver01.json +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/fixtures/__init__.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/fixtures/base.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/fixtures/fabricators.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/test_api.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/test_cli/__init__.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/test_cli/test_api.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/test_cli/test_torrent.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/test_client/__init__.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/test_client/test_client_config.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/test_config.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/test_data.py +0 -0
- {sciop_cli-0.2.3 → sciop_cli-0.3.0}/tests/test_temp.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: sciop-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: A CLI for sciop
|
|
5
5
|
Author-Email: cosmicexplorer <dmcC2@hypnicjerk.ai>, sneakers-the-rat <sneakers-the-rat@protonmail.com>
|
|
6
6
|
License: EUPL-1.2
|
|
@@ -20,6 +20,7 @@ Requires-Dist: humanize>=4.12.2
|
|
|
20
20
|
Requires-Dist: pydantic-settings>=2.9.1
|
|
21
21
|
Requires-Dist: keyring>=25.6.0
|
|
22
22
|
Requires-Dist: pyyaml>=6.0.2
|
|
23
|
+
Requires-Dist: zstandard>=0.22
|
|
23
24
|
Requires-Dist: qbittorrent-api>=2025.5.0
|
|
24
25
|
Requires-Dist: typing-extensions>=4.10.0; python_version < "3.12"
|
|
25
26
|
Provides-Extra: mypy
|
|
@@ -20,12 +20,13 @@ dependencies = [
|
|
|
20
20
|
"pydantic-settings>=2.9.1",
|
|
21
21
|
"keyring>=25.6.0",
|
|
22
22
|
"pyyaml>=6.0.2",
|
|
23
|
+
"zstandard>=0.22",
|
|
23
24
|
"qbittorrent-api>=2025.5.0",
|
|
24
25
|
"typing-extensions>=4.10.0;python_version<\"3.12\"",
|
|
25
26
|
]
|
|
26
27
|
requires-python = ">=3.11"
|
|
27
28
|
readme = "README.md"
|
|
28
|
-
version = "0.
|
|
29
|
+
version = "0.3.0"
|
|
29
30
|
|
|
30
31
|
[project.license]
|
|
31
32
|
text = "EUPL-1.2"
|
|
@@ -124,12 +125,9 @@ ignore = [
|
|
|
124
125
|
"ANN401",
|
|
125
126
|
"ANN204",
|
|
126
127
|
"B905",
|
|
127
|
-
"
|
|
128
|
-
"UP035",
|
|
128
|
+
"D107",
|
|
129
129
|
"UP007",
|
|
130
130
|
"UP038",
|
|
131
|
-
"D107",
|
|
132
|
-
"E712",
|
|
133
131
|
]
|
|
134
132
|
fixable = [
|
|
135
133
|
"ALL",
|
|
@@ -5,6 +5,7 @@ import click
|
|
|
5
5
|
from sciop_cli.cli.api import claims, login, upload
|
|
6
6
|
from sciop_cli.cli.client import client
|
|
7
7
|
from sciop_cli.cli.manifest import manifest
|
|
8
|
+
from sciop_cli.cli.pack import pack, unpack
|
|
8
9
|
from sciop_cli.cli.torrent import torrent
|
|
9
10
|
|
|
10
11
|
|
|
@@ -25,6 +26,8 @@ def cli(debug: bool) -> None:
|
|
|
25
26
|
cli.add_command(login)
|
|
26
27
|
cli.add_command(claims)
|
|
27
28
|
cli.add_command(upload)
|
|
29
|
+
cli.add_command(pack)
|
|
30
|
+
cli.add_command(unpack)
|
|
28
31
|
|
|
29
32
|
# command groups
|
|
30
33
|
cli.add_command(client)
|
|
@@ -2,8 +2,9 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
4
|
import warnings
|
|
5
|
+
from collections.abc import Iterable
|
|
5
6
|
from pathlib import Path
|
|
6
|
-
from typing import BinaryIO,
|
|
7
|
+
from typing import BinaryIO, TextIO
|
|
7
8
|
|
|
8
9
|
import click
|
|
9
10
|
from click_option_group import RequiredMutuallyExclusiveOptionGroup, optgroup
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import click
|
|
2
|
+
|
|
3
|
+
from sciop_cli.pack import ALGO_MAP, PACK_MODES, PackArgs, PackManager, run_packing_pipeline
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@click.command("pack")
|
|
7
|
+
@click.option(
|
|
8
|
+
"--path",
|
|
9
|
+
"-p",
|
|
10
|
+
required=True,
|
|
11
|
+
type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True),
|
|
12
|
+
help="Directory to pack (walked recursively).",
|
|
13
|
+
)
|
|
14
|
+
@click.option(
|
|
15
|
+
"-m",
|
|
16
|
+
"--mode",
|
|
17
|
+
type=click.Choice(["heuristic", "leaf", "depth"], case_sensitive=False),
|
|
18
|
+
default="heuristic",
|
|
19
|
+
show_default=True,
|
|
20
|
+
help=(
|
|
21
|
+
"Directory-selection strategy:\n"
|
|
22
|
+
" - heuristic → score each sub-directory using --min-filecount, "
|
|
23
|
+
"--max-mean-kib, and --min-total-mib (optionally --top-n or --weights); "
|
|
24
|
+
"compress the top matches.\n"
|
|
25
|
+
" - leaf → inspect only leaf dirs (contain files but no subfolders); "
|
|
26
|
+
"skip a leaf if >= --skip-threshold of its files are already compressed.\n"
|
|
27
|
+
" - depth → compress those dirs exactly --depth levels below base "
|
|
28
|
+
"(depth 1 = children, 2 = grandchildren, etc.)."
|
|
29
|
+
),
|
|
30
|
+
)
|
|
31
|
+
@click.option("--depth", type=int, default=None, help="Depth level for depth mode.")
|
|
32
|
+
@click.option(
|
|
33
|
+
"--skip-threshold",
|
|
34
|
+
type=float,
|
|
35
|
+
default=None,
|
|
36
|
+
help="Leaf mode: fraction of present compressed files in a leaf that causes skipping.",
|
|
37
|
+
)
|
|
38
|
+
@click.option("--min-filecount", type=int, default=None, help="Heuristic: minimum # files.")
|
|
39
|
+
@click.option("--max-mean-kib", type=int, default=None, help="Heuristic: max mean file size (KiB).")
|
|
40
|
+
@click.option("--min-total-mib", type=int, default=None, help="Heuristic: min total size (MiB).")
|
|
41
|
+
@click.option(
|
|
42
|
+
"--top-n",
|
|
43
|
+
type=int,
|
|
44
|
+
default=None,
|
|
45
|
+
help="Heuristic: cap number of directories to compress.",
|
|
46
|
+
)
|
|
47
|
+
@click.option(
|
|
48
|
+
"--hash-algo",
|
|
49
|
+
"-a",
|
|
50
|
+
type=click.Choice(list(ALGO_MAP.keys()), case_sensitive=False),
|
|
51
|
+
default="sha512",
|
|
52
|
+
show_default=True,
|
|
53
|
+
help="Which checksum algorithm to write into manifest-<algo>.txt",
|
|
54
|
+
)
|
|
55
|
+
def pack(
|
|
56
|
+
path: str,
|
|
57
|
+
mode: PACK_MODES = "heuristic",
|
|
58
|
+
depth: int | None = None,
|
|
59
|
+
skip_threshold: float | None = None,
|
|
60
|
+
min_filecount: int | None = None,
|
|
61
|
+
max_mean_kib: int | None = None,
|
|
62
|
+
min_total_mib: int | None = None,
|
|
63
|
+
top_n: int | None = None,
|
|
64
|
+
hash_algo: str = "sha512",
|
|
65
|
+
) -> None:
|
|
66
|
+
args = PackArgs.from_kwargs(
|
|
67
|
+
mode=mode,
|
|
68
|
+
base_dir=path,
|
|
69
|
+
depth=depth,
|
|
70
|
+
skip_threshold=skip_threshold,
|
|
71
|
+
min_filecount=min_filecount,
|
|
72
|
+
max_mean_kib=max_mean_kib,
|
|
73
|
+
min_total_mib=min_total_mib,
|
|
74
|
+
top_n=top_n,
|
|
75
|
+
hash_algo=hash_algo,
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
try:
|
|
79
|
+
run_packing_pipeline(args)
|
|
80
|
+
except Exception as e:
|
|
81
|
+
click.echo(click.style(f"Packing failed: {e}", fg="red"), err=True)
|
|
82
|
+
raise SystemExit(1) from e
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@click.command("unpack")
|
|
86
|
+
@click.option(
|
|
87
|
+
"-p",
|
|
88
|
+
"--base-dir",
|
|
89
|
+
required=True,
|
|
90
|
+
type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True),
|
|
91
|
+
help="Directory that contains <base>.packmap.json and the .tar.zst archives.",
|
|
92
|
+
)
|
|
93
|
+
def unpack(base_dir: str) -> None:
|
|
94
|
+
try:
|
|
95
|
+
pm = PackManager(PackArgs.from_kwargs(mode="leaf", base_dir=base_dir))
|
|
96
|
+
restored = pm.restore_directory()
|
|
97
|
+
click.echo(f"Restored {len(restored)} archive(s).")
|
|
98
|
+
except Exception as e:
|
|
99
|
+
click.echo(click.style(f"Restore failed: {e}", fg="red"), err=True)
|
|
100
|
+
raise SystemExit(1) from e
|
|
@@ -32,18 +32,6 @@ def torrent() -> None:
|
|
|
32
32
|
"""
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
@torrent.command()
|
|
36
|
-
def pack() -> None:
|
|
37
|
-
"""
|
|
38
|
-
Pack a directory to prepare it for torrent creation
|
|
39
|
-
|
|
40
|
-
- Generate a manifest for the directory
|
|
41
|
-
- Archive small files
|
|
42
|
-
- Emit a .packmap.json description of the packing operation
|
|
43
|
-
"""
|
|
44
|
-
raise NotImplementedError()
|
|
45
|
-
|
|
46
|
-
|
|
47
35
|
@torrent.command()
|
|
48
36
|
@click.option(
|
|
49
37
|
"-p",
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
from importlib.metadata import version
|
|
2
4
|
|
|
3
5
|
from sciop_cli.types import Size
|
|
@@ -12,3 +14,4 @@ EXCLUDE_FILES = (".DS_Store", "Thumbs.db")
|
|
|
12
14
|
BLOCK_SIZE = 16 * KiB
|
|
13
15
|
PIECE_SIZES = tuple(BLOCK_SIZE * (2**exp) for exp in range(14))
|
|
14
16
|
"""16KiB to 128MiB, powers of two * block size."""
|
|
17
|
+
CHECKSUM_CHUNK = 4 << 20 # 4 MiB
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import hashlib
|
|
2
|
+
import uuid
|
|
3
|
+
from datetime import UTC, datetime
|
|
4
|
+
from importlib.metadata import version
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Literal, Self
|
|
7
|
+
|
|
8
|
+
from pydantic import BaseModel, Field, model_validator
|
|
9
|
+
from tqdm import tqdm
|
|
10
|
+
|
|
11
|
+
from sciop_cli.const import CHECKSUM_CHUNK, DEFAULT_TORRENT_CREATOR
|
|
12
|
+
from sciop_cli.types import PosixPath, UTCDateTime
|
|
13
|
+
|
|
14
|
+
HASH_ALGOS = Literal["sha512", "sha256"]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class PackedDirectory(BaseModel):
|
|
18
|
+
"""A single packed directory within a packmap"""
|
|
19
|
+
|
|
20
|
+
src_dir: PosixPath
|
|
21
|
+
archive: PosixPath
|
|
22
|
+
pre_archive_bytes: int
|
|
23
|
+
post_archive_bytes: int
|
|
24
|
+
file_count: int
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class PackMap(BaseModel):
|
|
28
|
+
"""
|
|
29
|
+
Metadata file containing information about files before being compressed/packed,
|
|
30
|
+
as well as the parameterization of the packing.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
packmap_version: int = 1
|
|
34
|
+
sciop_cli_version: str = Field(default_factory=lambda: version("sciop-cli"))
|
|
35
|
+
run_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
|
|
36
|
+
timestamp: UTCDateTime = Field(default_factory=lambda: datetime.now(UTC))
|
|
37
|
+
packed_by: str = DEFAULT_TORRENT_CREATOR
|
|
38
|
+
archives: list[PackedDirectory] = Field(default_factory=list)
|
|
39
|
+
total_size_before: int
|
|
40
|
+
total_size_after: int
|
|
41
|
+
total_file_count: int
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class FileHash(BaseModel):
|
|
45
|
+
algo: HASH_ALGOS
|
|
46
|
+
path: PosixPath
|
|
47
|
+
digest: str
|
|
48
|
+
"""hexdigest of hash"""
|
|
49
|
+
|
|
50
|
+
@classmethod
|
|
51
|
+
def from_path(
|
|
52
|
+
cls,
|
|
53
|
+
path: Path,
|
|
54
|
+
algo: HASH_ALGOS,
|
|
55
|
+
relative_to: Path,
|
|
56
|
+
pbar: tqdm | None = None,
|
|
57
|
+
chunk_size: int = CHECKSUM_CHUNK,
|
|
58
|
+
) -> "FileHash":
|
|
59
|
+
h = hashlib.new(algo)
|
|
60
|
+
with path.open("rb") as f:
|
|
61
|
+
while chunk := f.read(chunk_size):
|
|
62
|
+
h.update(chunk)
|
|
63
|
+
if pbar:
|
|
64
|
+
pbar.update(len(chunk))
|
|
65
|
+
return FileHash(algo=algo, path=path.relative_to(relative_to), digest=h.hexdigest())
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class HashManifest(BaseModel):
|
|
69
|
+
"""Manifest of hashes and relative paths"""
|
|
70
|
+
|
|
71
|
+
algo: HASH_ALGOS
|
|
72
|
+
hashes: list[FileHash]
|
|
73
|
+
|
|
74
|
+
@model_validator(mode="after")
|
|
75
|
+
def matching_algos(self) -> Self:
|
|
76
|
+
"""all file hashes have the same algo as the manifest"""
|
|
77
|
+
assert all(
|
|
78
|
+
[hash.algo == self.algo for hash in self.hashes]
|
|
79
|
+
), "All hashes must be computed with the manifest's hashing algo"
|
|
80
|
+
return self
|
|
81
|
+
|
|
82
|
+
def write(self, path: Path) -> Path:
|
|
83
|
+
"""Write to a manifest-{algo}.txt file.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
path (Path): The *directory* to write the manifest into
|
|
87
|
+
(filename is fixed according to the algo).
|
|
88
|
+
"""
|
|
89
|
+
# invoke serialization
|
|
90
|
+
dumped = self.model_dump()
|
|
91
|
+
manifest_lines = [" ".join([h["digest"], h["path"]]) for h in dumped["hashes"]]
|
|
92
|
+
manifest = "\n".join(manifest_lines)
|
|
93
|
+
output = path / f"manifest-{dumped['algo']}.txt"
|
|
94
|
+
output.write_text(manifest)
|
|
95
|
+
return output
|
|
@@ -5,12 +5,12 @@ until we can split them off into a separate package and single-source them.
|
|
|
5
5
|
|
|
6
6
|
import sys
|
|
7
7
|
from datetime import datetime
|
|
8
|
-
from typing import Literal
|
|
8
|
+
from typing import Literal, NotRequired
|
|
9
9
|
|
|
10
10
|
if sys.version_info >= (3, 12):
|
|
11
|
-
from typing import
|
|
11
|
+
from typing import TypedDict
|
|
12
12
|
else:
|
|
13
|
-
from typing_extensions import
|
|
13
|
+
from typing_extensions import TypedDict
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
class Token(TypedDict):
|