sciop-cli 0.1.3__tar.gz → 0.1.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sciop_cli-0.1.3 → sciop_cli-0.1.5}/PKG-INFO +8 -6
- {sciop_cli-0.1.3 → sciop_cli-0.1.5}/pyproject.toml +28 -7
- sciop_cli-0.1.5/src/sciop_cli/api.py +272 -0
- sciop_cli-0.1.5/src/sciop_cli/cli/api.py +114 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.5}/src/sciop_cli/cli/main.py +6 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.5}/src/sciop_cli/cli/torrent.py +69 -5
- sciop_cli-0.1.5/src/sciop_cli/config.py +144 -0
- sciop_cli-0.1.5/src/sciop_cli/const.py +14 -0
- sciop_cli-0.1.5/src/sciop_cli/models/__init__.py +8 -0
- sciop_cli-0.1.5/src/sciop_cli/models/sciop.py +49 -0
- sciop_cli-0.1.5/src/sciop_cli/torrent.py +574 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.5}/src/sciop_cli/types.py +12 -0
- sciop_cli-0.1.5/tests/conftest.py +49 -0
- sciop_cli-0.1.5/tests/fixtures/__init__.py +17 -0
- sciop_cli-0.1.5/tests/fixtures/base.py +12 -0
- sciop_cli-0.1.5/tests/fixtures/fabricators.py +82 -0
- sciop_cli-0.1.5/tests/test_api.py +132 -0
- sciop_cli-0.1.5/tests/test_cli/test_api.py +46 -0
- sciop_cli-0.1.5/tests/test_config.py +55 -0
- sciop_cli-0.1.5/tests/test_torrent.py +116 -0
- sciop_cli-0.1.3/src/sciop_cli/const.py +0 -4
- sciop_cli-0.1.3/src/sciop_cli/file.py +0 -40
- sciop_cli-0.1.3/src/sciop_cli/torrent.py +0 -157
- sciop_cli-0.1.3/tests/.DS_Store +0 -0
- sciop_cli-0.1.3/tests/conftest.py +0 -3
- sciop_cli-0.1.3/tests/test_cli/__init__.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.5}/LICENSE +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.5}/README.md +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.5}/src/sciop_cli/__init__.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.5}/src/sciop_cli/__main__.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.5}/src/sciop_cli/cli/__init__.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.5}/src/sciop_cli/cli/manifest.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.5}/src/sciop_cli/data.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.5}/src/sciop_cli/exceptions.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.5}/src/sciop_cli/schema.py +0 -0
- {sciop_cli-0.1.3/src/sciop_cli/models → sciop_cli-0.1.5/tests}/__init__.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.5}/tests/data/basic/part_1/subdir/part_1 +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.5}/tests/data/basic/part_2/subdir/part_2 +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.5}/tests/data/basic_hybrid.torrent +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.5}/tests/data/basic_v2.torrent +0 -0
- {sciop_cli-0.1.3/tests → sciop_cli-0.1.5/tests/test_cli}/__init__.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.5}/tests/test_cli/test_torrent.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.5}/tests/test_data.py +0 -0
- {sciop_cli-0.1.3 → sciop_cli-0.1.5}/tests/test_temp.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: sciop-cli
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.5
|
|
4
4
|
Summary: A CLI for sciop
|
|
5
5
|
Author-Email: cosmicexplorer <dmcC2@hypnicjerk.ai>, sneakers-the-rat <sneakers-the-rat@protonmail.com>
|
|
6
6
|
License: EUPL-1.2
|
|
@@ -9,20 +9,22 @@ Requires-Dist: libtorrent>=2.0.11
|
|
|
9
9
|
Requires-Dist: click>=8.1.8
|
|
10
10
|
Requires-Dist: click-option-group>=0.5.6
|
|
11
11
|
Requires-Dist: pydantic>=2.10.6
|
|
12
|
-
Requires-Dist: pandera[mypy]>=0.23.1
|
|
13
12
|
Requires-Dist: bencode-rs>=0.0.7
|
|
14
|
-
Requires-Dist: platformdirs>=4.3.
|
|
13
|
+
Requires-Dist: platformdirs>=4.3.8
|
|
15
14
|
Requires-Dist: requests>=2.32.3
|
|
16
15
|
Requires-Dist: tqdm>=4.67.1
|
|
17
16
|
Requires-Dist: rich>=14.0.0
|
|
18
|
-
|
|
19
|
-
Requires-Dist:
|
|
17
|
+
Requires-Dist: humanize>=4.12.2
|
|
18
|
+
Requires-Dist: pydantic-settings>=2.9.1
|
|
19
|
+
Requires-Dist: keyring>=25.6.0
|
|
20
|
+
Requires-Dist: pyyaml>=6.0.2
|
|
20
21
|
Provides-Extra: mypy
|
|
21
22
|
Requires-Dist: mypy>=1.15; extra == "mypy"
|
|
22
23
|
Requires-Dist: types-tqdm>=4.67.0.20250401; extra == "mypy"
|
|
23
24
|
Requires-Dist: types-requests>=2.32.0.20250328; extra == "mypy"
|
|
25
|
+
Requires-Dist: types-PyYAML>=6.0.12.20250516; extra == "mypy"
|
|
24
26
|
Provides-Extra: dev
|
|
25
|
-
Requires-Dist: sciop-cli[mypy
|
|
27
|
+
Requires-Dist: sciop-cli[mypy]; extra == "dev"
|
|
26
28
|
Requires-Dist: ruff>=0.9.7; extra == "dev"
|
|
27
29
|
Requires-Dist: black>=25.1.0; extra == "dev"
|
|
28
30
|
Description-Content-Type: text/markdown
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sciop-cli"
|
|
3
|
-
|
|
3
|
+
dynamic = []
|
|
4
4
|
description = "A CLI for sciop"
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "cosmicexplorer", email = "dmcC2@hypnicjerk.ai" },
|
|
@@ -11,30 +11,32 @@ dependencies = [
|
|
|
11
11
|
"click>=8.1.8",
|
|
12
12
|
"click-option-group>=0.5.6",
|
|
13
13
|
"pydantic>=2.10.6",
|
|
14
|
-
"pandera[mypy]>=0.23.1",
|
|
15
14
|
"bencode-rs>=0.0.7",
|
|
16
|
-
"platformdirs>=4.3.
|
|
15
|
+
"platformdirs>=4.3.8",
|
|
17
16
|
"requests>=2.32.3",
|
|
18
17
|
"tqdm>=4.67.1",
|
|
19
18
|
"rich>=14.0.0",
|
|
19
|
+
"humanize>=4.12.2",
|
|
20
|
+
"pydantic-settings>=2.9.1",
|
|
21
|
+
"keyring>=25.6.0",
|
|
22
|
+
"pyyaml>=6.0.2",
|
|
20
23
|
]
|
|
21
24
|
requires-python = ">=3.11"
|
|
22
25
|
readme = "README.md"
|
|
26
|
+
version = "0.1.5"
|
|
23
27
|
|
|
24
28
|
[project.license]
|
|
25
29
|
text = "EUPL-1.2"
|
|
26
30
|
|
|
27
31
|
[project.optional-dependencies]
|
|
28
|
-
test = [
|
|
29
|
-
"pytest>=8.3.4",
|
|
30
|
-
]
|
|
31
32
|
mypy = [
|
|
32
33
|
"mypy>=1.15",
|
|
33
34
|
"types-tqdm>=4.67.0.20250401",
|
|
34
35
|
"types-requests>=2.32.0.20250328",
|
|
36
|
+
"types-PyYAML>=6.0.12.20250516",
|
|
35
37
|
]
|
|
36
38
|
dev = [
|
|
37
|
-
"sciop-cli[
|
|
39
|
+
"sciop-cli[mypy]",
|
|
38
40
|
"ruff>=0.9.7",
|
|
39
41
|
"black>=25.1.0",
|
|
40
42
|
]
|
|
@@ -42,6 +44,15 @@ dev = [
|
|
|
42
44
|
[project.scripts]
|
|
43
45
|
sciop-cli = "sciop_cli.cli.main:cli"
|
|
44
46
|
|
|
47
|
+
[dependency-groups]
|
|
48
|
+
test = [
|
|
49
|
+
"pytest>=8.3.4",
|
|
50
|
+
"sciop[test] @ git+https://codeberg.org/Safeguarding/sciop.git@main",
|
|
51
|
+
"pytest-asyncio>=1.0.0",
|
|
52
|
+
"httpx>=0.28.1",
|
|
53
|
+
"keyrings-alt>=5.0.2",
|
|
54
|
+
]
|
|
55
|
+
|
|
45
56
|
[build-system]
|
|
46
57
|
requires = [
|
|
47
58
|
"pdm-backend",
|
|
@@ -68,6 +79,11 @@ composite = [
|
|
|
68
79
|
"ruff check --fix",
|
|
69
80
|
]
|
|
70
81
|
|
|
82
|
+
[tool.pdm.version]
|
|
83
|
+
source = "scm"
|
|
84
|
+
tag_filter = "v*"
|
|
85
|
+
tag_regex = "v(?P<version>([1-9][0-9]*!)?(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*))*((a|b|c|rc)(0|[1-9][0-9]*))?(\\.post(0|[1-9][0-9]*))?(\\.dev(0|[1-9][0-9]*))?$)$"
|
|
86
|
+
|
|
71
87
|
[tool.ruff]
|
|
72
88
|
target-version = "py311"
|
|
73
89
|
include = [
|
|
@@ -148,3 +164,8 @@ show_column_numbers = true
|
|
|
148
164
|
show_error_code_links = true
|
|
149
165
|
pretty = true
|
|
150
166
|
color_output = true
|
|
167
|
+
|
|
168
|
+
[tool.pytest.ini_options]
|
|
169
|
+
asyncio_mode = "auto"
|
|
170
|
+
markers = " v1: tests for v1 torrents\n v2: tests for v2 torrents\n hybrid: tests for hybrid torrents\n"
|
|
171
|
+
asyncio_default_fixture_loop_scope = "session"
|
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Code to work with the sciop API :)
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from importlib.metadata import version
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, Literal, TypedDict, cast
|
|
8
|
+
from urllib.parse import urljoin
|
|
9
|
+
|
|
10
|
+
import requests
|
|
11
|
+
from pydantic import SecretStr
|
|
12
|
+
|
|
13
|
+
from sciop_cli.config import Config, get_config, set_config
|
|
14
|
+
from sciop_cli.models import DatasetClaim, Token, TorrentFile, Upload
|
|
15
|
+
|
|
16
|
+
HTTP_METHOD = Literal["GET", "POST", "PUT", "DELETE"]
|
|
17
|
+
USER_AGENT = f"sciop-cli ({version('sciop-cli')})"
|
|
18
|
+
|
|
19
|
+
DefaultHeaders = TypedDict("DefaultHeaders", {"Authorization": str, "User-Agent": str}, total=False)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def path_or_str(value: Path | str) -> str:
|
|
23
|
+
"""
|
|
24
|
+
handle args that can either be contained in a path or passed as a string
|
|
25
|
+
"""
|
|
26
|
+
if isinstance(value, Path):
|
|
27
|
+
return value.read_text()
|
|
28
|
+
try:
|
|
29
|
+
path = Path(value)
|
|
30
|
+
except Exception:
|
|
31
|
+
return value
|
|
32
|
+
|
|
33
|
+
if path.exists():
|
|
34
|
+
return path.read_text()
|
|
35
|
+
else:
|
|
36
|
+
return str(value)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def api_call(
|
|
40
|
+
path: str,
|
|
41
|
+
data: dict | None = None,
|
|
42
|
+
method: HTTP_METHOD = "POST",
|
|
43
|
+
data_encoding: Literal["form", "json"] = "json",
|
|
44
|
+
headers: dict | None = None,
|
|
45
|
+
files: dict | None = None,
|
|
46
|
+
instance_url: str | None = None,
|
|
47
|
+
api_prefix: Literal["/api/v1", False] = "/api/v1",
|
|
48
|
+
autorefresh_token: bool = True,
|
|
49
|
+
do_raise: bool = True,
|
|
50
|
+
) -> requests.Response:
|
|
51
|
+
"""
|
|
52
|
+
Generic request wrapper around sciop API requests.
|
|
53
|
+
|
|
54
|
+
Args:
|
|
55
|
+
path (str): The path beneath the instance url to call.
|
|
56
|
+
Includes the `api_prefix` by default,
|
|
57
|
+
and handles leading `/`'s with [urljoin][urllib.parse.urljoin]
|
|
58
|
+
so e.g. one can just use `"login"` to call `{instance_url}/api/v1/login`.
|
|
59
|
+
data (dict | None): The data to send in a POST request.
|
|
60
|
+
method (HTTP_METHOD): The HTTP method to use.
|
|
61
|
+
headers (dict | None): Extra headers to add to a request.
|
|
62
|
+
These override and are merged with the default headers from
|
|
63
|
+
[default_headers][sciop_cli.api.default_headers].
|
|
64
|
+
files (dict | None): Extra files to send in a POST request.
|
|
65
|
+
data_encoding (Literal["form", "json"]): If `json`, use request's `json=` parameter,
|
|
66
|
+
which sets `Content-Type`. If `form`, use `data=`, which defaults to formencoded data.
|
|
67
|
+
instance_url (str): Instance url to call to. Default is https://sciop.net
|
|
68
|
+
api_prefix (str): API prefix after the `instance_url`,
|
|
69
|
+
either `/api/v1` or `False` to access paths at the base url.
|
|
70
|
+
autorefresh_token (bool): If `True` (default), try and get a token if we have credentials.
|
|
71
|
+
Made `False` e.g. to avoid infinite recursion while logging in,
|
|
72
|
+
or if it is otherwise desirable to use only the preconfigured token.
|
|
73
|
+
do_raise (bool): If `True` (default), raise on status codes >=400
|
|
74
|
+
"""
|
|
75
|
+
config = get_config()
|
|
76
|
+
request_kwargs = _api_call_kwargs(
|
|
77
|
+
path=path,
|
|
78
|
+
config=config,
|
|
79
|
+
data=data,
|
|
80
|
+
method=method,
|
|
81
|
+
data_encoding=data_encoding,
|
|
82
|
+
headers=headers,
|
|
83
|
+
files=files,
|
|
84
|
+
instance_url=instance_url,
|
|
85
|
+
api_prefix=api_prefix,
|
|
86
|
+
autorefresh_token=autorefresh_token,
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
res = requests.request(**request_kwargs)
|
|
90
|
+
if res.status_code == 403 and config.token and not autorefresh_token:
|
|
91
|
+
# try once to get a new token if it just expired
|
|
92
|
+
token = get_token(force=True)
|
|
93
|
+
request_kwargs["headers"]["Authorization"] = f"Bearer {token}"
|
|
94
|
+
res = requests.request(**request_kwargs)
|
|
95
|
+
|
|
96
|
+
if do_raise:
|
|
97
|
+
res.raise_for_status()
|
|
98
|
+
return res
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _api_call_kwargs(
|
|
102
|
+
path: str,
|
|
103
|
+
config: Config,
|
|
104
|
+
data: dict | None = None,
|
|
105
|
+
method: HTTP_METHOD = "POST",
|
|
106
|
+
data_encoding: Literal["form", "json"] = "json",
|
|
107
|
+
headers: dict | None = None,
|
|
108
|
+
files: dict | None = None,
|
|
109
|
+
instance_url: str | None = None,
|
|
110
|
+
api_prefix: Literal["/api/v1", False] = "/api/v1",
|
|
111
|
+
autorefresh_token: bool = True,
|
|
112
|
+
) -> dict:
|
|
113
|
+
default = default_headers(
|
|
114
|
+
instance_url=instance_url, headers=headers, autorefresh_token=autorefresh_token
|
|
115
|
+
)
|
|
116
|
+
merged_headers = default | headers if headers else default
|
|
117
|
+
|
|
118
|
+
if instance_url is None:
|
|
119
|
+
instance_url = config.instance_url
|
|
120
|
+
|
|
121
|
+
if api_prefix:
|
|
122
|
+
path = "/".join([segment.strip("/") for segment in [api_prefix, path]])
|
|
123
|
+
|
|
124
|
+
url = urljoin(instance_url, path)
|
|
125
|
+
request_kwargs = {
|
|
126
|
+
"method": method,
|
|
127
|
+
"url": url,
|
|
128
|
+
"headers": merged_headers,
|
|
129
|
+
"timeout": config.request_timeout,
|
|
130
|
+
"files": files,
|
|
131
|
+
}
|
|
132
|
+
if data:
|
|
133
|
+
if data_encoding == "form":
|
|
134
|
+
request_kwargs["data"] = data
|
|
135
|
+
elif data_encoding == "json":
|
|
136
|
+
request_kwargs["json"] = data
|
|
137
|
+
else:
|
|
138
|
+
raise ValueError(f"data_encoding must be form or json, got {data_encoding}")
|
|
139
|
+
return request_kwargs
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def default_headers(autorefresh_token: bool = True, **kwargs: Any) -> DefaultHeaders:
|
|
143
|
+
token = get_token(autorefresh_token=autorefresh_token, **kwargs)
|
|
144
|
+
headers: DefaultHeaders = {"User-Agent": USER_AGENT}
|
|
145
|
+
if token:
|
|
146
|
+
headers["Authorization"] = f"Bearer {token}"
|
|
147
|
+
return headers
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def login(username: str, password: str, **kwargs: Any) -> str:
|
|
151
|
+
"""
|
|
152
|
+
Get a token from the API with a username and password
|
|
153
|
+
|
|
154
|
+
Args:
|
|
155
|
+
**kwargs: passed to [api_call][sciop_cli.api.api_call]
|
|
156
|
+
"""
|
|
157
|
+
res = api_call(
|
|
158
|
+
"login", data={"username": username, "password": password}, data_encoding="form", **kwargs
|
|
159
|
+
)
|
|
160
|
+
token: Token = res.json()
|
|
161
|
+
return token["access_token"]
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def get_token(
|
|
165
|
+
username: str | None = None,
|
|
166
|
+
password: str | None = None,
|
|
167
|
+
force: bool = False,
|
|
168
|
+
autorefresh_token: bool = True,
|
|
169
|
+
**kwargs: Any,
|
|
170
|
+
) -> str | None:
|
|
171
|
+
"""
|
|
172
|
+
Get a token from passed username/password or from config.
|
|
173
|
+
|
|
174
|
+
Reuses the already-existing token if present.
|
|
175
|
+
Sets the username, password (if passed) and received token if we get a new one.
|
|
176
|
+
|
|
177
|
+
Args:
|
|
178
|
+
username (str): Username. If None (default),
|
|
179
|
+
use from [get_config][sciop_cli.config.get_config].
|
|
180
|
+
Must be used with `password`
|
|
181
|
+
password (str): Password. If None (default),
|
|
182
|
+
use from [get_config][sciop_cli.config.get_config].
|
|
183
|
+
force (bool): If True, get a new token from the instance,
|
|
184
|
+
even if one is stored in the config.
|
|
185
|
+
**kwargs: passed to [api_call][sciop_cli.api.api_call] if a login request is made
|
|
186
|
+
|
|
187
|
+
Returns:
|
|
188
|
+
str: if token present
|
|
189
|
+
None: if no username/password has been given.
|
|
190
|
+
"""
|
|
191
|
+
config = get_config()
|
|
192
|
+
if config.token and not force:
|
|
193
|
+
return config.token.get_secret_value()
|
|
194
|
+
|
|
195
|
+
if not username and config.username:
|
|
196
|
+
username = config.username
|
|
197
|
+
if not password and config.password:
|
|
198
|
+
password = config.password.get_secret_value()
|
|
199
|
+
|
|
200
|
+
if username and password and autorefresh_token:
|
|
201
|
+
token = login(username=username, password=password, autorefresh_token=False, **kwargs)
|
|
202
|
+
config.token = SecretStr(token)
|
|
203
|
+
config.username = username
|
|
204
|
+
config.password = SecretStr(password)
|
|
205
|
+
set_config(config)
|
|
206
|
+
return token
|
|
207
|
+
else:
|
|
208
|
+
return None
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def claim_next(dataset: str) -> DatasetClaim | None:
|
|
212
|
+
res = api_call(f"claims/datasets/{dataset}/next", do_raise=False)
|
|
213
|
+
if res.status_code == 404 and "no unclaimed" in res.text.lower():
|
|
214
|
+
return None
|
|
215
|
+
res.raise_for_status()
|
|
216
|
+
claim = cast(DatasetClaim, res.json())
|
|
217
|
+
return DatasetClaim(**claim)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def upload_torrent(path: Path) -> TorrentFile:
|
|
221
|
+
with open(path, "rb") as f:
|
|
222
|
+
response = api_call("torrents", files={"file": f})
|
|
223
|
+
tf = cast(TorrentFile, response.json())
|
|
224
|
+
return TorrentFile(**tf)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def create_upload(
|
|
228
|
+
dataset: str,
|
|
229
|
+
dataset_parts: list[str] | None = None,
|
|
230
|
+
torrent_path: Path | None = None,
|
|
231
|
+
infohash: str | None = None,
|
|
232
|
+
method: str | Path | None = None,
|
|
233
|
+
description: str | Path | None = None,
|
|
234
|
+
) -> Upload:
|
|
235
|
+
"""
|
|
236
|
+
Create an upload for a dataset or dataset parts.
|
|
237
|
+
|
|
238
|
+
Must pass either a path to an existing .torrent file, or an infohash of a torrent
|
|
239
|
+
that has already been uploaded to the configured sciop instance.
|
|
240
|
+
|
|
241
|
+
Args:
|
|
242
|
+
dataset (str): Slug of the dataset to upload to
|
|
243
|
+
torrent_path (Path): Path to the .torrent file to upload - pass either this or `infohash`
|
|
244
|
+
infohash (str): Infohash of a torrent that has already been uploaded -
|
|
245
|
+
pass either this or `torrent_path`
|
|
246
|
+
method (str | Path | None): A description of how this upload was created,
|
|
247
|
+
Supports markdown as a path or string.
|
|
248
|
+
description (str | Path | None): Any description of the contents of the upload
|
|
249
|
+
that are not contained within the dataset description, e.g. additional structure,
|
|
250
|
+
if this is some sub-component of a dataset, and so on
|
|
251
|
+
"""
|
|
252
|
+
if torrent_path and infohash or not (torrent_path or infohash):
|
|
253
|
+
raise ValueError("Must pass torrent path OR infohash")
|
|
254
|
+
elif torrent_path:
|
|
255
|
+
tf = upload_torrent(path=torrent_path)
|
|
256
|
+
infohash = tf["v1_infohash"] if tf["version"] == "v1" else tf["v2_infohash"]
|
|
257
|
+
|
|
258
|
+
kwargs: dict[str, str | list[str] | None] = {
|
|
259
|
+
"infohash": infohash,
|
|
260
|
+
"dataset_slug": dataset,
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
if method:
|
|
264
|
+
kwargs["method"] = path_or_str(method)
|
|
265
|
+
if description:
|
|
266
|
+
kwargs["description"] = path_or_str(description)
|
|
267
|
+
if dataset_parts:
|
|
268
|
+
kwargs["part_slugs"] = dataset_parts
|
|
269
|
+
|
|
270
|
+
res = api_call("uploads", data=kwargs)
|
|
271
|
+
ul = cast(Upload, res.json())
|
|
272
|
+
return Upload(**ul)
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import json as json_
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
import click
|
|
5
|
+
from pydantic import SecretStr
|
|
6
|
+
from rich import print as pprint
|
|
7
|
+
from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
|
|
8
|
+
|
|
9
|
+
from sciop_cli.api import claim_next as claim_next_api
|
|
10
|
+
from sciop_cli.api import create_upload
|
|
11
|
+
from sciop_cli.api import login as api_login
|
|
12
|
+
from sciop_cli.config import get_config, set_config
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@click.command("login")
|
|
16
|
+
@click.option("-u", "--username", prompt=True, help="Username", required=True)
|
|
17
|
+
@click.option("-p", "--password", prompt=True, hide_input=True, help="Password", required=True)
|
|
18
|
+
@click.option("-i", "--instance-url", help="Instance to log in to, if None, sciop.net")
|
|
19
|
+
def login(username: str, password: str, instance_url: str | None = None) -> None:
|
|
20
|
+
"""
|
|
21
|
+
Login to sciop.net (or another instance), storing the password in a keyring
|
|
22
|
+
and getting a token
|
|
23
|
+
"""
|
|
24
|
+
token = api_login(username, password, instance_url=instance_url)
|
|
25
|
+
cfg = get_config()
|
|
26
|
+
cfg.username = username
|
|
27
|
+
cfg.password = SecretStr(password)
|
|
28
|
+
cfg.token = SecretStr(token)
|
|
29
|
+
set_config(cfg)
|
|
30
|
+
click.echo(f"Logged in as {username} ;)")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@click.command("upload")
|
|
34
|
+
@click.option("-d", "--dataset", help="Slug of the target dataset", required=True)
|
|
35
|
+
@click.option(
|
|
36
|
+
"-p",
|
|
37
|
+
"--part",
|
|
38
|
+
help="Slug of the part, or parts for the upload (optional)."
|
|
39
|
+
"Can be used multiple times to upload to multiple dataset parts",
|
|
40
|
+
multiple=True,
|
|
41
|
+
)
|
|
42
|
+
@click.option(
|
|
43
|
+
"-t",
|
|
44
|
+
"--torrent",
|
|
45
|
+
help="Path to the torrent file to upload",
|
|
46
|
+
required=False,
|
|
47
|
+
type=click.Path(),
|
|
48
|
+
default=None,
|
|
49
|
+
)
|
|
50
|
+
@click.option(
|
|
51
|
+
"-i",
|
|
52
|
+
"--infohash",
|
|
53
|
+
help="If no path is passed, an infohash of an already-uploaded torrent",
|
|
54
|
+
required=False,
|
|
55
|
+
default=None,
|
|
56
|
+
)
|
|
57
|
+
@click.option(
|
|
58
|
+
"-m",
|
|
59
|
+
"--method",
|
|
60
|
+
help="A description of how this upload was created. Supports markdown as a path or string.",
|
|
61
|
+
)
|
|
62
|
+
@click.option(
|
|
63
|
+
"-d",
|
|
64
|
+
"--description",
|
|
65
|
+
help="Any description of the contents of the upload that are not contained within "
|
|
66
|
+
"the dataset description, "
|
|
67
|
+
"e.g. additional structure, if this is some sub-component of a dataset, and so on. "
|
|
68
|
+
"Supports markdown as a path or string.",
|
|
69
|
+
)
|
|
70
|
+
def upload(
|
|
71
|
+
dataset: str,
|
|
72
|
+
part: list[str] | None = None,
|
|
73
|
+
torrent: Path | None = None,
|
|
74
|
+
infohash: str | None = None,
|
|
75
|
+
method: str | None = None,
|
|
76
|
+
description: str | None = None,
|
|
77
|
+
) -> None:
|
|
78
|
+
"""
|
|
79
|
+
Create an upload for a dataset or dataset parts.
|
|
80
|
+
|
|
81
|
+
Must pass either a path to an existing .torrent file, or an infohash of a torrent
|
|
82
|
+
that has already been uploaded to the configured sciop instance.
|
|
83
|
+
"""
|
|
84
|
+
progress = Progress(SpinnerColumn(), TimeElapsedColumn(), TextColumn("{task.description}"))
|
|
85
|
+
with progress:
|
|
86
|
+
progress.add_task(description="Uploading torrent...", total=None)
|
|
87
|
+
upload = create_upload(
|
|
88
|
+
dataset=dataset,
|
|
89
|
+
dataset_parts=part,
|
|
90
|
+
infohash=infohash,
|
|
91
|
+
method=method,
|
|
92
|
+
description=description,
|
|
93
|
+
)
|
|
94
|
+
pprint(upload)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@click.group("claims")
|
|
98
|
+
def claims() -> None:
|
|
99
|
+
"""Claims that we are scraping some data as part of a Quest :)"""
|
|
100
|
+
pass
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@claims.command("next")
|
|
104
|
+
@click.option("-d", "--dataset", required=True, help="Dataset to claim (as its slug)")
|
|
105
|
+
@click.option("-j", "--json", is_flag=True, default=False, help="Print response as json to stdout")
|
|
106
|
+
def next_claim(dataset: str, json: bool = False) -> None:
|
|
107
|
+
"""
|
|
108
|
+
Get the next unclaimed, un-uploaded dataset part for a dataset, printing it
|
|
109
|
+
"""
|
|
110
|
+
res = claim_next_api(dataset)
|
|
111
|
+
if json:
|
|
112
|
+
click.echo(json_.dumps(res))
|
|
113
|
+
else:
|
|
114
|
+
click.echo(res)
|
|
@@ -2,6 +2,7 @@ import logging
|
|
|
2
2
|
|
|
3
3
|
import click
|
|
4
4
|
|
|
5
|
+
from sciop_cli.cli.api import claims, login
|
|
5
6
|
from sciop_cli.cli.manifest import manifest
|
|
6
7
|
from sciop_cli.cli.torrent import torrent
|
|
7
8
|
|
|
@@ -19,5 +20,10 @@ def cli(debug: bool) -> None:
|
|
|
19
20
|
logging.basicConfig(level=logging.INFO)
|
|
20
21
|
|
|
21
22
|
|
|
23
|
+
# top-level API commands
|
|
24
|
+
cli.add_command(login)
|
|
25
|
+
cli.add_command(claims)
|
|
26
|
+
|
|
27
|
+
# command groups
|
|
22
28
|
cli.add_command(torrent)
|
|
23
29
|
cli.add_command(manifest)
|
|
@@ -1,11 +1,23 @@
|
|
|
1
1
|
from pathlib import Path
|
|
2
|
+
from typing import Literal as L
|
|
2
3
|
from typing import cast
|
|
3
4
|
|
|
4
5
|
import click
|
|
6
|
+
import humanize
|
|
7
|
+
from rich import print
|
|
8
|
+
from rich.console import Group
|
|
9
|
+
from rich.panel import Panel
|
|
10
|
+
from rich.table import Table
|
|
5
11
|
|
|
6
12
|
from sciop_cli.const import DEFAULT_TORRENT_CREATOR
|
|
7
13
|
from sciop_cli.data import get_default_trackers
|
|
8
|
-
from sciop_cli.torrent import
|
|
14
|
+
from sciop_cli.torrent import (
|
|
15
|
+
calculate_overhead,
|
|
16
|
+
calculate_total_pieces,
|
|
17
|
+
create_torrent,
|
|
18
|
+
find_optimal_piece_size,
|
|
19
|
+
iter_files,
|
|
20
|
+
)
|
|
9
21
|
from sciop_cli.types import PieceSize
|
|
10
22
|
|
|
11
23
|
|
|
@@ -59,8 +71,9 @@ def pack() -> None:
|
|
|
59
71
|
@click.option(
|
|
60
72
|
"-s",
|
|
61
73
|
"--piece-size",
|
|
62
|
-
default=
|
|
63
|
-
help="Piece size, in bytes"
|
|
74
|
+
default=None,
|
|
75
|
+
help="Piece size, in bytes. If not given, calculate piece size automatically."
|
|
76
|
+
"Use `sciop-cli torrent piece-size` to preview the ",
|
|
64
77
|
show_default=True,
|
|
65
78
|
)
|
|
66
79
|
@click.option(
|
|
@@ -114,7 +127,7 @@ def create(
|
|
|
114
127
|
path: Path,
|
|
115
128
|
tracker: list[str] | tuple[str] | None = None,
|
|
116
129
|
default_trackers: bool | None = None,
|
|
117
|
-
piece_size: PieceSize
|
|
130
|
+
piece_size: PieceSize | None = None,
|
|
118
131
|
comment: str | None = None,
|
|
119
132
|
creator: str = DEFAULT_TORRENT_CREATOR,
|
|
120
133
|
webseed: list[str] | None = None,
|
|
@@ -135,7 +148,14 @@ def create(
|
|
|
135
148
|
"""
|
|
136
149
|
# recast tuple to list or none rather than tuple or empty tuple
|
|
137
150
|
tracker = list(tracker) if tracker else None
|
|
151
|
+
version = "v2" if v2 else "hybrid"
|
|
152
|
+
version = cast(L["v2", "hybrid"], version)
|
|
153
|
+
|
|
154
|
+
if piece_size is None:
|
|
155
|
+
click.echo("No piece size specified, estimating optimal piece size")
|
|
156
|
+
piece_size = find_optimal_piece_size(path=Path(path).absolute(), version=version)
|
|
138
157
|
|
|
158
|
+
click.echo(f"Piece size estimated as {humanize.naturalsize(piece_size, binary=True)}")
|
|
139
159
|
if not tracker and (default_trackers is None or default_trackers):
|
|
140
160
|
click.echo(
|
|
141
161
|
"No trackers specified, using default trackers from "
|
|
@@ -154,7 +174,7 @@ def create(
|
|
|
154
174
|
creator=creator,
|
|
155
175
|
webseeds=webseed,
|
|
156
176
|
similar=similar,
|
|
157
|
-
version=
|
|
177
|
+
version=version,
|
|
158
178
|
pbar=progress,
|
|
159
179
|
bencode=True,
|
|
160
180
|
)
|
|
@@ -164,3 +184,47 @@ def create(
|
|
|
164
184
|
f.write(result)
|
|
165
185
|
else:
|
|
166
186
|
click.echo(result)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
@torrent.command("piece-size")
|
|
190
|
+
@click.option(
|
|
191
|
+
"-p",
|
|
192
|
+
"--path",
|
|
193
|
+
required=True,
|
|
194
|
+
help="Path to a directory or file to create .torrent from",
|
|
195
|
+
type=click.Path(exists=True),
|
|
196
|
+
)
|
|
197
|
+
def piece_size(path: Path) -> None:
|
|
198
|
+
"""
|
|
199
|
+
Show the optimal piece sizes calculated for each torrent version
|
|
200
|
+
for a given file or directory.
|
|
201
|
+
"""
|
|
202
|
+
path = Path(path)
|
|
203
|
+
paths = [path] if path.is_file() else list(iter_files(path))
|
|
204
|
+
|
|
205
|
+
sizes = [p.stat().st_size for p in paths]
|
|
206
|
+
versions = ("v1", "v2", "hybrid")
|
|
207
|
+
piece_sizes = {v: find_optimal_piece_size(path=paths, sizes=sizes, version=v) for v in versions}
|
|
208
|
+
n_pieces = {v: calculate_total_pieces(sizes, piece_sizes[v], v) for v in versions}
|
|
209
|
+
hybrid_overhead = sum(calculate_overhead(sizes, piece_sizes["hybrid"]))
|
|
210
|
+
|
|
211
|
+
summary = Table(show_header=False)
|
|
212
|
+
summary.add_column("", style="bold magenta")
|
|
213
|
+
summary.add_column("")
|
|
214
|
+
summary.add_row("N Files", humanize.number.intcomma(len(paths)))
|
|
215
|
+
summary.add_row("Total size", humanize.naturalsize(sum(sizes), binary=True))
|
|
216
|
+
|
|
217
|
+
pieces = Table(title="Piece sizes")
|
|
218
|
+
pieces.add_column("Version")
|
|
219
|
+
pieces.add_column("Piece Size")
|
|
220
|
+
pieces.add_column("N Pieces")
|
|
221
|
+
pieces.add_column("Padding Overhead")
|
|
222
|
+
for v in ("v1", "v2", "hybrid"):
|
|
223
|
+
row = [v, str(piece_sizes[v]), humanize.number.intcomma(n_pieces[v])]
|
|
224
|
+
|
|
225
|
+
if v == "hybrid":
|
|
226
|
+
row.append(humanize.naturalsize(hybrid_overhead, binary=True))
|
|
227
|
+
pieces.add_row(*row)
|
|
228
|
+
|
|
229
|
+
panel = Panel(Group(summary, pieces), title=str(path))
|
|
230
|
+
print(panel)
|