sciop-cli 0.1.4__tar.gz → 0.1.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/PKG-INFO +7 -6
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/pyproject.toml +24 -7
- sciop_cli-0.1.6/src/sciop_cli/api.py +280 -0
- sciop_cli-0.1.6/src/sciop_cli/cli/api.py +114 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/src/sciop_cli/cli/main.py +6 -0
- sciop_cli-0.1.6/src/sciop_cli/config.py +144 -0
- sciop_cli-0.1.6/src/sciop_cli/models/__init__.py +8 -0
- sciop_cli-0.1.6/src/sciop_cli/models/sciop.py +49 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/src/sciop_cli/torrent.py +24 -10
- sciop_cli-0.1.6/tests/conftest.py +49 -0
- sciop_cli-0.1.6/tests/data/distributions/nbu_arwen_ver01.json +1 -0
- sciop_cli-0.1.6/tests/fixtures/__init__.py +17 -0
- sciop_cli-0.1.6/tests/fixtures/base.py +12 -0
- sciop_cli-0.1.6/tests/fixtures/fabricators.py +82 -0
- sciop_cli-0.1.6/tests/test_api.py +132 -0
- sciop_cli-0.1.6/tests/test_cli/test_api.py +46 -0
- sciop_cli-0.1.6/tests/test_config.py +55 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/tests/test_torrent.py +22 -11
- sciop_cli-0.1.4/src/sciop_cli/file.py +0 -40
- sciop_cli-0.1.4/tests/.DS_Store +0 -0
- sciop_cli-0.1.4/tests/conftest.py +0 -3
- sciop_cli-0.1.4/tests/test_cli/__init__.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/LICENSE +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/README.md +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/src/sciop_cli/__init__.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/src/sciop_cli/__main__.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/src/sciop_cli/cli/__init__.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/src/sciop_cli/cli/manifest.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/src/sciop_cli/cli/torrent.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/src/sciop_cli/const.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/src/sciop_cli/data.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/src/sciop_cli/exceptions.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/src/sciop_cli/schema.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/src/sciop_cli/types.py +0 -0
- {sciop_cli-0.1.4/src/sciop_cli/models → sciop_cli-0.1.6/tests}/__init__.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/tests/data/basic/part_1/subdir/part_1 +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/tests/data/basic/part_2/subdir/part_2 +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/tests/data/basic_hybrid.torrent +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/tests/data/basic_v2.torrent +0 -0
- {sciop_cli-0.1.4/tests → sciop_cli-0.1.6/tests/test_cli}/__init__.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/tests/test_cli/test_torrent.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/tests/test_data.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.6}/tests/test_temp.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: sciop-cli
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.6
|
|
4
4
|
Summary: A CLI for sciop
|
|
5
5
|
Author-Email: cosmicexplorer <dmcC2@hypnicjerk.ai>, sneakers-the-rat <sneakers-the-rat@protonmail.com>
|
|
6
6
|
License: EUPL-1.2
|
|
@@ -9,21 +9,22 @@ Requires-Dist: libtorrent>=2.0.11
|
|
|
9
9
|
Requires-Dist: click>=8.1.8
|
|
10
10
|
Requires-Dist: click-option-group>=0.5.6
|
|
11
11
|
Requires-Dist: pydantic>=2.10.6
|
|
12
|
-
Requires-Dist: pandera[mypy]>=0.23.1
|
|
13
12
|
Requires-Dist: bencode-rs>=0.0.7
|
|
14
|
-
Requires-Dist: platformdirs>=4.3.
|
|
13
|
+
Requires-Dist: platformdirs>=4.3.8
|
|
15
14
|
Requires-Dist: requests>=2.32.3
|
|
16
15
|
Requires-Dist: tqdm>=4.67.1
|
|
17
16
|
Requires-Dist: rich>=14.0.0
|
|
18
17
|
Requires-Dist: humanize>=4.12.2
|
|
19
|
-
|
|
20
|
-
Requires-Dist:
|
|
18
|
+
Requires-Dist: pydantic-settings>=2.9.1
|
|
19
|
+
Requires-Dist: keyring>=25.6.0
|
|
20
|
+
Requires-Dist: pyyaml>=6.0.2
|
|
21
21
|
Provides-Extra: mypy
|
|
22
22
|
Requires-Dist: mypy>=1.15; extra == "mypy"
|
|
23
23
|
Requires-Dist: types-tqdm>=4.67.0.20250401; extra == "mypy"
|
|
24
24
|
Requires-Dist: types-requests>=2.32.0.20250328; extra == "mypy"
|
|
25
|
+
Requires-Dist: types-PyYAML>=6.0.12.20250516; extra == "mypy"
|
|
25
26
|
Provides-Extra: dev
|
|
26
|
-
Requires-Dist: sciop-cli[mypy
|
|
27
|
+
Requires-Dist: sciop-cli[mypy]; extra == "dev"
|
|
27
28
|
Requires-Dist: ruff>=0.9.7; extra == "dev"
|
|
28
29
|
Requires-Dist: black>=25.1.0; extra == "dev"
|
|
29
30
|
Description-Content-Type: text/markdown
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sciop-cli"
|
|
3
|
-
|
|
3
|
+
dynamic = []
|
|
4
4
|
description = "A CLI for sciop"
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "cosmicexplorer", email = "dmcC2@hypnicjerk.ai" },
|
|
@@ -11,31 +11,32 @@ dependencies = [
|
|
|
11
11
|
"click>=8.1.8",
|
|
12
12
|
"click-option-group>=0.5.6",
|
|
13
13
|
"pydantic>=2.10.6",
|
|
14
|
-
"pandera[mypy]>=0.23.1",
|
|
15
14
|
"bencode-rs>=0.0.7",
|
|
16
|
-
"platformdirs>=4.3.
|
|
15
|
+
"platformdirs>=4.3.8",
|
|
17
16
|
"requests>=2.32.3",
|
|
18
17
|
"tqdm>=4.67.1",
|
|
19
18
|
"rich>=14.0.0",
|
|
20
19
|
"humanize>=4.12.2",
|
|
20
|
+
"pydantic-settings>=2.9.1",
|
|
21
|
+
"keyring>=25.6.0",
|
|
22
|
+
"pyyaml>=6.0.2",
|
|
21
23
|
]
|
|
22
24
|
requires-python = ">=3.11"
|
|
23
25
|
readme = "README.md"
|
|
26
|
+
version = "0.1.6"
|
|
24
27
|
|
|
25
28
|
[project.license]
|
|
26
29
|
text = "EUPL-1.2"
|
|
27
30
|
|
|
28
31
|
[project.optional-dependencies]
|
|
29
|
-
test = [
|
|
30
|
-
"pytest>=8.3.4",
|
|
31
|
-
]
|
|
32
32
|
mypy = [
|
|
33
33
|
"mypy>=1.15",
|
|
34
34
|
"types-tqdm>=4.67.0.20250401",
|
|
35
35
|
"types-requests>=2.32.0.20250328",
|
|
36
|
+
"types-PyYAML>=6.0.12.20250516",
|
|
36
37
|
]
|
|
37
38
|
dev = [
|
|
38
|
-
"sciop-cli[
|
|
39
|
+
"sciop-cli[mypy]",
|
|
39
40
|
"ruff>=0.9.7",
|
|
40
41
|
"black>=25.1.0",
|
|
41
42
|
]
|
|
@@ -43,6 +44,15 @@ dev = [
|
|
|
43
44
|
[project.scripts]
|
|
44
45
|
sciop-cli = "sciop_cli.cli.main:cli"
|
|
45
46
|
|
|
47
|
+
[dependency-groups]
|
|
48
|
+
test = [
|
|
49
|
+
"pytest>=8.3.4",
|
|
50
|
+
"sciop[test] @ git+https://codeberg.org/Safeguarding/sciop.git@main",
|
|
51
|
+
"pytest-asyncio>=1.0.0",
|
|
52
|
+
"httpx>=0.28.1",
|
|
53
|
+
"keyrings-alt>=5.0.2",
|
|
54
|
+
]
|
|
55
|
+
|
|
46
56
|
[build-system]
|
|
47
57
|
requires = [
|
|
48
58
|
"pdm-backend",
|
|
@@ -69,6 +79,11 @@ composite = [
|
|
|
69
79
|
"ruff check --fix",
|
|
70
80
|
]
|
|
71
81
|
|
|
82
|
+
[tool.pdm.version]
|
|
83
|
+
source = "scm"
|
|
84
|
+
tag_filter = "v*"
|
|
85
|
+
tag_regex = "v(?P<version>([1-9][0-9]*!)?(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*))*((a|b|c|rc)(0|[1-9][0-9]*))?(\\.post(0|[1-9][0-9]*))?(\\.dev(0|[1-9][0-9]*))?$)$"
|
|
86
|
+
|
|
72
87
|
[tool.ruff]
|
|
73
88
|
target-version = "py311"
|
|
74
89
|
include = [
|
|
@@ -151,4 +166,6 @@ pretty = true
|
|
|
151
166
|
color_output = true
|
|
152
167
|
|
|
153
168
|
[tool.pytest.ini_options]
|
|
169
|
+
asyncio_mode = "auto"
|
|
154
170
|
markers = " v1: tests for v1 torrents\n v2: tests for v2 torrents\n hybrid: tests for hybrid torrents\n"
|
|
171
|
+
asyncio_default_fixture_loop_scope = "session"
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Code to work with the sciop API :)
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from importlib.metadata import version
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, Literal, TypedDict, cast
|
|
8
|
+
from urllib.parse import urljoin
|
|
9
|
+
|
|
10
|
+
import requests
|
|
11
|
+
from pydantic import SecretStr
|
|
12
|
+
|
|
13
|
+
from sciop_cli.config import Config, get_config, set_config
|
|
14
|
+
from sciop_cli.models import DatasetClaim, Token, TorrentFile, Upload
|
|
15
|
+
|
|
16
|
+
HTTP_METHOD = Literal["GET", "POST", "PUT", "DELETE"]
|
|
17
|
+
USER_AGENT = f"sciop-cli ({version('sciop-cli')})"
|
|
18
|
+
|
|
19
|
+
DefaultHeaders = TypedDict("DefaultHeaders", {"Authorization": str, "User-Agent": str}, total=False)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def path_or_str(value: Path | str) -> str:
|
|
23
|
+
"""
|
|
24
|
+
handle args that can either be contained in a path or passed as a string
|
|
25
|
+
"""
|
|
26
|
+
if isinstance(value, Path):
|
|
27
|
+
return value.read_text()
|
|
28
|
+
try:
|
|
29
|
+
path = Path(value)
|
|
30
|
+
except Exception:
|
|
31
|
+
return value
|
|
32
|
+
|
|
33
|
+
if path.exists():
|
|
34
|
+
return path.read_text()
|
|
35
|
+
else:
|
|
36
|
+
return str(value)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def api_call(
|
|
40
|
+
path: str,
|
|
41
|
+
data: dict | None = None,
|
|
42
|
+
method: HTTP_METHOD = "POST",
|
|
43
|
+
data_encoding: Literal["form", "json"] = "json",
|
|
44
|
+
headers: dict | None = None,
|
|
45
|
+
files: dict | None = None,
|
|
46
|
+
instance_url: str | None = None,
|
|
47
|
+
api_prefix: Literal["/api/v1", False] = "/api/v1",
|
|
48
|
+
autorefresh_token: bool = True,
|
|
49
|
+
timeout: float | None = None,
|
|
50
|
+
do_raise: bool = True,
|
|
51
|
+
) -> requests.Response:
|
|
52
|
+
"""
|
|
53
|
+
Generic request wrapper around sciop API requests.
|
|
54
|
+
|
|
55
|
+
Args:
|
|
56
|
+
path (str): The path beneath the instance url to call.
|
|
57
|
+
Includes the `api_prefix` by default,
|
|
58
|
+
and handles leading `/`'s with [urljoin][urllib.parse.urljoin]
|
|
59
|
+
so e.g. one can just use `"login"` to call `{instance_url}/api/v1/login`.
|
|
60
|
+
data (dict | None): The data to send in a POST request.
|
|
61
|
+
method (HTTP_METHOD): The HTTP method to use.
|
|
62
|
+
headers (dict | None): Extra headers to add to a request.
|
|
63
|
+
These override and are merged with the default headers from
|
|
64
|
+
[default_headers][sciop_cli.api.default_headers].
|
|
65
|
+
files (dict | None): Extra files to send in a POST request.
|
|
66
|
+
data_encoding (Literal["form", "json"]): If `json`, use request's `json=` parameter,
|
|
67
|
+
which sets `Content-Type`. If `form`, use `data=`, which defaults to formencoded data.
|
|
68
|
+
instance_url (str): Instance url to call to. Default is https://sciop.net
|
|
69
|
+
api_prefix (str): API prefix after the `instance_url`,
|
|
70
|
+
either `/api/v1` or `False` to access paths at the base url.
|
|
71
|
+
autorefresh_token (bool): If `True` (default), try and get a token if we have credentials.
|
|
72
|
+
Made `False` e.g. to avoid infinite recursion while logging in,
|
|
73
|
+
or if it is otherwise desirable to use only the preconfigured token.
|
|
74
|
+
timeout (float): Explicit request timeout. If `None`, use config.request_timeout
|
|
75
|
+
do_raise (bool): If `True` (default), raise on status codes >=400
|
|
76
|
+
"""
|
|
77
|
+
config = get_config()
|
|
78
|
+
request_kwargs = _api_call_kwargs(
|
|
79
|
+
path=path,
|
|
80
|
+
config=config,
|
|
81
|
+
data=data,
|
|
82
|
+
method=method,
|
|
83
|
+
data_encoding=data_encoding,
|
|
84
|
+
headers=headers,
|
|
85
|
+
files=files,
|
|
86
|
+
instance_url=instance_url,
|
|
87
|
+
api_prefix=api_prefix,
|
|
88
|
+
autorefresh_token=autorefresh_token,
|
|
89
|
+
timeout=timeout,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
res = requests.request(**request_kwargs)
|
|
93
|
+
if res.status_code == 403 and config.token and not autorefresh_token:
|
|
94
|
+
# try once to get a new token if it just expired
|
|
95
|
+
token = get_token(force=True)
|
|
96
|
+
request_kwargs["headers"]["Authorization"] = f"Bearer {token}"
|
|
97
|
+
res = requests.request(**request_kwargs)
|
|
98
|
+
|
|
99
|
+
if do_raise:
|
|
100
|
+
res.raise_for_status()
|
|
101
|
+
return res
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _api_call_kwargs(
|
|
105
|
+
path: str,
|
|
106
|
+
config: Config,
|
|
107
|
+
data: dict | None = None,
|
|
108
|
+
method: HTTP_METHOD = "POST",
|
|
109
|
+
data_encoding: Literal["form", "json"] = "json",
|
|
110
|
+
headers: dict | None = None,
|
|
111
|
+
files: dict | None = None,
|
|
112
|
+
instance_url: str | None = None,
|
|
113
|
+
api_prefix: Literal["/api/v1", False] = "/api/v1",
|
|
114
|
+
autorefresh_token: bool = True,
|
|
115
|
+
timeout: float | None = None,
|
|
116
|
+
) -> dict:
|
|
117
|
+
default = default_headers(
|
|
118
|
+
instance_url=instance_url, headers=headers, autorefresh_token=autorefresh_token
|
|
119
|
+
)
|
|
120
|
+
merged_headers = default | headers if headers else default
|
|
121
|
+
|
|
122
|
+
if instance_url is None:
|
|
123
|
+
instance_url = config.instance_url
|
|
124
|
+
|
|
125
|
+
if api_prefix:
|
|
126
|
+
path = "/".join([segment.strip("/") for segment in [api_prefix, path]])
|
|
127
|
+
|
|
128
|
+
url = urljoin(instance_url, path)
|
|
129
|
+
request_kwargs = {
|
|
130
|
+
"method": method,
|
|
131
|
+
"url": url,
|
|
132
|
+
"headers": merged_headers,
|
|
133
|
+
"timeout": timeout if timeout else config.request_timeout,
|
|
134
|
+
"files": files,
|
|
135
|
+
}
|
|
136
|
+
if data:
|
|
137
|
+
if data_encoding == "form":
|
|
138
|
+
request_kwargs["data"] = data
|
|
139
|
+
elif data_encoding == "json":
|
|
140
|
+
request_kwargs["json"] = data
|
|
141
|
+
else:
|
|
142
|
+
raise ValueError(f"data_encoding must be form or json, got {data_encoding}")
|
|
143
|
+
return request_kwargs
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def default_headers(autorefresh_token: bool = True, **kwargs: Any) -> DefaultHeaders:
|
|
147
|
+
token = get_token(autorefresh_token=autorefresh_token, **kwargs)
|
|
148
|
+
headers: DefaultHeaders = {"User-Agent": USER_AGENT}
|
|
149
|
+
if token:
|
|
150
|
+
headers["Authorization"] = f"Bearer {token}"
|
|
151
|
+
return headers
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def login(username: str, password: str, **kwargs: Any) -> str:
|
|
155
|
+
"""
|
|
156
|
+
Get a token from the API with a username and password
|
|
157
|
+
|
|
158
|
+
Args:
|
|
159
|
+
**kwargs: passed to [api_call][sciop_cli.api.api_call]
|
|
160
|
+
"""
|
|
161
|
+
res = api_call(
|
|
162
|
+
"login", data={"username": username, "password": password}, data_encoding="form", **kwargs
|
|
163
|
+
)
|
|
164
|
+
token: Token = res.json()
|
|
165
|
+
return token["access_token"]
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def get_token(
|
|
169
|
+
username: str | None = None,
|
|
170
|
+
password: str | None = None,
|
|
171
|
+
force: bool = False,
|
|
172
|
+
autorefresh_token: bool = True,
|
|
173
|
+
**kwargs: Any,
|
|
174
|
+
) -> str | None:
|
|
175
|
+
"""
|
|
176
|
+
Get a token from passed username/password or from config.
|
|
177
|
+
|
|
178
|
+
Reuses the already-existing token if present.
|
|
179
|
+
Sets the username, password (if passed) and received token if we get a new one.
|
|
180
|
+
|
|
181
|
+
Args:
|
|
182
|
+
username (str): Username. If None (default),
|
|
183
|
+
use from [get_config][sciop_cli.config.get_config].
|
|
184
|
+
Must be used with `password`
|
|
185
|
+
password (str): Password. If None (default),
|
|
186
|
+
use from [get_config][sciop_cli.config.get_config].
|
|
187
|
+
force (bool): If True, get a new token from the instance,
|
|
188
|
+
even if one is stored in the config.
|
|
189
|
+
**kwargs: passed to [api_call][sciop_cli.api.api_call] if a login request is made
|
|
190
|
+
|
|
191
|
+
Returns:
|
|
192
|
+
str: if token present
|
|
193
|
+
None: if no username/password has been given.
|
|
194
|
+
"""
|
|
195
|
+
config = get_config()
|
|
196
|
+
if config.token and not force:
|
|
197
|
+
return config.token.get_secret_value()
|
|
198
|
+
|
|
199
|
+
if not username and config.username:
|
|
200
|
+
username = config.username
|
|
201
|
+
if not password and config.password:
|
|
202
|
+
password = config.password.get_secret_value()
|
|
203
|
+
|
|
204
|
+
if username and password and autorefresh_token:
|
|
205
|
+
token = login(username=username, password=password, autorefresh_token=False, **kwargs)
|
|
206
|
+
config.token = SecretStr(token)
|
|
207
|
+
config.username = username
|
|
208
|
+
config.password = SecretStr(password)
|
|
209
|
+
set_config(config)
|
|
210
|
+
return token
|
|
211
|
+
else:
|
|
212
|
+
return None
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def claim_next(dataset: str) -> DatasetClaim | None:
|
|
216
|
+
res = api_call(f"claims/datasets/{dataset}/next", do_raise=False)
|
|
217
|
+
if res.status_code == 404 and "no unclaimed" in res.text.lower():
|
|
218
|
+
return None
|
|
219
|
+
res.raise_for_status()
|
|
220
|
+
claim = cast(DatasetClaim, res.json())
|
|
221
|
+
return DatasetClaim(**claim)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def upload_torrent(path: Path, timeout: float | None = 60.0) -> TorrentFile:
|
|
225
|
+
with open(path, "rb") as f:
|
|
226
|
+
response = api_call("torrents", files={"file": f}, timeout=timeout)
|
|
227
|
+
tf = cast(TorrentFile, response.json())
|
|
228
|
+
return TorrentFile(**tf)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def create_upload(
|
|
232
|
+
dataset: str,
|
|
233
|
+
dataset_parts: list[str] | None = None,
|
|
234
|
+
torrent_path: Path | None = None,
|
|
235
|
+
infohash: str | None = None,
|
|
236
|
+
method: str | Path | None = None,
|
|
237
|
+
description: str | Path | None = None,
|
|
238
|
+
timeout: float | None = 60.0,
|
|
239
|
+
) -> Upload:
|
|
240
|
+
"""
|
|
241
|
+
Create an upload for a dataset or dataset parts.
|
|
242
|
+
|
|
243
|
+
Must pass either a path to an existing .torrent file, or an infohash of a torrent
|
|
244
|
+
that has already been uploaded to the configured sciop instance.
|
|
245
|
+
|
|
246
|
+
Args:
|
|
247
|
+
dataset (str): Slug of the dataset to upload to
|
|
248
|
+
torrent_path (Path): Path to the .torrent file to upload - pass either this or `infohash`
|
|
249
|
+
infohash (str): Infohash of a torrent that has already been uploaded -
|
|
250
|
+
pass either this or `torrent_path`
|
|
251
|
+
method (str | Path | None): A description of how this upload was created,
|
|
252
|
+
Supports markdown as a path or string.
|
|
253
|
+
description (str | Path | None): Any description of the contents of the upload
|
|
254
|
+
that are not contained within the dataset description, e.g. additional structure,
|
|
255
|
+
if this is some sub-component of a dataset, and so on
|
|
256
|
+
timeout (float | None): Timeout in seconds to wait for the upload to complete.
|
|
257
|
+
Defaults to 60 seconds since processing uploads can take some time,
|
|
258
|
+
explicitly pass `None` to use `config.request_timeout`
|
|
259
|
+
"""
|
|
260
|
+
if torrent_path and infohash or not (torrent_path or infohash):
|
|
261
|
+
raise ValueError("Must pass torrent path OR infohash")
|
|
262
|
+
elif torrent_path:
|
|
263
|
+
tf = upload_torrent(path=torrent_path, timeout=timeout)
|
|
264
|
+
infohash = tf["v1_infohash"] if tf["version"] == "v1" else tf["v2_infohash"]
|
|
265
|
+
|
|
266
|
+
kwargs: dict[str, str | list[str] | None] = {
|
|
267
|
+
"infohash": infohash,
|
|
268
|
+
"dataset_slug": dataset,
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
if method:
|
|
272
|
+
kwargs["method"] = path_or_str(method)
|
|
273
|
+
if description:
|
|
274
|
+
kwargs["description"] = path_or_str(description)
|
|
275
|
+
if dataset_parts:
|
|
276
|
+
kwargs["part_slugs"] = dataset_parts
|
|
277
|
+
|
|
278
|
+
res = api_call("uploads", data=kwargs, timeout=timeout)
|
|
279
|
+
ul = cast(Upload, res.json())
|
|
280
|
+
return Upload(**ul)
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import json as json_
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
import click
|
|
5
|
+
from pydantic import SecretStr
|
|
6
|
+
from rich import print as pprint
|
|
7
|
+
from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
|
|
8
|
+
|
|
9
|
+
from sciop_cli.api import claim_next as claim_next_api
|
|
10
|
+
from sciop_cli.api import create_upload
|
|
11
|
+
from sciop_cli.api import login as api_login
|
|
12
|
+
from sciop_cli.config import get_config, set_config
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@click.command("login")
|
|
16
|
+
@click.option("-u", "--username", prompt=True, help="Username", required=True)
|
|
17
|
+
@click.option("-p", "--password", prompt=True, hide_input=True, help="Password", required=True)
|
|
18
|
+
@click.option("-i", "--instance-url", help="Instance to log in to, if None, sciop.net")
|
|
19
|
+
def login(username: str, password: str, instance_url: str | None = None) -> None:
|
|
20
|
+
"""
|
|
21
|
+
Login to sciop.net (or another instance), storing the password in a keyring
|
|
22
|
+
and getting a token
|
|
23
|
+
"""
|
|
24
|
+
token = api_login(username, password, instance_url=instance_url)
|
|
25
|
+
cfg = get_config()
|
|
26
|
+
cfg.username = username
|
|
27
|
+
cfg.password = SecretStr(password)
|
|
28
|
+
cfg.token = SecretStr(token)
|
|
29
|
+
set_config(cfg)
|
|
30
|
+
click.echo(f"Logged in as {username} ;)")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@click.command("upload")
|
|
34
|
+
@click.option("-d", "--dataset", help="Slug of the target dataset", required=True)
|
|
35
|
+
@click.option(
|
|
36
|
+
"-p",
|
|
37
|
+
"--part",
|
|
38
|
+
help="Slug of the part, or parts for the upload (optional)."
|
|
39
|
+
"Can be used multiple times to upload to multiple dataset parts",
|
|
40
|
+
multiple=True,
|
|
41
|
+
)
|
|
42
|
+
@click.option(
|
|
43
|
+
"-t",
|
|
44
|
+
"--torrent",
|
|
45
|
+
help="Path to the torrent file to upload",
|
|
46
|
+
required=False,
|
|
47
|
+
type=click.Path(),
|
|
48
|
+
default=None,
|
|
49
|
+
)
|
|
50
|
+
@click.option(
|
|
51
|
+
"-i",
|
|
52
|
+
"--infohash",
|
|
53
|
+
help="If no path is passed, an infohash of an already-uploaded torrent",
|
|
54
|
+
required=False,
|
|
55
|
+
default=None,
|
|
56
|
+
)
|
|
57
|
+
@click.option(
|
|
58
|
+
"-m",
|
|
59
|
+
"--method",
|
|
60
|
+
help="A description of how this upload was created. Supports markdown as a path or string.",
|
|
61
|
+
)
|
|
62
|
+
@click.option(
|
|
63
|
+
"-d",
|
|
64
|
+
"--description",
|
|
65
|
+
help="Any description of the contents of the upload that are not contained within "
|
|
66
|
+
"the dataset description, "
|
|
67
|
+
"e.g. additional structure, if this is some sub-component of a dataset, and so on. "
|
|
68
|
+
"Supports markdown as a path or string.",
|
|
69
|
+
)
|
|
70
|
+
def upload(
|
|
71
|
+
dataset: str,
|
|
72
|
+
part: list[str] | None = None,
|
|
73
|
+
torrent: Path | None = None,
|
|
74
|
+
infohash: str | None = None,
|
|
75
|
+
method: str | None = None,
|
|
76
|
+
description: str | None = None,
|
|
77
|
+
) -> None:
|
|
78
|
+
"""
|
|
79
|
+
Create an upload for a dataset or dataset parts.
|
|
80
|
+
|
|
81
|
+
Must pass either a path to an existing .torrent file, or an infohash of a torrent
|
|
82
|
+
that has already been uploaded to the configured sciop instance.
|
|
83
|
+
"""
|
|
84
|
+
progress = Progress(SpinnerColumn(), TimeElapsedColumn(), TextColumn("{task.description}"))
|
|
85
|
+
with progress:
|
|
86
|
+
progress.add_task(description="Uploading torrent...", total=None)
|
|
87
|
+
upload = create_upload(
|
|
88
|
+
dataset=dataset,
|
|
89
|
+
dataset_parts=part,
|
|
90
|
+
infohash=infohash,
|
|
91
|
+
method=method,
|
|
92
|
+
description=description,
|
|
93
|
+
)
|
|
94
|
+
pprint(upload)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@click.group("claims")
|
|
98
|
+
def claims() -> None:
|
|
99
|
+
"""Claims that we are scraping some data as part of a Quest :)"""
|
|
100
|
+
pass
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@claims.command("next")
|
|
104
|
+
@click.option("-d", "--dataset", required=True, help="Dataset to claim (as its slug)")
|
|
105
|
+
@click.option("-j", "--json", is_flag=True, default=False, help="Print response as json to stdout")
|
|
106
|
+
def next_claim(dataset: str, json: bool = False) -> None:
|
|
107
|
+
"""
|
|
108
|
+
Get the next unclaimed, un-uploaded dataset part for a dataset, printing it
|
|
109
|
+
"""
|
|
110
|
+
res = claim_next_api(dataset)
|
|
111
|
+
if json:
|
|
112
|
+
click.echo(json_.dumps(res))
|
|
113
|
+
else:
|
|
114
|
+
click.echo(res)
|
|
@@ -2,6 +2,7 @@ import logging
|
|
|
2
2
|
|
|
3
3
|
import click
|
|
4
4
|
|
|
5
|
+
from sciop_cli.cli.api import claims, login
|
|
5
6
|
from sciop_cli.cli.manifest import manifest
|
|
6
7
|
from sciop_cli.cli.torrent import torrent
|
|
7
8
|
|
|
@@ -19,5 +20,10 @@ def cli(debug: bool) -> None:
|
|
|
19
20
|
logging.basicConfig(level=logging.INFO)
|
|
20
21
|
|
|
21
22
|
|
|
23
|
+
# top-level API commands
|
|
24
|
+
cli.add_command(login)
|
|
25
|
+
cli.add_command(claims)
|
|
26
|
+
|
|
27
|
+
# command groups
|
|
22
28
|
cli.add_command(torrent)
|
|
23
29
|
cli.add_command(manifest)
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
from logging import getLogger
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Optional, Self
|
|
4
|
+
|
|
5
|
+
import keyring
|
|
6
|
+
import yaml
|
|
7
|
+
from platformdirs import PlatformDirs
|
|
8
|
+
from pydantic import SecretStr, model_validator
|
|
9
|
+
from pydantic_settings import (
|
|
10
|
+
BaseSettings,
|
|
11
|
+
PydanticBaseSettingsSource,
|
|
12
|
+
SettingsConfigDict,
|
|
13
|
+
YamlConfigSettingsSource,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
_dirs = PlatformDirs("sciop_cli", "sciop", ensure_exists=True)
|
|
17
|
+
_global_config = Path(_dirs.user_config_dir) / "sciop_cli.yaml"
|
|
18
|
+
_config: Optional["Config"] = None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def get_config() -> "Config":
|
|
22
|
+
global _config
|
|
23
|
+
if _config is None:
|
|
24
|
+
_config = Config()
|
|
25
|
+
return _config
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def set_config(cfg: "Config") -> "Config":
|
|
29
|
+
"""
|
|
30
|
+
Set config, dumping to the global config yaml file.
|
|
31
|
+
|
|
32
|
+
If a password is present, first try to save it in the keychain
|
|
33
|
+
and exclude it from the dump.
|
|
34
|
+
|
|
35
|
+
if we can't for some reason, dump it with a warning
|
|
36
|
+
"""
|
|
37
|
+
global _config
|
|
38
|
+
_config = cfg
|
|
39
|
+
|
|
40
|
+
logger = getLogger("sciop_cli.config")
|
|
41
|
+
if cfg.password and cfg.username:
|
|
42
|
+
try:
|
|
43
|
+
keyring.set_password("sciop_cli", cfg.username, cfg.password.get_secret_value())
|
|
44
|
+
dumped = cfg.model_dump(exclude={"password"})
|
|
45
|
+
except Exception as e:
|
|
46
|
+
logger.warning(
|
|
47
|
+
"Password present, but could not save in keyring. "
|
|
48
|
+
"Dumping in plaintext, which is insecure!\n"
|
|
49
|
+
f"Got exception: {e}"
|
|
50
|
+
)
|
|
51
|
+
dumped = cfg.model_dump()
|
|
52
|
+
dumped["password"] = dumped["password"].get_secret_value()
|
|
53
|
+
else:
|
|
54
|
+
dumped = cfg.model_dump()
|
|
55
|
+
|
|
56
|
+
if dumped.get("token") and isinstance(dumped["token"], SecretStr):
|
|
57
|
+
dumped["token"] = dumped["token"].get_secret_value()
|
|
58
|
+
|
|
59
|
+
with open(_global_config, "w") as f:
|
|
60
|
+
yaml.safe_dump(dumped, f)
|
|
61
|
+
logger.debug(f"Dumped config to {_global_config}")
|
|
62
|
+
return cfg
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class Config(BaseSettings):
|
|
66
|
+
"""
|
|
67
|
+
Environmental config for cli commands.
|
|
68
|
+
|
|
69
|
+
Keep this light - just what we need to run cli commands,
|
|
70
|
+
don't want this to be a major point of interaction.
|
|
71
|
+
|
|
72
|
+
Values can be set in a .env file with a SCIOP_CLI_ prefix,
|
|
73
|
+
a `sciop_cli.yaml` file in cwd,
|
|
74
|
+
or the `sciop_cli` user config dir provided by
|
|
75
|
+
[platformdirs](https://github.com/tox-dev/platformdirs)
|
|
76
|
+
(e.g. `~/.config/sciop_cli/sciop_cli.yaml`)
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
instance_url: str = "https://sciop.net"
|
|
80
|
+
username: str | None = None
|
|
81
|
+
password: SecretStr | None = None
|
|
82
|
+
request_timeout: float = 5
|
|
83
|
+
"""Default timeout to use in API requests."""
|
|
84
|
+
token: SecretStr | None = None
|
|
85
|
+
"""Expiring jwt, stored in config to reuse between cli calls"""
|
|
86
|
+
|
|
87
|
+
model_config = SettingsConfigDict(
|
|
88
|
+
env_file=".env",
|
|
89
|
+
env_file_encoding="utf-8",
|
|
90
|
+
env_prefix="sciop_cli_",
|
|
91
|
+
env_nested_delimiter="__",
|
|
92
|
+
extra="ignore",
|
|
93
|
+
nested_model_default_partial_update=True,
|
|
94
|
+
yaml_file="sciop_cli.yaml",
|
|
95
|
+
validate_assignment=True,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
@model_validator(mode="before")
|
|
99
|
+
@classmethod
|
|
100
|
+
def password_from_keychain(cls, value: dict | Self) -> dict | Self:
|
|
101
|
+
"""Try to get a password from keychain, if not provided"""
|
|
102
|
+
try:
|
|
103
|
+
if isinstance(value, dict) and value.get("username") and not value.get("password"):
|
|
104
|
+
maybe_password = keyring.get_password("sciop_cli", value["username"])
|
|
105
|
+
if maybe_password:
|
|
106
|
+
value["password"] = SecretStr(maybe_password)
|
|
107
|
+
elif isinstance(value, cls) and value.username and value.password is None:
|
|
108
|
+
maybe_password = keyring.get_password("sciop_cli", value.username)
|
|
109
|
+
if maybe_password:
|
|
110
|
+
value.password = SecretStr(maybe_password)
|
|
111
|
+
except Exception:
|
|
112
|
+
# e.g. no supported keyring backends available
|
|
113
|
+
return value
|
|
114
|
+
return value
|
|
115
|
+
|
|
116
|
+
@classmethod
|
|
117
|
+
def settings_customise_sources(
|
|
118
|
+
cls,
|
|
119
|
+
settings_cls: type[BaseSettings],
|
|
120
|
+
init_settings: PydanticBaseSettingsSource,
|
|
121
|
+
env_settings: PydanticBaseSettingsSource,
|
|
122
|
+
dotenv_settings: PydanticBaseSettingsSource,
|
|
123
|
+
file_secret_settings: PydanticBaseSettingsSource,
|
|
124
|
+
) -> tuple[PydanticBaseSettingsSource, ...]:
|
|
125
|
+
"""
|
|
126
|
+
Read from the following sources,
|
|
127
|
+
in order such that later sources in the list override earlier sources
|
|
128
|
+
|
|
129
|
+
- `{config_dir}/sciop_cli.yaml`
|
|
130
|
+
- `sciop_cli.yaml` (in cwd)
|
|
131
|
+
- `.env` (in cwd)
|
|
132
|
+
- environment variables prefixed with `SCIOP_`
|
|
133
|
+
- arguments passed on config object initialization
|
|
134
|
+
|
|
135
|
+
See [pydantic settings docs](https://docs.pydantic.dev/latest/concepts/pydantic_settings/#customise-settings-sources)
|
|
136
|
+
"""
|
|
137
|
+
global_source = YamlConfigSettingsSource(settings_cls, yaml_file=_global_config)
|
|
138
|
+
return (
|
|
139
|
+
init_settings,
|
|
140
|
+
env_settings,
|
|
141
|
+
dotenv_settings,
|
|
142
|
+
YamlConfigSettingsSource(settings_cls),
|
|
143
|
+
global_source,
|
|
144
|
+
)
|