sciop-cli 0.1.4__tar.gz → 0.1.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/PKG-INFO +7 -6
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/pyproject.toml +24 -7
- sciop_cli-0.1.5/src/sciop_cli/api.py +272 -0
- sciop_cli-0.1.5/src/sciop_cli/cli/api.py +114 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/cli/main.py +6 -0
- sciop_cli-0.1.5/src/sciop_cli/config.py +144 -0
- sciop_cli-0.1.5/src/sciop_cli/models/__init__.py +8 -0
- sciop_cli-0.1.5/src/sciop_cli/models/sciop.py +49 -0
- sciop_cli-0.1.5/tests/conftest.py +49 -0
- sciop_cli-0.1.5/tests/fixtures/__init__.py +17 -0
- sciop_cli-0.1.5/tests/fixtures/base.py +12 -0
- sciop_cli-0.1.5/tests/fixtures/fabricators.py +82 -0
- sciop_cli-0.1.5/tests/test_api.py +132 -0
- sciop_cli-0.1.5/tests/test_cli/test_api.py +46 -0
- sciop_cli-0.1.5/tests/test_config.py +55 -0
- sciop_cli-0.1.4/src/sciop_cli/file.py +0 -40
- sciop_cli-0.1.4/tests/.DS_Store +0 -0
- sciop_cli-0.1.4/tests/conftest.py +0 -3
- sciop_cli-0.1.4/tests/test_cli/__init__.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/LICENSE +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/README.md +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/__init__.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/__main__.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/cli/__init__.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/cli/manifest.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/cli/torrent.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/const.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/data.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/exceptions.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/schema.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/torrent.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/types.py +0 -0
- {sciop_cli-0.1.4/src/sciop_cli/models → sciop_cli-0.1.5/tests}/__init__.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/tests/data/basic/part_1/subdir/part_1 +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/tests/data/basic/part_2/subdir/part_2 +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/tests/data/basic_hybrid.torrent +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/tests/data/basic_v2.torrent +0 -0
- {sciop_cli-0.1.4/tests → sciop_cli-0.1.5/tests/test_cli}/__init__.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/tests/test_cli/test_torrent.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/tests/test_data.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/tests/test_temp.py +0 -0
- {sciop_cli-0.1.4 → sciop_cli-0.1.5}/tests/test_torrent.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: sciop-cli
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.5
|
|
4
4
|
Summary: A CLI for sciop
|
|
5
5
|
Author-Email: cosmicexplorer <dmcC2@hypnicjerk.ai>, sneakers-the-rat <sneakers-the-rat@protonmail.com>
|
|
6
6
|
License: EUPL-1.2
|
|
@@ -9,21 +9,22 @@ Requires-Dist: libtorrent>=2.0.11
|
|
|
9
9
|
Requires-Dist: click>=8.1.8
|
|
10
10
|
Requires-Dist: click-option-group>=0.5.6
|
|
11
11
|
Requires-Dist: pydantic>=2.10.6
|
|
12
|
-
Requires-Dist: pandera[mypy]>=0.23.1
|
|
13
12
|
Requires-Dist: bencode-rs>=0.0.7
|
|
14
|
-
Requires-Dist: platformdirs>=4.3.
|
|
13
|
+
Requires-Dist: platformdirs>=4.3.8
|
|
15
14
|
Requires-Dist: requests>=2.32.3
|
|
16
15
|
Requires-Dist: tqdm>=4.67.1
|
|
17
16
|
Requires-Dist: rich>=14.0.0
|
|
18
17
|
Requires-Dist: humanize>=4.12.2
|
|
19
|
-
|
|
20
|
-
Requires-Dist:
|
|
18
|
+
Requires-Dist: pydantic-settings>=2.9.1
|
|
19
|
+
Requires-Dist: keyring>=25.6.0
|
|
20
|
+
Requires-Dist: pyyaml>=6.0.2
|
|
21
21
|
Provides-Extra: mypy
|
|
22
22
|
Requires-Dist: mypy>=1.15; extra == "mypy"
|
|
23
23
|
Requires-Dist: types-tqdm>=4.67.0.20250401; extra == "mypy"
|
|
24
24
|
Requires-Dist: types-requests>=2.32.0.20250328; extra == "mypy"
|
|
25
|
+
Requires-Dist: types-PyYAML>=6.0.12.20250516; extra == "mypy"
|
|
25
26
|
Provides-Extra: dev
|
|
26
|
-
Requires-Dist: sciop-cli[mypy
|
|
27
|
+
Requires-Dist: sciop-cli[mypy]; extra == "dev"
|
|
27
28
|
Requires-Dist: ruff>=0.9.7; extra == "dev"
|
|
28
29
|
Requires-Dist: black>=25.1.0; extra == "dev"
|
|
29
30
|
Description-Content-Type: text/markdown
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sciop-cli"
|
|
3
|
-
|
|
3
|
+
dynamic = []
|
|
4
4
|
description = "A CLI for sciop"
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "cosmicexplorer", email = "dmcC2@hypnicjerk.ai" },
|
|
@@ -11,31 +11,32 @@ dependencies = [
|
|
|
11
11
|
"click>=8.1.8",
|
|
12
12
|
"click-option-group>=0.5.6",
|
|
13
13
|
"pydantic>=2.10.6",
|
|
14
|
-
"pandera[mypy]>=0.23.1",
|
|
15
14
|
"bencode-rs>=0.0.7",
|
|
16
|
-
"platformdirs>=4.3.
|
|
15
|
+
"platformdirs>=4.3.8",
|
|
17
16
|
"requests>=2.32.3",
|
|
18
17
|
"tqdm>=4.67.1",
|
|
19
18
|
"rich>=14.0.0",
|
|
20
19
|
"humanize>=4.12.2",
|
|
20
|
+
"pydantic-settings>=2.9.1",
|
|
21
|
+
"keyring>=25.6.0",
|
|
22
|
+
"pyyaml>=6.0.2",
|
|
21
23
|
]
|
|
22
24
|
requires-python = ">=3.11"
|
|
23
25
|
readme = "README.md"
|
|
26
|
+
version = "0.1.5"
|
|
24
27
|
|
|
25
28
|
[project.license]
|
|
26
29
|
text = "EUPL-1.2"
|
|
27
30
|
|
|
28
31
|
[project.optional-dependencies]
|
|
29
|
-
test = [
|
|
30
|
-
"pytest>=8.3.4",
|
|
31
|
-
]
|
|
32
32
|
mypy = [
|
|
33
33
|
"mypy>=1.15",
|
|
34
34
|
"types-tqdm>=4.67.0.20250401",
|
|
35
35
|
"types-requests>=2.32.0.20250328",
|
|
36
|
+
"types-PyYAML>=6.0.12.20250516",
|
|
36
37
|
]
|
|
37
38
|
dev = [
|
|
38
|
-
"sciop-cli[
|
|
39
|
+
"sciop-cli[mypy]",
|
|
39
40
|
"ruff>=0.9.7",
|
|
40
41
|
"black>=25.1.0",
|
|
41
42
|
]
|
|
@@ -43,6 +44,15 @@ dev = [
|
|
|
43
44
|
[project.scripts]
|
|
44
45
|
sciop-cli = "sciop_cli.cli.main:cli"
|
|
45
46
|
|
|
47
|
+
[dependency-groups]
|
|
48
|
+
test = [
|
|
49
|
+
"pytest>=8.3.4",
|
|
50
|
+
"sciop[test] @ git+https://codeberg.org/Safeguarding/sciop.git@main",
|
|
51
|
+
"pytest-asyncio>=1.0.0",
|
|
52
|
+
"httpx>=0.28.1",
|
|
53
|
+
"keyrings-alt>=5.0.2",
|
|
54
|
+
]
|
|
55
|
+
|
|
46
56
|
[build-system]
|
|
47
57
|
requires = [
|
|
48
58
|
"pdm-backend",
|
|
@@ -69,6 +79,11 @@ composite = [
|
|
|
69
79
|
"ruff check --fix",
|
|
70
80
|
]
|
|
71
81
|
|
|
82
|
+
[tool.pdm.version]
|
|
83
|
+
source = "scm"
|
|
84
|
+
tag_filter = "v*"
|
|
85
|
+
tag_regex = "v(?P<version>([1-9][0-9]*!)?(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*))*((a|b|c|rc)(0|[1-9][0-9]*))?(\\.post(0|[1-9][0-9]*))?(\\.dev(0|[1-9][0-9]*))?$)$"
|
|
86
|
+
|
|
72
87
|
[tool.ruff]
|
|
73
88
|
target-version = "py311"
|
|
74
89
|
include = [
|
|
@@ -151,4 +166,6 @@ pretty = true
|
|
|
151
166
|
color_output = true
|
|
152
167
|
|
|
153
168
|
[tool.pytest.ini_options]
|
|
169
|
+
asyncio_mode = "auto"
|
|
154
170
|
markers = " v1: tests for v1 torrents\n v2: tests for v2 torrents\n hybrid: tests for hybrid torrents\n"
|
|
171
|
+
asyncio_default_fixture_loop_scope = "session"
|
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Code to work with the sciop API :)
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from importlib.metadata import version
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, Literal, TypedDict, cast
|
|
8
|
+
from urllib.parse import urljoin
|
|
9
|
+
|
|
10
|
+
import requests
|
|
11
|
+
from pydantic import SecretStr
|
|
12
|
+
|
|
13
|
+
from sciop_cli.config import Config, get_config, set_config
|
|
14
|
+
from sciop_cli.models import DatasetClaim, Token, TorrentFile, Upload
|
|
15
|
+
|
|
16
|
+
HTTP_METHOD = Literal["GET", "POST", "PUT", "DELETE"]
|
|
17
|
+
USER_AGENT = f"sciop-cli ({version('sciop-cli')})"
|
|
18
|
+
|
|
19
|
+
DefaultHeaders = TypedDict("DefaultHeaders", {"Authorization": str, "User-Agent": str}, total=False)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def path_or_str(value: Path | str) -> str:
|
|
23
|
+
"""
|
|
24
|
+
handle args that can either be contained in a path or passed as a string
|
|
25
|
+
"""
|
|
26
|
+
if isinstance(value, Path):
|
|
27
|
+
return value.read_text()
|
|
28
|
+
try:
|
|
29
|
+
path = Path(value)
|
|
30
|
+
except Exception:
|
|
31
|
+
return value
|
|
32
|
+
|
|
33
|
+
if path.exists():
|
|
34
|
+
return path.read_text()
|
|
35
|
+
else:
|
|
36
|
+
return str(value)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def api_call(
|
|
40
|
+
path: str,
|
|
41
|
+
data: dict | None = None,
|
|
42
|
+
method: HTTP_METHOD = "POST",
|
|
43
|
+
data_encoding: Literal["form", "json"] = "json",
|
|
44
|
+
headers: dict | None = None,
|
|
45
|
+
files: dict | None = None,
|
|
46
|
+
instance_url: str | None = None,
|
|
47
|
+
api_prefix: Literal["/api/v1", False] = "/api/v1",
|
|
48
|
+
autorefresh_token: bool = True,
|
|
49
|
+
do_raise: bool = True,
|
|
50
|
+
) -> requests.Response:
|
|
51
|
+
"""
|
|
52
|
+
Generic request wrapper around sciop API requests.
|
|
53
|
+
|
|
54
|
+
Args:
|
|
55
|
+
path (str): The path beneath the instance url to call.
|
|
56
|
+
Includes the `api_prefix` by default,
|
|
57
|
+
and handles leading `/`'s with [urljoin][urllib.parse.urljoin]
|
|
58
|
+
so e.g. one can just use `"login"` to call `{instance_url}/api/v1/login`.
|
|
59
|
+
data (dict | None): The data to send in a POST request.
|
|
60
|
+
method (HTTP_METHOD): The HTTP method to use.
|
|
61
|
+
headers (dict | None): Extra headers to add to a request.
|
|
62
|
+
These override and are merged with the default headers from
|
|
63
|
+
[default_headers][sciop_cli.api.default_headers].
|
|
64
|
+
files (dict | None): Extra files to send in a POST request.
|
|
65
|
+
data_encoding (Literal["form", "json"]): If `json`, use request's `json=` parameter,
|
|
66
|
+
which sets `Content-Type`. If `form`, use `data=`, which defaults to formencoded data.
|
|
67
|
+
instance_url (str): Instance url to call to. Default is https://sciop.net
|
|
68
|
+
api_prefix (str): API prefix after the `instance_url`,
|
|
69
|
+
either `/api/v1` or `False` to access paths at the base url.
|
|
70
|
+
autorefresh_token (bool): If `True` (default), try and get a token if we have credentials.
|
|
71
|
+
Made `False` e.g. to avoid infinite recursion while logging in,
|
|
72
|
+
or if it is otherwise desirable to use only the preconfigured token.
|
|
73
|
+
do_raise (bool): If `True` (default), raise on status codes >=400
|
|
74
|
+
"""
|
|
75
|
+
config = get_config()
|
|
76
|
+
request_kwargs = _api_call_kwargs(
|
|
77
|
+
path=path,
|
|
78
|
+
config=config,
|
|
79
|
+
data=data,
|
|
80
|
+
method=method,
|
|
81
|
+
data_encoding=data_encoding,
|
|
82
|
+
headers=headers,
|
|
83
|
+
files=files,
|
|
84
|
+
instance_url=instance_url,
|
|
85
|
+
api_prefix=api_prefix,
|
|
86
|
+
autorefresh_token=autorefresh_token,
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
res = requests.request(**request_kwargs)
|
|
90
|
+
if res.status_code == 403 and config.token and not autorefresh_token:
|
|
91
|
+
# try once to get a new token if it just expired
|
|
92
|
+
token = get_token(force=True)
|
|
93
|
+
request_kwargs["headers"]["Authorization"] = f"Bearer {token}"
|
|
94
|
+
res = requests.request(**request_kwargs)
|
|
95
|
+
|
|
96
|
+
if do_raise:
|
|
97
|
+
res.raise_for_status()
|
|
98
|
+
return res
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _api_call_kwargs(
|
|
102
|
+
path: str,
|
|
103
|
+
config: Config,
|
|
104
|
+
data: dict | None = None,
|
|
105
|
+
method: HTTP_METHOD = "POST",
|
|
106
|
+
data_encoding: Literal["form", "json"] = "json",
|
|
107
|
+
headers: dict | None = None,
|
|
108
|
+
files: dict | None = None,
|
|
109
|
+
instance_url: str | None = None,
|
|
110
|
+
api_prefix: Literal["/api/v1", False] = "/api/v1",
|
|
111
|
+
autorefresh_token: bool = True,
|
|
112
|
+
) -> dict:
|
|
113
|
+
default = default_headers(
|
|
114
|
+
instance_url=instance_url, headers=headers, autorefresh_token=autorefresh_token
|
|
115
|
+
)
|
|
116
|
+
merged_headers = default | headers if headers else default
|
|
117
|
+
|
|
118
|
+
if instance_url is None:
|
|
119
|
+
instance_url = config.instance_url
|
|
120
|
+
|
|
121
|
+
if api_prefix:
|
|
122
|
+
path = "/".join([segment.strip("/") for segment in [api_prefix, path]])
|
|
123
|
+
|
|
124
|
+
url = urljoin(instance_url, path)
|
|
125
|
+
request_kwargs = {
|
|
126
|
+
"method": method,
|
|
127
|
+
"url": url,
|
|
128
|
+
"headers": merged_headers,
|
|
129
|
+
"timeout": config.request_timeout,
|
|
130
|
+
"files": files,
|
|
131
|
+
}
|
|
132
|
+
if data:
|
|
133
|
+
if data_encoding == "form":
|
|
134
|
+
request_kwargs["data"] = data
|
|
135
|
+
elif data_encoding == "json":
|
|
136
|
+
request_kwargs["json"] = data
|
|
137
|
+
else:
|
|
138
|
+
raise ValueError(f"data_encoding must be form or json, got {data_encoding}")
|
|
139
|
+
return request_kwargs
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def default_headers(autorefresh_token: bool = True, **kwargs: Any) -> DefaultHeaders:
|
|
143
|
+
token = get_token(autorefresh_token=autorefresh_token, **kwargs)
|
|
144
|
+
headers: DefaultHeaders = {"User-Agent": USER_AGENT}
|
|
145
|
+
if token:
|
|
146
|
+
headers["Authorization"] = f"Bearer {token}"
|
|
147
|
+
return headers
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def login(username: str, password: str, **kwargs: Any) -> str:
|
|
151
|
+
"""
|
|
152
|
+
Get a token from the API with a username and password
|
|
153
|
+
|
|
154
|
+
Args:
|
|
155
|
+
**kwargs: passed to [api_call][sciop_cli.api.api_call]
|
|
156
|
+
"""
|
|
157
|
+
res = api_call(
|
|
158
|
+
"login", data={"username": username, "password": password}, data_encoding="form", **kwargs
|
|
159
|
+
)
|
|
160
|
+
token: Token = res.json()
|
|
161
|
+
return token["access_token"]
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def get_token(
|
|
165
|
+
username: str | None = None,
|
|
166
|
+
password: str | None = None,
|
|
167
|
+
force: bool = False,
|
|
168
|
+
autorefresh_token: bool = True,
|
|
169
|
+
**kwargs: Any,
|
|
170
|
+
) -> str | None:
|
|
171
|
+
"""
|
|
172
|
+
Get a token from passed username/password or from config.
|
|
173
|
+
|
|
174
|
+
Reuses the already-existing token if present.
|
|
175
|
+
Sets the username, password (if passed) and received token if we get a new one.
|
|
176
|
+
|
|
177
|
+
Args:
|
|
178
|
+
username (str): Username. If None (default),
|
|
179
|
+
use from [get_config][sciop_cli.config.get_config].
|
|
180
|
+
Must be used with `password`
|
|
181
|
+
password (str): Password. If None (default),
|
|
182
|
+
use from [get_config][sciop_cli.config.get_config].
|
|
183
|
+
force (bool): If True, get a new token from the instance,
|
|
184
|
+
even if one is stored in the config.
|
|
185
|
+
**kwargs: passed to [api_call][sciop_cli.api.api_call] if a login request is made
|
|
186
|
+
|
|
187
|
+
Returns:
|
|
188
|
+
str: if token present
|
|
189
|
+
None: if no username/password has been given.
|
|
190
|
+
"""
|
|
191
|
+
config = get_config()
|
|
192
|
+
if config.token and not force:
|
|
193
|
+
return config.token.get_secret_value()
|
|
194
|
+
|
|
195
|
+
if not username and config.username:
|
|
196
|
+
username = config.username
|
|
197
|
+
if not password and config.password:
|
|
198
|
+
password = config.password.get_secret_value()
|
|
199
|
+
|
|
200
|
+
if username and password and autorefresh_token:
|
|
201
|
+
token = login(username=username, password=password, autorefresh_token=False, **kwargs)
|
|
202
|
+
config.token = SecretStr(token)
|
|
203
|
+
config.username = username
|
|
204
|
+
config.password = SecretStr(password)
|
|
205
|
+
set_config(config)
|
|
206
|
+
return token
|
|
207
|
+
else:
|
|
208
|
+
return None
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def claim_next(dataset: str) -> DatasetClaim | None:
|
|
212
|
+
res = api_call(f"claims/datasets/{dataset}/next", do_raise=False)
|
|
213
|
+
if res.status_code == 404 and "no unclaimed" in res.text.lower():
|
|
214
|
+
return None
|
|
215
|
+
res.raise_for_status()
|
|
216
|
+
claim = cast(DatasetClaim, res.json())
|
|
217
|
+
return DatasetClaim(**claim)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def upload_torrent(path: Path) -> TorrentFile:
|
|
221
|
+
with open(path, "rb") as f:
|
|
222
|
+
response = api_call("torrents", files={"file": f})
|
|
223
|
+
tf = cast(TorrentFile, response.json())
|
|
224
|
+
return TorrentFile(**tf)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def create_upload(
|
|
228
|
+
dataset: str,
|
|
229
|
+
dataset_parts: list[str] | None = None,
|
|
230
|
+
torrent_path: Path | None = None,
|
|
231
|
+
infohash: str | None = None,
|
|
232
|
+
method: str | Path | None = None,
|
|
233
|
+
description: str | Path | None = None,
|
|
234
|
+
) -> Upload:
|
|
235
|
+
"""
|
|
236
|
+
Create an upload for a dataset or dataset parts.
|
|
237
|
+
|
|
238
|
+
Must pass either a path to an existing .torrent file, or an infohash of a torrent
|
|
239
|
+
that has already been uploaded to the configured sciop instance.
|
|
240
|
+
|
|
241
|
+
Args:
|
|
242
|
+
dataset (str): Slug of the dataset to upload to
|
|
243
|
+
torrent_path (Path): Path to the .torrent file to upload - pass either this or `infohash`
|
|
244
|
+
infohash (str): Infohash of a torrent that has already been uploaded -
|
|
245
|
+
pass either this or `torrent_path`
|
|
246
|
+
method (str | Path | None): A description of how this upload was created,
|
|
247
|
+
Supports markdown as a path or string.
|
|
248
|
+
description (str | Path | None): Any description of the contents of the upload
|
|
249
|
+
that are not contained within the dataset description, e.g. additional structure,
|
|
250
|
+
if this is some sub-component of a dataset, and so on
|
|
251
|
+
"""
|
|
252
|
+
if torrent_path and infohash or not (torrent_path or infohash):
|
|
253
|
+
raise ValueError("Must pass torrent path OR infohash")
|
|
254
|
+
elif torrent_path:
|
|
255
|
+
tf = upload_torrent(path=torrent_path)
|
|
256
|
+
infohash = tf["v1_infohash"] if tf["version"] == "v1" else tf["v2_infohash"]
|
|
257
|
+
|
|
258
|
+
kwargs: dict[str, str | list[str] | None] = {
|
|
259
|
+
"infohash": infohash,
|
|
260
|
+
"dataset_slug": dataset,
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
if method:
|
|
264
|
+
kwargs["method"] = path_or_str(method)
|
|
265
|
+
if description:
|
|
266
|
+
kwargs["description"] = path_or_str(description)
|
|
267
|
+
if dataset_parts:
|
|
268
|
+
kwargs["part_slugs"] = dataset_parts
|
|
269
|
+
|
|
270
|
+
res = api_call("uploads", data=kwargs)
|
|
271
|
+
ul = cast(Upload, res.json())
|
|
272
|
+
return Upload(**ul)
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import json as json_
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
import click
|
|
5
|
+
from pydantic import SecretStr
|
|
6
|
+
from rich import print as pprint
|
|
7
|
+
from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
|
|
8
|
+
|
|
9
|
+
from sciop_cli.api import claim_next as claim_next_api
|
|
10
|
+
from sciop_cli.api import create_upload
|
|
11
|
+
from sciop_cli.api import login as api_login
|
|
12
|
+
from sciop_cli.config import get_config, set_config
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@click.command("login")
|
|
16
|
+
@click.option("-u", "--username", prompt=True, help="Username", required=True)
|
|
17
|
+
@click.option("-p", "--password", prompt=True, hide_input=True, help="Password", required=True)
|
|
18
|
+
@click.option("-i", "--instance-url", help="Instance to log in to, if None, sciop.net")
|
|
19
|
+
def login(username: str, password: str, instance_url: str | None = None) -> None:
|
|
20
|
+
"""
|
|
21
|
+
Login to sciop.net (or another instance), storing the password in a keyring
|
|
22
|
+
and getting a token
|
|
23
|
+
"""
|
|
24
|
+
token = api_login(username, password, instance_url=instance_url)
|
|
25
|
+
cfg = get_config()
|
|
26
|
+
cfg.username = username
|
|
27
|
+
cfg.password = SecretStr(password)
|
|
28
|
+
cfg.token = SecretStr(token)
|
|
29
|
+
set_config(cfg)
|
|
30
|
+
click.echo(f"Logged in as {username} ;)")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@click.command("upload")
|
|
34
|
+
@click.option("-d", "--dataset", help="Slug of the target dataset", required=True)
|
|
35
|
+
@click.option(
|
|
36
|
+
"-p",
|
|
37
|
+
"--part",
|
|
38
|
+
help="Slug of the part, or parts for the upload (optional)."
|
|
39
|
+
"Can be used multiple times to upload to multiple dataset parts",
|
|
40
|
+
multiple=True,
|
|
41
|
+
)
|
|
42
|
+
@click.option(
|
|
43
|
+
"-t",
|
|
44
|
+
"--torrent",
|
|
45
|
+
help="Path to the torrent file to upload",
|
|
46
|
+
required=False,
|
|
47
|
+
type=click.Path(),
|
|
48
|
+
default=None,
|
|
49
|
+
)
|
|
50
|
+
@click.option(
|
|
51
|
+
"-i",
|
|
52
|
+
"--infohash",
|
|
53
|
+
help="If no path is passed, an infohash of an already-uploaded torrent",
|
|
54
|
+
required=False,
|
|
55
|
+
default=None,
|
|
56
|
+
)
|
|
57
|
+
@click.option(
|
|
58
|
+
"-m",
|
|
59
|
+
"--method",
|
|
60
|
+
help="A description of how this upload was created. Supports markdown as a path or string.",
|
|
61
|
+
)
|
|
62
|
+
@click.option(
|
|
63
|
+
"-d",
|
|
64
|
+
"--description",
|
|
65
|
+
help="Any description of the contents of the upload that are not contained within "
|
|
66
|
+
"the dataset description, "
|
|
67
|
+
"e.g. additional structure, if this is some sub-component of a dataset, and so on. "
|
|
68
|
+
"Supports markdown as a path or string.",
|
|
69
|
+
)
|
|
70
|
+
def upload(
|
|
71
|
+
dataset: str,
|
|
72
|
+
part: list[str] | None = None,
|
|
73
|
+
torrent: Path | None = None,
|
|
74
|
+
infohash: str | None = None,
|
|
75
|
+
method: str | None = None,
|
|
76
|
+
description: str | None = None,
|
|
77
|
+
) -> None:
|
|
78
|
+
"""
|
|
79
|
+
Create an upload for a dataset or dataset parts.
|
|
80
|
+
|
|
81
|
+
Must pass either a path to an existing .torrent file, or an infohash of a torrent
|
|
82
|
+
that has already been uploaded to the configured sciop instance.
|
|
83
|
+
"""
|
|
84
|
+
progress = Progress(SpinnerColumn(), TimeElapsedColumn(), TextColumn("{task.description}"))
|
|
85
|
+
with progress:
|
|
86
|
+
progress.add_task(description="Uploading torrent...", total=None)
|
|
87
|
+
upload = create_upload(
|
|
88
|
+
dataset=dataset,
|
|
89
|
+
dataset_parts=part,
|
|
90
|
+
infohash=infohash,
|
|
91
|
+
method=method,
|
|
92
|
+
description=description,
|
|
93
|
+
)
|
|
94
|
+
pprint(upload)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@click.group("claims")
|
|
98
|
+
def claims() -> None:
|
|
99
|
+
"""Claims that we are scraping some data as part of a Quest :)"""
|
|
100
|
+
pass
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@claims.command("next")
|
|
104
|
+
@click.option("-d", "--dataset", required=True, help="Dataset to claim (as its slug)")
|
|
105
|
+
@click.option("-j", "--json", is_flag=True, default=False, help="Print response as json to stdout")
|
|
106
|
+
def next_claim(dataset: str, json: bool = False) -> None:
|
|
107
|
+
"""
|
|
108
|
+
Get the next unclaimed, un-uploaded dataset part for a dataset, printing it
|
|
109
|
+
"""
|
|
110
|
+
res = claim_next_api(dataset)
|
|
111
|
+
if json:
|
|
112
|
+
click.echo(json_.dumps(res))
|
|
113
|
+
else:
|
|
114
|
+
click.echo(res)
|
|
@@ -2,6 +2,7 @@ import logging
|
|
|
2
2
|
|
|
3
3
|
import click
|
|
4
4
|
|
|
5
|
+
from sciop_cli.cli.api import claims, login
|
|
5
6
|
from sciop_cli.cli.manifest import manifest
|
|
6
7
|
from sciop_cli.cli.torrent import torrent
|
|
7
8
|
|
|
@@ -19,5 +20,10 @@ def cli(debug: bool) -> None:
|
|
|
19
20
|
logging.basicConfig(level=logging.INFO)
|
|
20
21
|
|
|
21
22
|
|
|
23
|
+
# top-level API commands
|
|
24
|
+
cli.add_command(login)
|
|
25
|
+
cli.add_command(claims)
|
|
26
|
+
|
|
27
|
+
# command groups
|
|
22
28
|
cli.add_command(torrent)
|
|
23
29
|
cli.add_command(manifest)
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
from logging import getLogger
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Optional, Self
|
|
4
|
+
|
|
5
|
+
import keyring
|
|
6
|
+
import yaml
|
|
7
|
+
from platformdirs import PlatformDirs
|
|
8
|
+
from pydantic import SecretStr, model_validator
|
|
9
|
+
from pydantic_settings import (
|
|
10
|
+
BaseSettings,
|
|
11
|
+
PydanticBaseSettingsSource,
|
|
12
|
+
SettingsConfigDict,
|
|
13
|
+
YamlConfigSettingsSource,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
_dirs = PlatformDirs("sciop_cli", "sciop", ensure_exists=True)
|
|
17
|
+
_global_config = Path(_dirs.user_config_dir) / "sciop_cli.yaml"
|
|
18
|
+
_config: Optional["Config"] = None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def get_config() -> "Config":
|
|
22
|
+
global _config
|
|
23
|
+
if _config is None:
|
|
24
|
+
_config = Config()
|
|
25
|
+
return _config
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def set_config(cfg: "Config") -> "Config":
|
|
29
|
+
"""
|
|
30
|
+
Set config, dumping to the global config yaml file.
|
|
31
|
+
|
|
32
|
+
If a password is present, first try to save it in the keychain
|
|
33
|
+
and exclude it from the dump.
|
|
34
|
+
|
|
35
|
+
if we can't for some reason, dump it with a warning
|
|
36
|
+
"""
|
|
37
|
+
global _config
|
|
38
|
+
_config = cfg
|
|
39
|
+
|
|
40
|
+
logger = getLogger("sciop_cli.config")
|
|
41
|
+
if cfg.password and cfg.username:
|
|
42
|
+
try:
|
|
43
|
+
keyring.set_password("sciop_cli", cfg.username, cfg.password.get_secret_value())
|
|
44
|
+
dumped = cfg.model_dump(exclude={"password"})
|
|
45
|
+
except Exception as e:
|
|
46
|
+
logger.warning(
|
|
47
|
+
"Password present, but could not save in keyring. "
|
|
48
|
+
"Dumping in plaintext, which is insecure!\n"
|
|
49
|
+
f"Got exception: {e}"
|
|
50
|
+
)
|
|
51
|
+
dumped = cfg.model_dump()
|
|
52
|
+
dumped["password"] = dumped["password"].get_secret_value()
|
|
53
|
+
else:
|
|
54
|
+
dumped = cfg.model_dump()
|
|
55
|
+
|
|
56
|
+
if dumped.get("token") and isinstance(dumped["token"], SecretStr):
|
|
57
|
+
dumped["token"] = dumped["token"].get_secret_value()
|
|
58
|
+
|
|
59
|
+
with open(_global_config, "w") as f:
|
|
60
|
+
yaml.safe_dump(dumped, f)
|
|
61
|
+
logger.debug(f"Dumped config to {_global_config}")
|
|
62
|
+
return cfg
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class Config(BaseSettings):
|
|
66
|
+
"""
|
|
67
|
+
Environmental config for cli commands.
|
|
68
|
+
|
|
69
|
+
Keep this light - just what we need to run cli commands,
|
|
70
|
+
don't want this to be a major point of interaction.
|
|
71
|
+
|
|
72
|
+
Values can be set in a .env file with a SCIOP_CLI_ prefix,
|
|
73
|
+
a `sciop_cli.yaml` file in cwd,
|
|
74
|
+
or the `sciop_cli` user config dir provided by
|
|
75
|
+
[platformdirs](https://github.com/tox-dev/platformdirs)
|
|
76
|
+
(e.g. `~/.config/sciop_cli/sciop_cli.yaml`)
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
instance_url: str = "https://sciop.net"
|
|
80
|
+
username: str | None = None
|
|
81
|
+
password: SecretStr | None = None
|
|
82
|
+
request_timeout: float = 5
|
|
83
|
+
"""Default timeout to use in API requests."""
|
|
84
|
+
token: SecretStr | None = None
|
|
85
|
+
"""Expiring jwt, stored in config to reuse between cli calls"""
|
|
86
|
+
|
|
87
|
+
model_config = SettingsConfigDict(
|
|
88
|
+
env_file=".env",
|
|
89
|
+
env_file_encoding="utf-8",
|
|
90
|
+
env_prefix="sciop_cli_",
|
|
91
|
+
env_nested_delimiter="__",
|
|
92
|
+
extra="ignore",
|
|
93
|
+
nested_model_default_partial_update=True,
|
|
94
|
+
yaml_file="sciop_cli.yaml",
|
|
95
|
+
validate_assignment=True,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
@model_validator(mode="before")
|
|
99
|
+
@classmethod
|
|
100
|
+
def password_from_keychain(cls, value: dict | Self) -> dict | Self:
|
|
101
|
+
"""Try to get a password from keychain, if not provided"""
|
|
102
|
+
try:
|
|
103
|
+
if isinstance(value, dict) and value.get("username") and not value.get("password"):
|
|
104
|
+
maybe_password = keyring.get_password("sciop_cli", value["username"])
|
|
105
|
+
if maybe_password:
|
|
106
|
+
value["password"] = SecretStr(maybe_password)
|
|
107
|
+
elif isinstance(value, cls) and value.username and value.password is None:
|
|
108
|
+
maybe_password = keyring.get_password("sciop_cli", value.username)
|
|
109
|
+
if maybe_password:
|
|
110
|
+
value.password = SecretStr(maybe_password)
|
|
111
|
+
except Exception:
|
|
112
|
+
# e.g. no supported keyring backends available
|
|
113
|
+
return value
|
|
114
|
+
return value
|
|
115
|
+
|
|
116
|
+
@classmethod
|
|
117
|
+
def settings_customise_sources(
|
|
118
|
+
cls,
|
|
119
|
+
settings_cls: type[BaseSettings],
|
|
120
|
+
init_settings: PydanticBaseSettingsSource,
|
|
121
|
+
env_settings: PydanticBaseSettingsSource,
|
|
122
|
+
dotenv_settings: PydanticBaseSettingsSource,
|
|
123
|
+
file_secret_settings: PydanticBaseSettingsSource,
|
|
124
|
+
) -> tuple[PydanticBaseSettingsSource, ...]:
|
|
125
|
+
"""
|
|
126
|
+
Read from the following sources,
|
|
127
|
+
in order such that later sources in the list override earlier sources
|
|
128
|
+
|
|
129
|
+
- `{config_dir}/sciop_cli.yaml`
|
|
130
|
+
- `sciop_cli.yaml` (in cwd)
|
|
131
|
+
- `.env` (in cwd)
|
|
132
|
+
- environment variables prefixed with `SCIOP_`
|
|
133
|
+
- arguments passed on config object initialization
|
|
134
|
+
|
|
135
|
+
See [pydantic settings docs](https://docs.pydantic.dev/latest/concepts/pydantic_settings/#customise-settings-sources)
|
|
136
|
+
"""
|
|
137
|
+
global_source = YamlConfigSettingsSource(settings_cls, yaml_file=_global_config)
|
|
138
|
+
return (
|
|
139
|
+
init_settings,
|
|
140
|
+
env_settings,
|
|
141
|
+
dotenv_settings,
|
|
142
|
+
YamlConfigSettingsSource(settings_cls),
|
|
143
|
+
global_source,
|
|
144
|
+
)
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Placeholder models that mimic sciop's models,
|
|
3
|
+
until we can split them off into a separate package and single-source them.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
from typing import Literal, NotRequired, TypedDict
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Token(TypedDict):
|
|
11
|
+
access_token: str
|
|
12
|
+
token_type: Literal["bearer"]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class DatasetClaim(TypedDict):
|
|
16
|
+
created_at: datetime
|
|
17
|
+
updated_at: datetime
|
|
18
|
+
dataset: str
|
|
19
|
+
dataset_part: NotRequired[str]
|
|
20
|
+
account: str
|
|
21
|
+
status: Literal["in_progress", "completed"]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class TorrentFile(TypedDict):
|
|
25
|
+
file_name: str
|
|
26
|
+
v1_infohash: NotRequired[str | None]
|
|
27
|
+
v2_infohash: NotRequired[str | None]
|
|
28
|
+
short_hash: str
|
|
29
|
+
version: Literal["v1", "v2", "hybrid"]
|
|
30
|
+
total_size: int
|
|
31
|
+
piece_size: int
|
|
32
|
+
torrent_size: int
|
|
33
|
+
announce_urls: list[str]
|
|
34
|
+
seeders: NotRequired[int | None]
|
|
35
|
+
leechers: NotRequired[int | None]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class Upload(TypedDict):
|
|
39
|
+
created_at: datetime
|
|
40
|
+
updated_at: datetime
|
|
41
|
+
is_approved: bool
|
|
42
|
+
is_removed: bool
|
|
43
|
+
method: str | None
|
|
44
|
+
description: str | None
|
|
45
|
+
dataset: str
|
|
46
|
+
dataset_parts: NotRequired[list[str]]
|
|
47
|
+
seeders: NotRequired[int | None]
|
|
48
|
+
leechers: NotRequired[int | None]
|
|
49
|
+
torrent: TorrentFile
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
from sciop.testing.fixtures.config import set_config as sciop_set_config # noqa: F401
|
|
5
|
+
from sciop.testing.fixtures.db import * # noqa: F401
|
|
6
|
+
from sciop.testing.fixtures.server import * # noqa: F401
|
|
7
|
+
|
|
8
|
+
from sciop_cli.config import Config
|
|
9
|
+
|
|
10
|
+
from .fixtures import *
|
|
11
|
+
|
|
12
|
+
DATA_DIR = Path(__file__).parent / "data"
|
|
13
|
+
|
|
14
|
+
pytest_plugins = ("sciop.testing.plugin",)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@pytest.fixture(autouse=True, scope="session")
|
|
18
|
+
def session_monkeypatch_config(
|
|
19
|
+
monkeypatch_session: pytest.MonkeyPatch, tmp_path_factory: pytest.TempPathFactory
|
|
20
|
+
) -> None:
|
|
21
|
+
"""sessionwide baseline for our config..."""
|
|
22
|
+
from sciop_cli import config
|
|
23
|
+
from sciop_cli.config import Config, set_config
|
|
24
|
+
|
|
25
|
+
session_dir = tmp_path_factory.mktemp("sciop_cli_session")
|
|
26
|
+
monkeypatch_session.setattr(config, "_global_config", session_dir / "sciop_cli_test.yaml")
|
|
27
|
+
|
|
28
|
+
new_config = Config(
|
|
29
|
+
username=None,
|
|
30
|
+
password=None,
|
|
31
|
+
token=None,
|
|
32
|
+
instance_url="http://127.0.0.1:8080",
|
|
33
|
+
request_timeout=10,
|
|
34
|
+
)
|
|
35
|
+
set_config(new_config)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@pytest.fixture()
|
|
39
|
+
def fresh_config(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Config:
|
|
40
|
+
from sciop_cli import config
|
|
41
|
+
from sciop_cli.config import Config, set_config
|
|
42
|
+
|
|
43
|
+
monkeypatch.setattr(config, "_global_config", tmp_path / "sciop_cli_test.yaml")
|
|
44
|
+
|
|
45
|
+
new_config = Config(
|
|
46
|
+
username=None, password=None, token=None, instance_url="http://127.0.0.1:8080"
|
|
47
|
+
)
|
|
48
|
+
set_config(new_config)
|
|
49
|
+
return new_config
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from .base import monkeypatch_session
|
|
2
|
+
from .fabricators import account, as_admin, claims_setup, config_logged_in
|
|
3
|
+
|
|
4
|
+
# from .sciop import client, engine, patch_config, run_server, session
|
|
5
|
+
|
|
6
|
+
__all__ = [
|
|
7
|
+
"account",
|
|
8
|
+
"as_admin",
|
|
9
|
+
"claims_setup",
|
|
10
|
+
# "client",
|
|
11
|
+
"config_logged_in",
|
|
12
|
+
# "engine",
|
|
13
|
+
"monkeypatch_session",
|
|
14
|
+
# "patch_config",
|
|
15
|
+
# "session",
|
|
16
|
+
# "run_server",
|
|
17
|
+
]
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
from typing import TYPE_CHECKING
|
|
2
|
+
|
|
3
|
+
import httpx
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from sciop_cli.config import Config, set_config
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from sciop.models import Account, Dataset
|
|
10
|
+
from sciop.testing.server import UvicornSyncServer
|
|
11
|
+
from sqlmodel import Session
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@pytest.fixture
|
|
15
|
+
async def account(run_server_sync: "UvicornSyncServer") -> tuple[str, str]:
|
|
16
|
+
# say we have set our login info through the cli...
|
|
17
|
+
username = "testuser"
|
|
18
|
+
password = "testuser12345"
|
|
19
|
+
async with httpx.AsyncClient() as client:
|
|
20
|
+
account = await client.post(
|
|
21
|
+
"http://localhost:8080/api/v1/register",
|
|
22
|
+
data={"username": username, "password": password},
|
|
23
|
+
)
|
|
24
|
+
assert account.status_code == 200
|
|
25
|
+
return username, password
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@pytest.fixture()
|
|
29
|
+
async def config_logged_in(account: tuple[str, str], fresh_config: Config) -> Config:
|
|
30
|
+
# say we have set our login info through the cli...
|
|
31
|
+
username, password = account
|
|
32
|
+
fresh_config.username = username
|
|
33
|
+
fresh_config.password = password
|
|
34
|
+
set_config(fresh_config)
|
|
35
|
+
return fresh_config
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@pytest.fixture()
|
|
39
|
+
async def as_admin(
|
|
40
|
+
fresh_config: Config, session: "Session", run_server_sync: "UvicornSyncServer"
|
|
41
|
+
) -> "Account":
|
|
42
|
+
from sciop.api.auth import get_password_hash
|
|
43
|
+
from sciop.models import Account, Scope
|
|
44
|
+
|
|
45
|
+
username = "admin"
|
|
46
|
+
password = "adminadmin12"
|
|
47
|
+
hashed_pw = get_password_hash(password)
|
|
48
|
+
admin = Scope.get_item("admin", session=session)
|
|
49
|
+
account = Account(username=username, hashed_password=hashed_pw, scopes=[admin])
|
|
50
|
+
session.add(account)
|
|
51
|
+
session.commit()
|
|
52
|
+
session.refresh(account)
|
|
53
|
+
fresh_config.username = username
|
|
54
|
+
fresh_config.password = password
|
|
55
|
+
set_config(fresh_config)
|
|
56
|
+
return account
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@pytest.fixture()
|
|
60
|
+
def claims_setup(as_admin: "Account", session: "Session") -> "Dataset":
|
|
61
|
+
"""
|
|
62
|
+
Crappy in-place data setup pending exporting sciop fixtures in a testing module
|
|
63
|
+
"""
|
|
64
|
+
from sciop.models import Dataset, DatasetPart, Tag
|
|
65
|
+
|
|
66
|
+
tag = Tag(tag="default")
|
|
67
|
+
ds = Dataset(
|
|
68
|
+
title="default",
|
|
69
|
+
slug="default",
|
|
70
|
+
publisher="example",
|
|
71
|
+
tags=[tag],
|
|
72
|
+
account=as_admin,
|
|
73
|
+
is_approved=True,
|
|
74
|
+
)
|
|
75
|
+
parts = [
|
|
76
|
+
DatasetPart(part_slug=letter, is_approved=True, dataset=ds) for letter in ("a", "b", "c")
|
|
77
|
+
]
|
|
78
|
+
ds.parts = parts
|
|
79
|
+
session.add(ds)
|
|
80
|
+
session.commit()
|
|
81
|
+
session.refresh(ds)
|
|
82
|
+
return ds
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
from typing import TYPE_CHECKING
|
|
2
|
+
|
|
3
|
+
import httpx
|
|
4
|
+
import pytest
|
|
5
|
+
from pydantic import TypeAdapter
|
|
6
|
+
|
|
7
|
+
from sciop_cli.api import claim_next, create_upload, get_token, login, upload_torrent
|
|
8
|
+
from sciop_cli.config import Config, get_config
|
|
9
|
+
from sciop_cli.models.sciop import TorrentFile, Upload
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from sciop.models import Account
|
|
13
|
+
from torf import Torrent
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@pytest.mark.asyncio
|
|
17
|
+
async def test_login(fresh_config: Config, run_server_sync):
|
|
18
|
+
"""
|
|
19
|
+
The login command logs us in!
|
|
20
|
+
"""
|
|
21
|
+
username = "testuser"
|
|
22
|
+
password = "testuser12345"
|
|
23
|
+
async with httpx.AsyncClient() as client:
|
|
24
|
+
account = await client.post(
|
|
25
|
+
"http://127.0.0.1:8080/api/v1/register",
|
|
26
|
+
data={"username": username, "password": password},
|
|
27
|
+
timeout=10,
|
|
28
|
+
)
|
|
29
|
+
assert account.status_code == 200
|
|
30
|
+
|
|
31
|
+
token = login(username, password)
|
|
32
|
+
assert isinstance(token, str)
|
|
33
|
+
|
|
34
|
+
res = await client.get(
|
|
35
|
+
"http://localhost:8080/self/", headers={"Authorization": f"Bearer {token}"}
|
|
36
|
+
)
|
|
37
|
+
assert res.status_code == 200
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@pytest.mark.asyncio
|
|
41
|
+
async def test_get_token(config_logged_in: Config, run_server_sync):
|
|
42
|
+
"""
|
|
43
|
+
get_token gets a token and sets it in the config
|
|
44
|
+
"""
|
|
45
|
+
cfg = config_logged_in
|
|
46
|
+
assert cfg.token is None
|
|
47
|
+
assert get_config().token is None
|
|
48
|
+
|
|
49
|
+
# then we should get a token and set it in the config
|
|
50
|
+
token = get_token()
|
|
51
|
+
assert token is not None
|
|
52
|
+
assert get_config().token.get_secret_value() == token
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@pytest.mark.asyncio
|
|
56
|
+
async def test_get_next_claim(as_admin: "Account", run_server_sync, claims_setup):
|
|
57
|
+
"""
|
|
58
|
+
Should be able to call get next claim!
|
|
59
|
+
"""
|
|
60
|
+
responses = [claim_next("default") for _ in range(3)]
|
|
61
|
+
for letter, response in zip(("a", "b", "c"), responses):
|
|
62
|
+
assert response["dataset_part"] == letter
|
|
63
|
+
assert response["status"] == "in_progress"
|
|
64
|
+
|
|
65
|
+
assert claim_next("default") is None
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@pytest.mark.asyncio
|
|
69
|
+
async def test_upload_torrent(as_admin: "Account", run_server_sync, torrent, tmp_path):
|
|
70
|
+
"""
|
|
71
|
+
We can upload a torrent!
|
|
72
|
+
"""
|
|
73
|
+
torrent: Torrent = torrent()
|
|
74
|
+
path = tmp_path / "test.torrent"
|
|
75
|
+
torrent.write(path)
|
|
76
|
+
res = upload_torrent(path)
|
|
77
|
+
|
|
78
|
+
adapter = TypeAdapter(TorrentFile)
|
|
79
|
+
adapter.validate_python(res)
|
|
80
|
+
assert res["v1_infohash"] == torrent.infohash
|
|
81
|
+
assert res["file_name"] == "test.torrent"
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@pytest.mark.asyncio
|
|
85
|
+
@pytest.mark.parametrize("mode", ["infohash", "path"])
|
|
86
|
+
async def test_create_upload(
|
|
87
|
+
as_admin: "Account",
|
|
88
|
+
dataset,
|
|
89
|
+
run_server_sync,
|
|
90
|
+
torrent,
|
|
91
|
+
tmp_path,
|
|
92
|
+
mode: str,
|
|
93
|
+
):
|
|
94
|
+
"""
|
|
95
|
+
We can create an upload item, either by passing an infohash or a path
|
|
96
|
+
"""
|
|
97
|
+
slug = "default"
|
|
98
|
+
path = tmp_path / "test.torrent"
|
|
99
|
+
|
|
100
|
+
dataset(slug=slug)
|
|
101
|
+
torrent: Torrent = torrent()
|
|
102
|
+
torrent.write(path)
|
|
103
|
+
|
|
104
|
+
if mode == "infohash":
|
|
105
|
+
upload_torrent(path)
|
|
106
|
+
ul = create_upload(dataset=slug, infohash=torrent.infohash)
|
|
107
|
+
elif mode == "path":
|
|
108
|
+
ul = create_upload(dataset=slug, torrent_path=path)
|
|
109
|
+
else:
|
|
110
|
+
raise ValueError()
|
|
111
|
+
|
|
112
|
+
adapter = TypeAdapter(Upload)
|
|
113
|
+
adapter.validate_python(ul)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@pytest.mark.asyncio
|
|
117
|
+
@pytest.mark.parametrize("parts", [["part-1"], ["part-1", "part-2"]])
|
|
118
|
+
async def test_create_upload_part(
|
|
119
|
+
as_admin: "Account", dataset, run_server_sync, torrent, tmp_path, parts: list[str]
|
|
120
|
+
):
|
|
121
|
+
"""
|
|
122
|
+
We can upload to a dataset part
|
|
123
|
+
"""
|
|
124
|
+
slug = "default"
|
|
125
|
+
path = tmp_path / "test.torrent"
|
|
126
|
+
|
|
127
|
+
dataset(slug=slug, parts=[{"part_slug": p} for p in parts])
|
|
128
|
+
torrent: Torrent = torrent()
|
|
129
|
+
torrent.write(path)
|
|
130
|
+
|
|
131
|
+
ul = create_upload(dataset=slug, dataset_parts=parts, torrent_path=path)
|
|
132
|
+
assert ul["dataset_parts"] == parts
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from typing import TYPE_CHECKING
|
|
3
|
+
|
|
4
|
+
from click.testing import CliRunner
|
|
5
|
+
|
|
6
|
+
from sciop_cli.cli.api import login as login_cli
|
|
7
|
+
from sciop_cli.cli.api import next_claim as next_claim_cli
|
|
8
|
+
from sciop_cli.config import Config, get_config
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from .sciop import UvicornSyncServer
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def test_login(
|
|
15
|
+
run_server_sync: "UvicornSyncServer", fresh_config: Config, account: tuple[str, str]
|
|
16
|
+
):
|
|
17
|
+
"""
|
|
18
|
+
We can log in through the CLI!
|
|
19
|
+
"""
|
|
20
|
+
cfg = get_config()
|
|
21
|
+
assert cfg.username is None
|
|
22
|
+
assert cfg.password is None
|
|
23
|
+
runner = CliRunner()
|
|
24
|
+
|
|
25
|
+
res = runner.invoke(login_cli, ["--username", account[0], "--password", account[1]])
|
|
26
|
+
assert res.exit_code == 0
|
|
27
|
+
cfg2: Config = get_config()
|
|
28
|
+
assert cfg2.username == account[0]
|
|
29
|
+
assert cfg2.password.get_secret_value() == account[1]
|
|
30
|
+
assert cfg2.token is not None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_get_next_claim(as_admin, claims_setup, run_server_sync):
|
|
34
|
+
"""
|
|
35
|
+
After logging in, we can get the next dataset part and print it as json
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
runner = CliRunner()
|
|
39
|
+
|
|
40
|
+
res = runner.invoke(next_claim_cli, ["--dataset", "default", "--json"])
|
|
41
|
+
assert res.exit_code == 0
|
|
42
|
+
out = json.loads(res.stdout.splitlines()[-1])
|
|
43
|
+
assert out["status"] == "in_progress"
|
|
44
|
+
assert out["dataset"] == "default"
|
|
45
|
+
assert out["dataset_part"] == "a"
|
|
46
|
+
assert out["account"] == "admin"
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
import keyring
|
|
4
|
+
import pytest
|
|
5
|
+
import yaml
|
|
6
|
+
|
|
7
|
+
from sciop_cli import config
|
|
8
|
+
from sciop_cli.config import Config, set_config
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def test_instantiation_defaults(monkeypatch, tmp_path):
|
|
12
|
+
"""We should be able to instantiate config without setting any values"""
|
|
13
|
+
monkeypatch.chdir(tmp_path)
|
|
14
|
+
monkeypatch.setattr(config, "_global_config", Path(tmp_path) / "sciop_cli_test.yaml")
|
|
15
|
+
_ = Config()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def test_password_from_keychain(request: pytest.FixtureRequest):
|
|
19
|
+
"""We can get a password from the keychain automatically"""
|
|
20
|
+
username = f"__test__{request.node.name}"
|
|
21
|
+
expected = "__testpassword__"
|
|
22
|
+
keyring.set_password("sciop_cli", username, expected)
|
|
23
|
+
|
|
24
|
+
cfg = Config(username=username)
|
|
25
|
+
assert cfg.password
|
|
26
|
+
assert cfg.password.get_secret_value() == expected
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def test_set_config_no_password(monkeypatch, tmp_path: Path, request: pytest.FixtureRequest):
|
|
30
|
+
"""
|
|
31
|
+
When we dump the config, and we have a password, and we can access a keyring,
|
|
32
|
+
we should remove the password from the dump, save it in the keyring.
|
|
33
|
+
"""
|
|
34
|
+
username = f"__test__{request.node.name}"
|
|
35
|
+
expected = "__testpassword__"
|
|
36
|
+
cfg_path = Path(tmp_path) / "sciop_cli_test.yaml"
|
|
37
|
+
monkeypatch.chdir(tmp_path)
|
|
38
|
+
monkeypatch.setattr(config, "_global_config", cfg_path)
|
|
39
|
+
if keyring.get_password("sciop_cli", username):
|
|
40
|
+
keyring.delete_password("sciop_cli", username)
|
|
41
|
+
|
|
42
|
+
assert not keyring.get_password("sciop_cli", username)
|
|
43
|
+
cfg = Config(username=username, password=expected)
|
|
44
|
+
cfg = set_config(cfg)
|
|
45
|
+
|
|
46
|
+
with open(cfg_path) as f:
|
|
47
|
+
dumped = yaml.safe_load(f)
|
|
48
|
+
assert dumped["username"] == username
|
|
49
|
+
assert "password" not in dumped
|
|
50
|
+
assert keyring.get_password("sciop_cli", username) == expected
|
|
51
|
+
|
|
52
|
+
# and we should get it again when we instantiate
|
|
53
|
+
loaded = Config()
|
|
54
|
+
assert loaded.username == username
|
|
55
|
+
assert loaded.password.get_secret_value() == expected
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
from pathlib import Path
|
|
2
|
-
|
|
3
|
-
import pandera as pa
|
|
4
|
-
from pandera.typing import DataFrame, Series
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
class Files(pa.DataFrameModel):
|
|
8
|
-
filename: Series[str]
|
|
9
|
-
directory: Series[str]
|
|
10
|
-
size: Series[int]
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
@pa.check_types
|
|
14
|
-
def read_directory(directory: Path) -> DataFrame[Files]:
|
|
15
|
-
names = []
|
|
16
|
-
dirs = []
|
|
17
|
-
sizes = []
|
|
18
|
-
for f in directory.glob("**/*"):
|
|
19
|
-
if not f.is_file():
|
|
20
|
-
continue
|
|
21
|
-
names.append(f.name)
|
|
22
|
-
dirs.append(str(f.parent))
|
|
23
|
-
sizes.append(f.stat().st_size)
|
|
24
|
-
return DataFrame[Files]({"filename": names, "directory": dirs, "size": sizes})
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
def directory_size(directory: Path) -> int:
|
|
28
|
-
"""Recursive directoy size, in bytes"""
|
|
29
|
-
return sum(f.stat().st_size for f in directory.glob("**/*") if f.is_file())
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
# def should_zip(directory: Path, piece_size: int, n_file_threshold:int = 20) -> bool:
|
|
33
|
-
# """
|
|
34
|
-
# FIXME: nvm just load all the files into pandas and do it vectorized
|
|
35
|
-
# """
|
|
36
|
-
# pass
|
|
37
|
-
# # n_small_files = 0
|
|
38
|
-
# # for f in directory.glob("*"):
|
|
39
|
-
# # if f.is_file() and f.stat().st_size < n_file_threshold:
|
|
40
|
-
# # n_small_files += 1
|
sciop_cli-0.1.4/tests/.DS_Store
DELETED
|
Binary file
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|