sciop-cli 0.1.4__tar.gz → 0.1.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/PKG-INFO +7 -6
  2. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/pyproject.toml +24 -7
  3. sciop_cli-0.1.5/src/sciop_cli/api.py +272 -0
  4. sciop_cli-0.1.5/src/sciop_cli/cli/api.py +114 -0
  5. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/cli/main.py +6 -0
  6. sciop_cli-0.1.5/src/sciop_cli/config.py +144 -0
  7. sciop_cli-0.1.5/src/sciop_cli/models/__init__.py +8 -0
  8. sciop_cli-0.1.5/src/sciop_cli/models/sciop.py +49 -0
  9. sciop_cli-0.1.5/tests/conftest.py +49 -0
  10. sciop_cli-0.1.5/tests/fixtures/__init__.py +17 -0
  11. sciop_cli-0.1.5/tests/fixtures/base.py +12 -0
  12. sciop_cli-0.1.5/tests/fixtures/fabricators.py +82 -0
  13. sciop_cli-0.1.5/tests/test_api.py +132 -0
  14. sciop_cli-0.1.5/tests/test_cli/test_api.py +46 -0
  15. sciop_cli-0.1.5/tests/test_config.py +55 -0
  16. sciop_cli-0.1.4/src/sciop_cli/file.py +0 -40
  17. sciop_cli-0.1.4/tests/.DS_Store +0 -0
  18. sciop_cli-0.1.4/tests/conftest.py +0 -3
  19. sciop_cli-0.1.4/tests/test_cli/__init__.py +0 -0
  20. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/LICENSE +0 -0
  21. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/README.md +0 -0
  22. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/__init__.py +0 -0
  23. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/__main__.py +0 -0
  24. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/cli/__init__.py +0 -0
  25. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/cli/manifest.py +0 -0
  26. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/cli/torrent.py +0 -0
  27. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/const.py +0 -0
  28. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/data.py +0 -0
  29. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/exceptions.py +0 -0
  30. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/schema.py +0 -0
  31. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/torrent.py +0 -0
  32. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/src/sciop_cli/types.py +0 -0
  33. {sciop_cli-0.1.4/src/sciop_cli/models → sciop_cli-0.1.5/tests}/__init__.py +0 -0
  34. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/tests/data/basic/part_1/subdir/part_1 +0 -0
  35. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/tests/data/basic/part_2/subdir/part_2 +0 -0
  36. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/tests/data/basic_hybrid.torrent +0 -0
  37. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/tests/data/basic_v2.torrent +0 -0
  38. {sciop_cli-0.1.4/tests → sciop_cli-0.1.5/tests/test_cli}/__init__.py +0 -0
  39. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/tests/test_cli/test_torrent.py +0 -0
  40. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/tests/test_data.py +0 -0
  41. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/tests/test_temp.py +0 -0
  42. {sciop_cli-0.1.4 → sciop_cli-0.1.5}/tests/test_torrent.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: sciop-cli
3
- Version: 0.1.4
3
+ Version: 0.1.5
4
4
  Summary: A CLI for sciop
5
5
  Author-Email: cosmicexplorer <dmcC2@hypnicjerk.ai>, sneakers-the-rat <sneakers-the-rat@protonmail.com>
6
6
  License: EUPL-1.2
@@ -9,21 +9,22 @@ Requires-Dist: libtorrent>=2.0.11
9
9
  Requires-Dist: click>=8.1.8
10
10
  Requires-Dist: click-option-group>=0.5.6
11
11
  Requires-Dist: pydantic>=2.10.6
12
- Requires-Dist: pandera[mypy]>=0.23.1
13
12
  Requires-Dist: bencode-rs>=0.0.7
14
- Requires-Dist: platformdirs>=4.3.7
13
+ Requires-Dist: platformdirs>=4.3.8
15
14
  Requires-Dist: requests>=2.32.3
16
15
  Requires-Dist: tqdm>=4.67.1
17
16
  Requires-Dist: rich>=14.0.0
18
17
  Requires-Dist: humanize>=4.12.2
19
- Provides-Extra: test
20
- Requires-Dist: pytest>=8.3.4; extra == "test"
18
+ Requires-Dist: pydantic-settings>=2.9.1
19
+ Requires-Dist: keyring>=25.6.0
20
+ Requires-Dist: pyyaml>=6.0.2
21
21
  Provides-Extra: mypy
22
22
  Requires-Dist: mypy>=1.15; extra == "mypy"
23
23
  Requires-Dist: types-tqdm>=4.67.0.20250401; extra == "mypy"
24
24
  Requires-Dist: types-requests>=2.32.0.20250328; extra == "mypy"
25
+ Requires-Dist: types-PyYAML>=6.0.12.20250516; extra == "mypy"
25
26
  Provides-Extra: dev
26
- Requires-Dist: sciop-cli[mypy,test]; extra == "dev"
27
+ Requires-Dist: sciop-cli[mypy]; extra == "dev"
27
28
  Requires-Dist: ruff>=0.9.7; extra == "dev"
28
29
  Requires-Dist: black>=25.1.0; extra == "dev"
29
30
  Description-Content-Type: text/markdown
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "sciop-cli"
3
- version = "0.1.4"
3
+ dynamic = []
4
4
  description = "A CLI for sciop"
5
5
  authors = [
6
6
  { name = "cosmicexplorer", email = "dmcC2@hypnicjerk.ai" },
@@ -11,31 +11,32 @@ dependencies = [
11
11
  "click>=8.1.8",
12
12
  "click-option-group>=0.5.6",
13
13
  "pydantic>=2.10.6",
14
- "pandera[mypy]>=0.23.1",
15
14
  "bencode-rs>=0.0.7",
16
- "platformdirs>=4.3.7",
15
+ "platformdirs>=4.3.8",
17
16
  "requests>=2.32.3",
18
17
  "tqdm>=4.67.1",
19
18
  "rich>=14.0.0",
20
19
  "humanize>=4.12.2",
20
+ "pydantic-settings>=2.9.1",
21
+ "keyring>=25.6.0",
22
+ "pyyaml>=6.0.2",
21
23
  ]
22
24
  requires-python = ">=3.11"
23
25
  readme = "README.md"
26
+ version = "0.1.5"
24
27
 
25
28
  [project.license]
26
29
  text = "EUPL-1.2"
27
30
 
28
31
  [project.optional-dependencies]
29
- test = [
30
- "pytest>=8.3.4",
31
- ]
32
32
  mypy = [
33
33
  "mypy>=1.15",
34
34
  "types-tqdm>=4.67.0.20250401",
35
35
  "types-requests>=2.32.0.20250328",
36
+ "types-PyYAML>=6.0.12.20250516",
36
37
  ]
37
38
  dev = [
38
- "sciop-cli[test,mypy]",
39
+ "sciop-cli[mypy]",
39
40
  "ruff>=0.9.7",
40
41
  "black>=25.1.0",
41
42
  ]
@@ -43,6 +44,15 @@ dev = [
43
44
  [project.scripts]
44
45
  sciop-cli = "sciop_cli.cli.main:cli"
45
46
 
47
+ [dependency-groups]
48
+ test = [
49
+ "pytest>=8.3.4",
50
+ "sciop[test] @ git+https://codeberg.org/Safeguarding/sciop.git@main",
51
+ "pytest-asyncio>=1.0.0",
52
+ "httpx>=0.28.1",
53
+ "keyrings-alt>=5.0.2",
54
+ ]
55
+
46
56
  [build-system]
47
57
  requires = [
48
58
  "pdm-backend",
@@ -69,6 +79,11 @@ composite = [
69
79
  "ruff check --fix",
70
80
  ]
71
81
 
82
+ [tool.pdm.version]
83
+ source = "scm"
84
+ tag_filter = "v*"
85
+ tag_regex = "v(?P<version>([1-9][0-9]*!)?(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*))*((a|b|c|rc)(0|[1-9][0-9]*))?(\\.post(0|[1-9][0-9]*))?(\\.dev(0|[1-9][0-9]*))?$)$"
86
+
72
87
  [tool.ruff]
73
88
  target-version = "py311"
74
89
  include = [
@@ -151,4 +166,6 @@ pretty = true
151
166
  color_output = true
152
167
 
153
168
  [tool.pytest.ini_options]
169
+ asyncio_mode = "auto"
154
170
  markers = " v1: tests for v1 torrents\n v2: tests for v2 torrents\n hybrid: tests for hybrid torrents\n"
171
+ asyncio_default_fixture_loop_scope = "session"
@@ -0,0 +1,272 @@
1
+ """
2
+ Code to work with the sciop API :)
3
+ """
4
+
5
+ from importlib.metadata import version
6
+ from pathlib import Path
7
+ from typing import Any, Literal, TypedDict, cast
8
+ from urllib.parse import urljoin
9
+
10
+ import requests
11
+ from pydantic import SecretStr
12
+
13
+ from sciop_cli.config import Config, get_config, set_config
14
+ from sciop_cli.models import DatasetClaim, Token, TorrentFile, Upload
15
+
16
+ HTTP_METHOD = Literal["GET", "POST", "PUT", "DELETE"]
17
+ USER_AGENT = f"sciop-cli ({version('sciop-cli')})"
18
+
19
+ DefaultHeaders = TypedDict("DefaultHeaders", {"Authorization": str, "User-Agent": str}, total=False)
20
+
21
+
22
+ def path_or_str(value: Path | str) -> str:
23
+ """
24
+ handle args that can either be contained in a path or passed as a string
25
+ """
26
+ if isinstance(value, Path):
27
+ return value.read_text()
28
+ try:
29
+ path = Path(value)
30
+ except Exception:
31
+ return value
32
+
33
+ if path.exists():
34
+ return path.read_text()
35
+ else:
36
+ return str(value)
37
+
38
+
39
+ def api_call(
40
+ path: str,
41
+ data: dict | None = None,
42
+ method: HTTP_METHOD = "POST",
43
+ data_encoding: Literal["form", "json"] = "json",
44
+ headers: dict | None = None,
45
+ files: dict | None = None,
46
+ instance_url: str | None = None,
47
+ api_prefix: Literal["/api/v1", False] = "/api/v1",
48
+ autorefresh_token: bool = True,
49
+ do_raise: bool = True,
50
+ ) -> requests.Response:
51
+ """
52
+ Generic request wrapper around sciop API requests.
53
+
54
+ Args:
55
+ path (str): The path beneath the instance url to call.
56
+ Includes the `api_prefix` by default,
57
+ and handles leading `/`'s with [urljoin][urllib.parse.urljoin]
58
+ so e.g. one can just use `"login"` to call `{instance_url}/api/v1/login`.
59
+ data (dict | None): The data to send in a POST request.
60
+ method (HTTP_METHOD): The HTTP method to use.
61
+ headers (dict | None): Extra headers to add to a request.
62
+ These override and are merged with the default headers from
63
+ [default_headers][sciop_cli.api.default_headers].
64
+ files (dict | None): Extra files to send in a POST request.
65
+ data_encoding (Literal["form", "json"]): If `json`, use request's `json=` parameter,
66
+ which sets `Content-Type`. If `form`, use `data=`, which defaults to formencoded data.
67
+ instance_url (str): Instance url to call to. Default is https://sciop.net
68
+ api_prefix (str): API prefix after the `instance_url`,
69
+ either `/api/v1` or `False` to access paths at the base url.
70
+ autorefresh_token (bool): If `True` (default), try and get a token if we have credentials.
71
+ Made `False` e.g. to avoid infinite recursion while logging in,
72
+ or if it is otherwise desirable to use only the preconfigured token.
73
+ do_raise (bool): If `True` (default), raise on status codes >=400
74
+ """
75
+ config = get_config()
76
+ request_kwargs = _api_call_kwargs(
77
+ path=path,
78
+ config=config,
79
+ data=data,
80
+ method=method,
81
+ data_encoding=data_encoding,
82
+ headers=headers,
83
+ files=files,
84
+ instance_url=instance_url,
85
+ api_prefix=api_prefix,
86
+ autorefresh_token=autorefresh_token,
87
+ )
88
+
89
+ res = requests.request(**request_kwargs)
90
+ if res.status_code == 403 and config.token and not autorefresh_token:
91
+ # try once to get a new token if it just expired
92
+ token = get_token(force=True)
93
+ request_kwargs["headers"]["Authorization"] = f"Bearer {token}"
94
+ res = requests.request(**request_kwargs)
95
+
96
+ if do_raise:
97
+ res.raise_for_status()
98
+ return res
99
+
100
+
101
+ def _api_call_kwargs(
102
+ path: str,
103
+ config: Config,
104
+ data: dict | None = None,
105
+ method: HTTP_METHOD = "POST",
106
+ data_encoding: Literal["form", "json"] = "json",
107
+ headers: dict | None = None,
108
+ files: dict | None = None,
109
+ instance_url: str | None = None,
110
+ api_prefix: Literal["/api/v1", False] = "/api/v1",
111
+ autorefresh_token: bool = True,
112
+ ) -> dict:
113
+ default = default_headers(
114
+ instance_url=instance_url, headers=headers, autorefresh_token=autorefresh_token
115
+ )
116
+ merged_headers = default | headers if headers else default
117
+
118
+ if instance_url is None:
119
+ instance_url = config.instance_url
120
+
121
+ if api_prefix:
122
+ path = "/".join([segment.strip("/") for segment in [api_prefix, path]])
123
+
124
+ url = urljoin(instance_url, path)
125
+ request_kwargs = {
126
+ "method": method,
127
+ "url": url,
128
+ "headers": merged_headers,
129
+ "timeout": config.request_timeout,
130
+ "files": files,
131
+ }
132
+ if data:
133
+ if data_encoding == "form":
134
+ request_kwargs["data"] = data
135
+ elif data_encoding == "json":
136
+ request_kwargs["json"] = data
137
+ else:
138
+ raise ValueError(f"data_encoding must be form or json, got {data_encoding}")
139
+ return request_kwargs
140
+
141
+
142
+ def default_headers(autorefresh_token: bool = True, **kwargs: Any) -> DefaultHeaders:
143
+ token = get_token(autorefresh_token=autorefresh_token, **kwargs)
144
+ headers: DefaultHeaders = {"User-Agent": USER_AGENT}
145
+ if token:
146
+ headers["Authorization"] = f"Bearer {token}"
147
+ return headers
148
+
149
+
150
+ def login(username: str, password: str, **kwargs: Any) -> str:
151
+ """
152
+ Get a token from the API with a username and password
153
+
154
+ Args:
155
+ **kwargs: passed to [api_call][sciop_cli.api.api_call]
156
+ """
157
+ res = api_call(
158
+ "login", data={"username": username, "password": password}, data_encoding="form", **kwargs
159
+ )
160
+ token: Token = res.json()
161
+ return token["access_token"]
162
+
163
+
164
+ def get_token(
165
+ username: str | None = None,
166
+ password: str | None = None,
167
+ force: bool = False,
168
+ autorefresh_token: bool = True,
169
+ **kwargs: Any,
170
+ ) -> str | None:
171
+ """
172
+ Get a token from passed username/password or from config.
173
+
174
+ Reuses the already-existing token if present.
175
+ Sets the username, password (if passed) and received token if we get a new one.
176
+
177
+ Args:
178
+ username (str): Username. If None (default),
179
+ use from [get_config][sciop_cli.config.get_config].
180
+ Must be used with `password`
181
+ password (str): Password. If None (default),
182
+ use from [get_config][sciop_cli.config.get_config].
183
+ force (bool): If True, get a new token from the instance,
184
+ even if one is stored in the config.
185
+ **kwargs: passed to [api_call][sciop_cli.api.api_call] if a login request is made
186
+
187
+ Returns:
188
+ str: if token present
189
+ None: if no username/password has been given.
190
+ """
191
+ config = get_config()
192
+ if config.token and not force:
193
+ return config.token.get_secret_value()
194
+
195
+ if not username and config.username:
196
+ username = config.username
197
+ if not password and config.password:
198
+ password = config.password.get_secret_value()
199
+
200
+ if username and password and autorefresh_token:
201
+ token = login(username=username, password=password, autorefresh_token=False, **kwargs)
202
+ config.token = SecretStr(token)
203
+ config.username = username
204
+ config.password = SecretStr(password)
205
+ set_config(config)
206
+ return token
207
+ else:
208
+ return None
209
+
210
+
211
+ def claim_next(dataset: str) -> DatasetClaim | None:
212
+ res = api_call(f"claims/datasets/{dataset}/next", do_raise=False)
213
+ if res.status_code == 404 and "no unclaimed" in res.text.lower():
214
+ return None
215
+ res.raise_for_status()
216
+ claim = cast(DatasetClaim, res.json())
217
+ return DatasetClaim(**claim)
218
+
219
+
220
+ def upload_torrent(path: Path) -> TorrentFile:
221
+ with open(path, "rb") as f:
222
+ response = api_call("torrents", files={"file": f})
223
+ tf = cast(TorrentFile, response.json())
224
+ return TorrentFile(**tf)
225
+
226
+
227
+ def create_upload(
228
+ dataset: str,
229
+ dataset_parts: list[str] | None = None,
230
+ torrent_path: Path | None = None,
231
+ infohash: str | None = None,
232
+ method: str | Path | None = None,
233
+ description: str | Path | None = None,
234
+ ) -> Upload:
235
+ """
236
+ Create an upload for a dataset or dataset parts.
237
+
238
+ Must pass either a path to an existing .torrent file, or an infohash of a torrent
239
+ that has already been uploaded to the configured sciop instance.
240
+
241
+ Args:
242
+ dataset (str): Slug of the dataset to upload to
243
+ torrent_path (Path): Path to the .torrent file to upload - pass either this or `infohash`
244
+ infohash (str): Infohash of a torrent that has already been uploaded -
245
+ pass either this or `torrent_path`
246
+ method (str | Path | None): A description of how this upload was created,
247
+ Supports markdown as a path or string.
248
+ description (str | Path | None): Any description of the contents of the upload
249
+ that are not contained within the dataset description, e.g. additional structure,
250
+ if this is some sub-component of a dataset, and so on
251
+ """
252
+ if torrent_path and infohash or not (torrent_path or infohash):
253
+ raise ValueError("Must pass torrent path OR infohash")
254
+ elif torrent_path:
255
+ tf = upload_torrent(path=torrent_path)
256
+ infohash = tf["v1_infohash"] if tf["version"] == "v1" else tf["v2_infohash"]
257
+
258
+ kwargs: dict[str, str | list[str] | None] = {
259
+ "infohash": infohash,
260
+ "dataset_slug": dataset,
261
+ }
262
+
263
+ if method:
264
+ kwargs["method"] = path_or_str(method)
265
+ if description:
266
+ kwargs["description"] = path_or_str(description)
267
+ if dataset_parts:
268
+ kwargs["part_slugs"] = dataset_parts
269
+
270
+ res = api_call("uploads", data=kwargs)
271
+ ul = cast(Upload, res.json())
272
+ return Upload(**ul)
@@ -0,0 +1,114 @@
1
+ import json as json_
2
+ from pathlib import Path
3
+
4
+ import click
5
+ from pydantic import SecretStr
6
+ from rich import print as pprint
7
+ from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
8
+
9
+ from sciop_cli.api import claim_next as claim_next_api
10
+ from sciop_cli.api import create_upload
11
+ from sciop_cli.api import login as api_login
12
+ from sciop_cli.config import get_config, set_config
13
+
14
+
15
+ @click.command("login")
16
+ @click.option("-u", "--username", prompt=True, help="Username", required=True)
17
+ @click.option("-p", "--password", prompt=True, hide_input=True, help="Password", required=True)
18
+ @click.option("-i", "--instance-url", help="Instance to log in to, if None, sciop.net")
19
+ def login(username: str, password: str, instance_url: str | None = None) -> None:
20
+ """
21
+ Login to sciop.net (or another instance), storing the password in a keyring
22
+ and getting a token
23
+ """
24
+ token = api_login(username, password, instance_url=instance_url)
25
+ cfg = get_config()
26
+ cfg.username = username
27
+ cfg.password = SecretStr(password)
28
+ cfg.token = SecretStr(token)
29
+ set_config(cfg)
30
+ click.echo(f"Logged in as {username} ;)")
31
+
32
+
33
+ @click.command("upload")
34
+ @click.option("-d", "--dataset", help="Slug of the target dataset", required=True)
35
+ @click.option(
36
+ "-p",
37
+ "--part",
38
+ help="Slug of the part, or parts for the upload (optional)."
39
+ "Can be used multiple times to upload to multiple dataset parts",
40
+ multiple=True,
41
+ )
42
+ @click.option(
43
+ "-t",
44
+ "--torrent",
45
+ help="Path to the torrent file to upload",
46
+ required=False,
47
+ type=click.Path(),
48
+ default=None,
49
+ )
50
+ @click.option(
51
+ "-i",
52
+ "--infohash",
53
+ help="If no path is passed, an infohash of an already-uploaded torrent",
54
+ required=False,
55
+ default=None,
56
+ )
57
+ @click.option(
58
+ "-m",
59
+ "--method",
60
+ help="A description of how this upload was created. Supports markdown as a path or string.",
61
+ )
62
+ @click.option(
63
+ "-d",
64
+ "--description",
65
+ help="Any description of the contents of the upload that are not contained within "
66
+ "the dataset description, "
67
+ "e.g. additional structure, if this is some sub-component of a dataset, and so on. "
68
+ "Supports markdown as a path or string.",
69
+ )
70
+ def upload(
71
+ dataset: str,
72
+ part: list[str] | None = None,
73
+ torrent: Path | None = None,
74
+ infohash: str | None = None,
75
+ method: str | None = None,
76
+ description: str | None = None,
77
+ ) -> None:
78
+ """
79
+ Create an upload for a dataset or dataset parts.
80
+
81
+ Must pass either a path to an existing .torrent file, or an infohash of a torrent
82
+ that has already been uploaded to the configured sciop instance.
83
+ """
84
+ progress = Progress(SpinnerColumn(), TimeElapsedColumn(), TextColumn("{task.description}"))
85
+ with progress:
86
+ progress.add_task(description="Uploading torrent...", total=None)
87
+ upload = create_upload(
88
+ dataset=dataset,
89
+ dataset_parts=part,
90
+ infohash=infohash,
91
+ method=method,
92
+ description=description,
93
+ )
94
+ pprint(upload)
95
+
96
+
97
+ @click.group("claims")
98
+ def claims() -> None:
99
+ """Claims that we are scraping some data as part of a Quest :)"""
100
+ pass
101
+
102
+
103
+ @claims.command("next")
104
+ @click.option("-d", "--dataset", required=True, help="Dataset to claim (as its slug)")
105
+ @click.option("-j", "--json", is_flag=True, default=False, help="Print response as json to stdout")
106
+ def next_claim(dataset: str, json: bool = False) -> None:
107
+ """
108
+ Get the next unclaimed, un-uploaded dataset part for a dataset, printing it
109
+ """
110
+ res = claim_next_api(dataset)
111
+ if json:
112
+ click.echo(json_.dumps(res))
113
+ else:
114
+ click.echo(res)
@@ -2,6 +2,7 @@ import logging
2
2
 
3
3
  import click
4
4
 
5
+ from sciop_cli.cli.api import claims, login
5
6
  from sciop_cli.cli.manifest import manifest
6
7
  from sciop_cli.cli.torrent import torrent
7
8
 
@@ -19,5 +20,10 @@ def cli(debug: bool) -> None:
19
20
  logging.basicConfig(level=logging.INFO)
20
21
 
21
22
 
23
+ # top-level API commands
24
+ cli.add_command(login)
25
+ cli.add_command(claims)
26
+
27
+ # command groups
22
28
  cli.add_command(torrent)
23
29
  cli.add_command(manifest)
@@ -0,0 +1,144 @@
1
+ from logging import getLogger
2
+ from pathlib import Path
3
+ from typing import Optional, Self
4
+
5
+ import keyring
6
+ import yaml
7
+ from platformdirs import PlatformDirs
8
+ from pydantic import SecretStr, model_validator
9
+ from pydantic_settings import (
10
+ BaseSettings,
11
+ PydanticBaseSettingsSource,
12
+ SettingsConfigDict,
13
+ YamlConfigSettingsSource,
14
+ )
15
+
16
+ _dirs = PlatformDirs("sciop_cli", "sciop", ensure_exists=True)
17
+ _global_config = Path(_dirs.user_config_dir) / "sciop_cli.yaml"
18
+ _config: Optional["Config"] = None
19
+
20
+
21
+ def get_config() -> "Config":
22
+ global _config
23
+ if _config is None:
24
+ _config = Config()
25
+ return _config
26
+
27
+
28
+ def set_config(cfg: "Config") -> "Config":
29
+ """
30
+ Set config, dumping to the global config yaml file.
31
+
32
+ If a password is present, first try to save it in the keychain
33
+ and exclude it from the dump.
34
+
35
+ if we can't for some reason, dump it with a warning
36
+ """
37
+ global _config
38
+ _config = cfg
39
+
40
+ logger = getLogger("sciop_cli.config")
41
+ if cfg.password and cfg.username:
42
+ try:
43
+ keyring.set_password("sciop_cli", cfg.username, cfg.password.get_secret_value())
44
+ dumped = cfg.model_dump(exclude={"password"})
45
+ except Exception as e:
46
+ logger.warning(
47
+ "Password present, but could not save in keyring. "
48
+ "Dumping in plaintext, which is insecure!\n"
49
+ f"Got exception: {e}"
50
+ )
51
+ dumped = cfg.model_dump()
52
+ dumped["password"] = dumped["password"].get_secret_value()
53
+ else:
54
+ dumped = cfg.model_dump()
55
+
56
+ if dumped.get("token") and isinstance(dumped["token"], SecretStr):
57
+ dumped["token"] = dumped["token"].get_secret_value()
58
+
59
+ with open(_global_config, "w") as f:
60
+ yaml.safe_dump(dumped, f)
61
+ logger.debug(f"Dumped config to {_global_config}")
62
+ return cfg
63
+
64
+
65
+ class Config(BaseSettings):
66
+ """
67
+ Environmental config for cli commands.
68
+
69
+ Keep this light - just what we need to run cli commands,
70
+ don't want this to be a major point of interaction.
71
+
72
+ Values can be set in a .env file with a SCIOP_CLI_ prefix,
73
+ a `sciop_cli.yaml` file in cwd,
74
+ or the `sciop_cli` user config dir provided by
75
+ [platformdirs](https://github.com/tox-dev/platformdirs)
76
+ (e.g. `~/.config/sciop_cli/sciop_cli.yaml`)
77
+ """
78
+
79
+ instance_url: str = "https://sciop.net"
80
+ username: str | None = None
81
+ password: SecretStr | None = None
82
+ request_timeout: float = 5
83
+ """Default timeout to use in API requests."""
84
+ token: SecretStr | None = None
85
+ """Expiring jwt, stored in config to reuse between cli calls"""
86
+
87
+ model_config = SettingsConfigDict(
88
+ env_file=".env",
89
+ env_file_encoding="utf-8",
90
+ env_prefix="sciop_cli_",
91
+ env_nested_delimiter="__",
92
+ extra="ignore",
93
+ nested_model_default_partial_update=True,
94
+ yaml_file="sciop_cli.yaml",
95
+ validate_assignment=True,
96
+ )
97
+
98
+ @model_validator(mode="before")
99
+ @classmethod
100
+ def password_from_keychain(cls, value: dict | Self) -> dict | Self:
101
+ """Try to get a password from keychain, if not provided"""
102
+ try:
103
+ if isinstance(value, dict) and value.get("username") and not value.get("password"):
104
+ maybe_password = keyring.get_password("sciop_cli", value["username"])
105
+ if maybe_password:
106
+ value["password"] = SecretStr(maybe_password)
107
+ elif isinstance(value, cls) and value.username and value.password is None:
108
+ maybe_password = keyring.get_password("sciop_cli", value.username)
109
+ if maybe_password:
110
+ value.password = SecretStr(maybe_password)
111
+ except Exception:
112
+ # e.g. no supported keyring backends available
113
+ return value
114
+ return value
115
+
116
+ @classmethod
117
+ def settings_customise_sources(
118
+ cls,
119
+ settings_cls: type[BaseSettings],
120
+ init_settings: PydanticBaseSettingsSource,
121
+ env_settings: PydanticBaseSettingsSource,
122
+ dotenv_settings: PydanticBaseSettingsSource,
123
+ file_secret_settings: PydanticBaseSettingsSource,
124
+ ) -> tuple[PydanticBaseSettingsSource, ...]:
125
+ """
126
+ Read from the following sources,
127
+ in order such that later sources in the list override earlier sources
128
+
129
+ - `{config_dir}/sciop_cli.yaml`
130
+ - `sciop_cli.yaml` (in cwd)
131
+ - `.env` (in cwd)
132
+ - environment variables prefixed with `SCIOP_`
133
+ - arguments passed on config object initialization
134
+
135
+ See [pydantic settings docs](https://docs.pydantic.dev/latest/concepts/pydantic_settings/#customise-settings-sources)
136
+ """
137
+ global_source = YamlConfigSettingsSource(settings_cls, yaml_file=_global_config)
138
+ return (
139
+ init_settings,
140
+ env_settings,
141
+ dotenv_settings,
142
+ YamlConfigSettingsSource(settings_cls),
143
+ global_source,
144
+ )
@@ -0,0 +1,8 @@
1
+ from sciop_cli.models.sciop import DatasetClaim, Token, TorrentFile, Upload
2
+
3
+ __all__ = [
4
+ "DatasetClaim",
5
+ "Token",
6
+ "TorrentFile",
7
+ "Upload",
8
+ ]
@@ -0,0 +1,49 @@
1
+ """
2
+ Placeholder models that mimic sciop's models,
3
+ until we can split them off into a separate package and single-source them.
4
+ """
5
+
6
+ from datetime import datetime
7
+ from typing import Literal, NotRequired, TypedDict
8
+
9
+
10
+ class Token(TypedDict):
11
+ access_token: str
12
+ token_type: Literal["bearer"]
13
+
14
+
15
+ class DatasetClaim(TypedDict):
16
+ created_at: datetime
17
+ updated_at: datetime
18
+ dataset: str
19
+ dataset_part: NotRequired[str]
20
+ account: str
21
+ status: Literal["in_progress", "completed"]
22
+
23
+
24
+ class TorrentFile(TypedDict):
25
+ file_name: str
26
+ v1_infohash: NotRequired[str | None]
27
+ v2_infohash: NotRequired[str | None]
28
+ short_hash: str
29
+ version: Literal["v1", "v2", "hybrid"]
30
+ total_size: int
31
+ piece_size: int
32
+ torrent_size: int
33
+ announce_urls: list[str]
34
+ seeders: NotRequired[int | None]
35
+ leechers: NotRequired[int | None]
36
+
37
+
38
+ class Upload(TypedDict):
39
+ created_at: datetime
40
+ updated_at: datetime
41
+ is_approved: bool
42
+ is_removed: bool
43
+ method: str | None
44
+ description: str | None
45
+ dataset: str
46
+ dataset_parts: NotRequired[list[str]]
47
+ seeders: NotRequired[int | None]
48
+ leechers: NotRequired[int | None]
49
+ torrent: TorrentFile
@@ -0,0 +1,49 @@
1
+ from pathlib import Path
2
+
3
+ import pytest
4
+ from sciop.testing.fixtures.config import set_config as sciop_set_config # noqa: F401
5
+ from sciop.testing.fixtures.db import * # noqa: F401
6
+ from sciop.testing.fixtures.server import * # noqa: F401
7
+
8
+ from sciop_cli.config import Config
9
+
10
+ from .fixtures import *
11
+
12
+ DATA_DIR = Path(__file__).parent / "data"
13
+
14
+ pytest_plugins = ("sciop.testing.plugin",)
15
+
16
+
17
+ @pytest.fixture(autouse=True, scope="session")
18
+ def session_monkeypatch_config(
19
+ monkeypatch_session: pytest.MonkeyPatch, tmp_path_factory: pytest.TempPathFactory
20
+ ) -> None:
21
+ """sessionwide baseline for our config..."""
22
+ from sciop_cli import config
23
+ from sciop_cli.config import Config, set_config
24
+
25
+ session_dir = tmp_path_factory.mktemp("sciop_cli_session")
26
+ monkeypatch_session.setattr(config, "_global_config", session_dir / "sciop_cli_test.yaml")
27
+
28
+ new_config = Config(
29
+ username=None,
30
+ password=None,
31
+ token=None,
32
+ instance_url="http://127.0.0.1:8080",
33
+ request_timeout=10,
34
+ )
35
+ set_config(new_config)
36
+
37
+
38
+ @pytest.fixture()
39
+ def fresh_config(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Config:
40
+ from sciop_cli import config
41
+ from sciop_cli.config import Config, set_config
42
+
43
+ monkeypatch.setattr(config, "_global_config", tmp_path / "sciop_cli_test.yaml")
44
+
45
+ new_config = Config(
46
+ username=None, password=None, token=None, instance_url="http://127.0.0.1:8080"
47
+ )
48
+ set_config(new_config)
49
+ return new_config
@@ -0,0 +1,17 @@
1
+ from .base import monkeypatch_session
2
+ from .fabricators import account, as_admin, claims_setup, config_logged_in
3
+
4
+ # from .sciop import client, engine, patch_config, run_server, session
5
+
6
+ __all__ = [
7
+ "account",
8
+ "as_admin",
9
+ "claims_setup",
10
+ # "client",
11
+ "config_logged_in",
12
+ # "engine",
13
+ "monkeypatch_session",
14
+ # "patch_config",
15
+ # "session",
16
+ # "run_server",
17
+ ]
@@ -0,0 +1,12 @@
1
+ import pytest
2
+ from pytest import MonkeyPatch
3
+
4
+
5
+ @pytest.fixture(scope="session")
6
+ def monkeypatch_session() -> MonkeyPatch:
7
+ """
8
+ Monkeypatch you can use at the session scope!
9
+ """
10
+ mpatch = MonkeyPatch()
11
+ yield mpatch
12
+ mpatch.undo()
@@ -0,0 +1,82 @@
1
+ from typing import TYPE_CHECKING
2
+
3
+ import httpx
4
+ import pytest
5
+
6
+ from sciop_cli.config import Config, set_config
7
+
8
+ if TYPE_CHECKING:
9
+ from sciop.models import Account, Dataset
10
+ from sciop.testing.server import UvicornSyncServer
11
+ from sqlmodel import Session
12
+
13
+
14
+ @pytest.fixture
15
+ async def account(run_server_sync: "UvicornSyncServer") -> tuple[str, str]:
16
+ # say we have set our login info through the cli...
17
+ username = "testuser"
18
+ password = "testuser12345"
19
+ async with httpx.AsyncClient() as client:
20
+ account = await client.post(
21
+ "http://localhost:8080/api/v1/register",
22
+ data={"username": username, "password": password},
23
+ )
24
+ assert account.status_code == 200
25
+ return username, password
26
+
27
+
28
+ @pytest.fixture()
29
+ async def config_logged_in(account: tuple[str, str], fresh_config: Config) -> Config:
30
+ # say we have set our login info through the cli...
31
+ username, password = account
32
+ fresh_config.username = username
33
+ fresh_config.password = password
34
+ set_config(fresh_config)
35
+ return fresh_config
36
+
37
+
38
+ @pytest.fixture()
39
+ async def as_admin(
40
+ fresh_config: Config, session: "Session", run_server_sync: "UvicornSyncServer"
41
+ ) -> "Account":
42
+ from sciop.api.auth import get_password_hash
43
+ from sciop.models import Account, Scope
44
+
45
+ username = "admin"
46
+ password = "adminadmin12"
47
+ hashed_pw = get_password_hash(password)
48
+ admin = Scope.get_item("admin", session=session)
49
+ account = Account(username=username, hashed_password=hashed_pw, scopes=[admin])
50
+ session.add(account)
51
+ session.commit()
52
+ session.refresh(account)
53
+ fresh_config.username = username
54
+ fresh_config.password = password
55
+ set_config(fresh_config)
56
+ return account
57
+
58
+
59
+ @pytest.fixture()
60
+ def claims_setup(as_admin: "Account", session: "Session") -> "Dataset":
61
+ """
62
+ Crappy in-place data setup pending exporting sciop fixtures in a testing module
63
+ """
64
+ from sciop.models import Dataset, DatasetPart, Tag
65
+
66
+ tag = Tag(tag="default")
67
+ ds = Dataset(
68
+ title="default",
69
+ slug="default",
70
+ publisher="example",
71
+ tags=[tag],
72
+ account=as_admin,
73
+ is_approved=True,
74
+ )
75
+ parts = [
76
+ DatasetPart(part_slug=letter, is_approved=True, dataset=ds) for letter in ("a", "b", "c")
77
+ ]
78
+ ds.parts = parts
79
+ session.add(ds)
80
+ session.commit()
81
+ session.refresh(ds)
82
+ return ds
@@ -0,0 +1,132 @@
1
+ from typing import TYPE_CHECKING
2
+
3
+ import httpx
4
+ import pytest
5
+ from pydantic import TypeAdapter
6
+
7
+ from sciop_cli.api import claim_next, create_upload, get_token, login, upload_torrent
8
+ from sciop_cli.config import Config, get_config
9
+ from sciop_cli.models.sciop import TorrentFile, Upload
10
+
11
+ if TYPE_CHECKING:
12
+ from sciop.models import Account
13
+ from torf import Torrent
14
+
15
+
16
+ @pytest.mark.asyncio
17
+ async def test_login(fresh_config: Config, run_server_sync):
18
+ """
19
+ The login command logs us in!
20
+ """
21
+ username = "testuser"
22
+ password = "testuser12345"
23
+ async with httpx.AsyncClient() as client:
24
+ account = await client.post(
25
+ "http://127.0.0.1:8080/api/v1/register",
26
+ data={"username": username, "password": password},
27
+ timeout=10,
28
+ )
29
+ assert account.status_code == 200
30
+
31
+ token = login(username, password)
32
+ assert isinstance(token, str)
33
+
34
+ res = await client.get(
35
+ "http://localhost:8080/self/", headers={"Authorization": f"Bearer {token}"}
36
+ )
37
+ assert res.status_code == 200
38
+
39
+
40
+ @pytest.mark.asyncio
41
+ async def test_get_token(config_logged_in: Config, run_server_sync):
42
+ """
43
+ get_token gets a token and sets it in the config
44
+ """
45
+ cfg = config_logged_in
46
+ assert cfg.token is None
47
+ assert get_config().token is None
48
+
49
+ # then we should get a token and set it in the config
50
+ token = get_token()
51
+ assert token is not None
52
+ assert get_config().token.get_secret_value() == token
53
+
54
+
55
+ @pytest.mark.asyncio
56
+ async def test_get_next_claim(as_admin: "Account", run_server_sync, claims_setup):
57
+ """
58
+ Should be able to call get next claim!
59
+ """
60
+ responses = [claim_next("default") for _ in range(3)]
61
+ for letter, response in zip(("a", "b", "c"), responses):
62
+ assert response["dataset_part"] == letter
63
+ assert response["status"] == "in_progress"
64
+
65
+ assert claim_next("default") is None
66
+
67
+
68
+ @pytest.mark.asyncio
69
+ async def test_upload_torrent(as_admin: "Account", run_server_sync, torrent, tmp_path):
70
+ """
71
+ We can upload a torrent!
72
+ """
73
+ torrent: Torrent = torrent()
74
+ path = tmp_path / "test.torrent"
75
+ torrent.write(path)
76
+ res = upload_torrent(path)
77
+
78
+ adapter = TypeAdapter(TorrentFile)
79
+ adapter.validate_python(res)
80
+ assert res["v1_infohash"] == torrent.infohash
81
+ assert res["file_name"] == "test.torrent"
82
+
83
+
84
+ @pytest.mark.asyncio
85
+ @pytest.mark.parametrize("mode", ["infohash", "path"])
86
+ async def test_create_upload(
87
+ as_admin: "Account",
88
+ dataset,
89
+ run_server_sync,
90
+ torrent,
91
+ tmp_path,
92
+ mode: str,
93
+ ):
94
+ """
95
+ We can create an upload item, either by passing an infohash or a path
96
+ """
97
+ slug = "default"
98
+ path = tmp_path / "test.torrent"
99
+
100
+ dataset(slug=slug)
101
+ torrent: Torrent = torrent()
102
+ torrent.write(path)
103
+
104
+ if mode == "infohash":
105
+ upload_torrent(path)
106
+ ul = create_upload(dataset=slug, infohash=torrent.infohash)
107
+ elif mode == "path":
108
+ ul = create_upload(dataset=slug, torrent_path=path)
109
+ else:
110
+ raise ValueError()
111
+
112
+ adapter = TypeAdapter(Upload)
113
+ adapter.validate_python(ul)
114
+
115
+
116
+ @pytest.mark.asyncio
117
+ @pytest.mark.parametrize("parts", [["part-1"], ["part-1", "part-2"]])
118
+ async def test_create_upload_part(
119
+ as_admin: "Account", dataset, run_server_sync, torrent, tmp_path, parts: list[str]
120
+ ):
121
+ """
122
+ We can upload to a dataset part
123
+ """
124
+ slug = "default"
125
+ path = tmp_path / "test.torrent"
126
+
127
+ dataset(slug=slug, parts=[{"part_slug": p} for p in parts])
128
+ torrent: Torrent = torrent()
129
+ torrent.write(path)
130
+
131
+ ul = create_upload(dataset=slug, dataset_parts=parts, torrent_path=path)
132
+ assert ul["dataset_parts"] == parts
@@ -0,0 +1,46 @@
1
+ import json
2
+ from typing import TYPE_CHECKING
3
+
4
+ from click.testing import CliRunner
5
+
6
+ from sciop_cli.cli.api import login as login_cli
7
+ from sciop_cli.cli.api import next_claim as next_claim_cli
8
+ from sciop_cli.config import Config, get_config
9
+
10
+ if TYPE_CHECKING:
11
+ from .sciop import UvicornSyncServer
12
+
13
+
14
+ def test_login(
15
+ run_server_sync: "UvicornSyncServer", fresh_config: Config, account: tuple[str, str]
16
+ ):
17
+ """
18
+ We can log in through the CLI!
19
+ """
20
+ cfg = get_config()
21
+ assert cfg.username is None
22
+ assert cfg.password is None
23
+ runner = CliRunner()
24
+
25
+ res = runner.invoke(login_cli, ["--username", account[0], "--password", account[1]])
26
+ assert res.exit_code == 0
27
+ cfg2: Config = get_config()
28
+ assert cfg2.username == account[0]
29
+ assert cfg2.password.get_secret_value() == account[1]
30
+ assert cfg2.token is not None
31
+
32
+
33
+ def test_get_next_claim(as_admin, claims_setup, run_server_sync):
34
+ """
35
+ After logging in, we can get the next dataset part and print it as json
36
+ """
37
+
38
+ runner = CliRunner()
39
+
40
+ res = runner.invoke(next_claim_cli, ["--dataset", "default", "--json"])
41
+ assert res.exit_code == 0
42
+ out = json.loads(res.stdout.splitlines()[-1])
43
+ assert out["status"] == "in_progress"
44
+ assert out["dataset"] == "default"
45
+ assert out["dataset_part"] == "a"
46
+ assert out["account"] == "admin"
@@ -0,0 +1,55 @@
1
+ from pathlib import Path
2
+
3
+ import keyring
4
+ import pytest
5
+ import yaml
6
+
7
+ from sciop_cli import config
8
+ from sciop_cli.config import Config, set_config
9
+
10
+
11
+ def test_instantiation_defaults(monkeypatch, tmp_path):
12
+ """We should be able to instantiate config without setting any values"""
13
+ monkeypatch.chdir(tmp_path)
14
+ monkeypatch.setattr(config, "_global_config", Path(tmp_path) / "sciop_cli_test.yaml")
15
+ _ = Config()
16
+
17
+
18
+ def test_password_from_keychain(request: pytest.FixtureRequest):
19
+ """We can get a password from the keychain automatically"""
20
+ username = f"__test__{request.node.name}"
21
+ expected = "__testpassword__"
22
+ keyring.set_password("sciop_cli", username, expected)
23
+
24
+ cfg = Config(username=username)
25
+ assert cfg.password
26
+ assert cfg.password.get_secret_value() == expected
27
+
28
+
29
+ def test_set_config_no_password(monkeypatch, tmp_path: Path, request: pytest.FixtureRequest):
30
+ """
31
+ When we dump the config, and we have a password, and we can access a keyring,
32
+ we should remove the password from the dump, save it in the keyring.
33
+ """
34
+ username = f"__test__{request.node.name}"
35
+ expected = "__testpassword__"
36
+ cfg_path = Path(tmp_path) / "sciop_cli_test.yaml"
37
+ monkeypatch.chdir(tmp_path)
38
+ monkeypatch.setattr(config, "_global_config", cfg_path)
39
+ if keyring.get_password("sciop_cli", username):
40
+ keyring.delete_password("sciop_cli", username)
41
+
42
+ assert not keyring.get_password("sciop_cli", username)
43
+ cfg = Config(username=username, password=expected)
44
+ cfg = set_config(cfg)
45
+
46
+ with open(cfg_path) as f:
47
+ dumped = yaml.safe_load(f)
48
+ assert dumped["username"] == username
49
+ assert "password" not in dumped
50
+ assert keyring.get_password("sciop_cli", username) == expected
51
+
52
+ # and we should get it again when we instantiate
53
+ loaded = Config()
54
+ assert loaded.username == username
55
+ assert loaded.password.get_secret_value() == expected
@@ -1,40 +0,0 @@
1
- from pathlib import Path
2
-
3
- import pandera as pa
4
- from pandera.typing import DataFrame, Series
5
-
6
-
7
- class Files(pa.DataFrameModel):
8
- filename: Series[str]
9
- directory: Series[str]
10
- size: Series[int]
11
-
12
-
13
- @pa.check_types
14
- def read_directory(directory: Path) -> DataFrame[Files]:
15
- names = []
16
- dirs = []
17
- sizes = []
18
- for f in directory.glob("**/*"):
19
- if not f.is_file():
20
- continue
21
- names.append(f.name)
22
- dirs.append(str(f.parent))
23
- sizes.append(f.stat().st_size)
24
- return DataFrame[Files]({"filename": names, "directory": dirs, "size": sizes})
25
-
26
-
27
- def directory_size(directory: Path) -> int:
28
- """Recursive directoy size, in bytes"""
29
- return sum(f.stat().st_size for f in directory.glob("**/*") if f.is_file())
30
-
31
-
32
- # def should_zip(directory: Path, piece_size: int, n_file_threshold:int = 20) -> bool:
33
- # """
34
- # FIXME: nvm just load all the files into pandas and do it vectorized
35
- # """
36
- # pass
37
- # # n_small_files = 0
38
- # # for f in directory.glob("*"):
39
- # # if f.is_file() and f.stat().st_size < n_file_threshold:
40
- # # n_small_files += 1
Binary file
@@ -1,3 +0,0 @@
1
- from pathlib import Path
2
-
3
- DATA_DIR = Path(__file__).parent / "data"
File without changes
File without changes
File without changes
File without changes
File without changes