tai42-storage-github 2.0.2__tar.gz → 3.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tai42_storage_github-2.0.2/src/tai42_storage_github.egg-info → tai42_storage_github-3.0.0}/PKG-INFO +10 -5
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/README.md +9 -4
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/pyproject.toml +1 -1
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/src/tai42_storage_github/settings.py +3 -2
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/src/tai42_storage_github/storage.py +61 -25
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/src/tai42_storage_github/tai-plugin.yml +1 -1
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0/src/tai42_storage_github.egg-info}/PKG-INFO +10 -5
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/tests/test_binary.py +19 -2
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/tests/test_delete_dir.py +24 -15
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/tests/test_text.py +70 -0
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/LICENSE +0 -0
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/NOTICE +0 -0
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/setup.cfg +0 -0
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/src/tai42_storage_github/__init__.py +0 -0
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/src/tai42_storage_github/client.py +0 -0
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/src/tai42_storage_github/docs/index.mdx +0 -0
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/src/tai42_storage_github/py.typed +0 -0
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/src/tai42_storage_github.egg-info/SOURCES.txt +0 -0
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/src/tai42_storage_github.egg-info/dependency_links.txt +0 -0
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/src/tai42_storage_github.egg-info/requires.txt +0 -0
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/src/tai42_storage_github.egg-info/top_level.txt +0 -0
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/tests/test_import_graph.py +0 -0
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/tests/test_plugin_spec.py +0 -0
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/tests/test_registration.py +0 -0
- {tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/tests/test_security.py +0 -0
{tai42_storage_github-2.0.2/src/tai42_storage_github.egg-info → tai42_storage_github-3.0.0}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tai42-storage-github
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.0.0
|
|
4
4
|
Summary: GitHub-backed Storage provider (text + binary/media) for the TAI ecosystem.
|
|
5
5
|
Author-email: tai42 <oss@tai42.ai>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -34,7 +34,7 @@ A GitHub-backed `Storage` provider for the TAI ecosystem. It stores content as
|
|
|
34
34
|
files in a GitHub repository and serves the full `tai42_contract.storage.Storage`
|
|
35
35
|
surface — the text methods (`load` / `list` / `upload` / `delete` / `delete_dir`)
|
|
36
36
|
plus the metadata read `stat`, the bounded read `load_range` and the binary
|
|
37
|
-
methods (`load_bytes` / `upload_bytes`).
|
|
37
|
+
methods (`load_bytes` / `upload_bytes` / `load_object`).
|
|
38
38
|
|
|
39
39
|
## The TAI ecosystem
|
|
40
40
|
|
|
@@ -100,11 +100,12 @@ a log line, repr, or error message.
|
|
|
100
100
|
|
|
101
101
|
## Reads and writes
|
|
102
102
|
|
|
103
|
-
- **Reads** (`load`, `load_bytes`) are one Contents API call each at the
|
|
103
|
+
- **Reads** (`load`, `load_bytes`, `load_object`) are one Contents API call each at the
|
|
104
104
|
configured branch, asking for the raw form
|
|
105
105
|
(`Accept: application/vnd.github.raw+json`), which serves every byte of a file
|
|
106
106
|
of any size. A directory answers its JSON listing and reads as
|
|
107
|
-
`FileNotFoundError`.
|
|
107
|
+
`FileNotFoundError`. `load_object` infers the content type from the path suffix,
|
|
108
|
+
as `stat` does.
|
|
108
109
|
- **Ranged reads** (`load_range`) resolve the branch's head commit through the
|
|
109
110
|
Git refs API, then read `raw.githubusercontent.com` at that commit with a
|
|
110
111
|
`Range` header and `Accept-Encoding: identity` (the raw host compresses, and a
|
|
@@ -120,7 +121,11 @@ a log line, repr, or error message.
|
|
|
120
121
|
are capped conservatively at 1 MiB. A larger payload raises `ValueError` before
|
|
121
122
|
the request rather than truncating; the API's own `422` is the backstop.
|
|
122
123
|
- **Listing** uses the recursive Git Trees API (one request) and raises loudly on
|
|
123
|
-
a `truncated` response rather than acting on a partial listing.
|
|
124
|
+
a `truncated` response rather than acting on a partial listing. `list(directory)`
|
|
125
|
+
reads the tree of that directory alone (`<branch>:<directory>`), so it grows with
|
|
126
|
+
the directory, not the repository; a directory that does not exist answers `[]`
|
|
127
|
+
once the branch is confirmed to exist. `delete_dir` lists the directory the same
|
|
128
|
+
way.
|
|
124
129
|
- **`stat`** is one Contents API call asking for the object form
|
|
125
130
|
(`Accept: application/vnd.github.object+json`): it answers the file's `size`,
|
|
126
131
|
and infers `content_type` from the path suffix — GitHub stores no per-object
|
|
@@ -6,7 +6,7 @@ A GitHub-backed `Storage` provider for the TAI ecosystem. It stores content as
|
|
|
6
6
|
files in a GitHub repository and serves the full `tai42_contract.storage.Storage`
|
|
7
7
|
surface — the text methods (`load` / `list` / `upload` / `delete` / `delete_dir`)
|
|
8
8
|
plus the metadata read `stat`, the bounded read `load_range` and the binary
|
|
9
|
-
methods (`load_bytes` / `upload_bytes`).
|
|
9
|
+
methods (`load_bytes` / `upload_bytes` / `load_object`).
|
|
10
10
|
|
|
11
11
|
## The TAI ecosystem
|
|
12
12
|
|
|
@@ -72,11 +72,12 @@ a log line, repr, or error message.
|
|
|
72
72
|
|
|
73
73
|
## Reads and writes
|
|
74
74
|
|
|
75
|
-
- **Reads** (`load`, `load_bytes`) are one Contents API call each at the
|
|
75
|
+
- **Reads** (`load`, `load_bytes`, `load_object`) are one Contents API call each at the
|
|
76
76
|
configured branch, asking for the raw form
|
|
77
77
|
(`Accept: application/vnd.github.raw+json`), which serves every byte of a file
|
|
78
78
|
of any size. A directory answers its JSON listing and reads as
|
|
79
|
-
`FileNotFoundError`.
|
|
79
|
+
`FileNotFoundError`. `load_object` infers the content type from the path suffix,
|
|
80
|
+
as `stat` does.
|
|
80
81
|
- **Ranged reads** (`load_range`) resolve the branch's head commit through the
|
|
81
82
|
Git refs API, then read `raw.githubusercontent.com` at that commit with a
|
|
82
83
|
`Range` header and `Accept-Encoding: identity` (the raw host compresses, and a
|
|
@@ -92,7 +93,11 @@ a log line, repr, or error message.
|
|
|
92
93
|
are capped conservatively at 1 MiB. A larger payload raises `ValueError` before
|
|
93
94
|
the request rather than truncating; the API's own `422` is the backstop.
|
|
94
95
|
- **Listing** uses the recursive Git Trees API (one request) and raises loudly on
|
|
95
|
-
a `truncated` response rather than acting on a partial listing.
|
|
96
|
+
a `truncated` response rather than acting on a partial listing. `list(directory)`
|
|
97
|
+
reads the tree of that directory alone (`<branch>:<directory>`), so it grows with
|
|
98
|
+
the directory, not the repository; a directory that does not exist answers `[]`
|
|
99
|
+
once the branch is confirmed to exist. `delete_dir` lists the directory the same
|
|
100
|
+
way.
|
|
96
101
|
- **`stat`** is one Contents API call asking for the object form
|
|
97
102
|
(`Accept: application/vnd.github.object+json`): it answers the file's `size`,
|
|
98
103
|
and infers `content_type` from the path suffix — GitHub stores no per-object
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "tai42-storage-github"
|
|
7
|
-
version = "
|
|
7
|
+
version = "3.0.0"
|
|
8
8
|
description = "GitHub-backed Storage provider (text + binary/media) for the TAI ecosystem."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.13"
|
{tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/src/tai42_storage_github/settings.py
RENAMED
|
@@ -24,8 +24,9 @@ class GithubStorageSettings(TaiBaseSettings):
|
|
|
24
24
|
# defaulting to the real GitHub hosts. Overridable (like ``S3Settings.endpoint``)
|
|
25
25
|
# so a test can point the plugin at a hermetic stand-in. Each carries
|
|
26
26
|
# ``{username}``/``{repo}`` placeholders the provider fills per request; the ref
|
|
27
|
-
# and trees templates also ``{branch}
|
|
28
|
-
# branch
|
|
27
|
+
# and trees templates also ``{branch}`` (the trees template's is the tree read: the
|
|
28
|
+
# branch, or ``branch:directory`` for a directory listing), and the raw template
|
|
29
|
+
# ``{commit}``, the branch head commit a ranged read resolves first.
|
|
29
30
|
raw_base_url: str = "https://raw.githubusercontent.com/{username}/{repo}/{commit}"
|
|
30
31
|
contents_api_url: str = "https://api.github.com/repos/{username}/{repo}/contents"
|
|
31
32
|
ref_api_url: str = "https://api.github.com/repos/{username}/{repo}/git/ref/heads/{branch}"
|
{tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/src/tai42_storage_github/storage.py
RENAMED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
"""GitHub-backed :class:`~tai42_contract.storage.Storage` provider.
|
|
2
2
|
|
|
3
|
-
``stat``, ``load`` and ``
|
|
4
|
-
configured branch: the object form answers the size, the raw form every byte
|
|
5
|
-
the file
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
3
|
+
``stat``, ``load``, ``load_bytes`` and ``load_object`` are one Contents API call each
|
|
4
|
+
at the configured branch: the object form answers the size, the raw form every byte
|
|
5
|
+
of the file (GitHub stores no content type, so it is inferred from the path).
|
|
6
|
+
``list`` of a directory reads only that directory's tree. ``load_range`` resolves the
|
|
7
|
+
branch's head commit through the Git refs API and reads the raw host at that
|
|
8
|
+
commit with a ``Range`` header, so a ranged read answers the branch's current
|
|
9
|
+
bytes. Writes use the Contents API (base64) and are capped below the API limit,
|
|
10
|
+
raising before the request rather than truncating.
|
|
9
11
|
"""
|
|
10
12
|
|
|
11
13
|
from __future__ import annotations
|
|
@@ -14,10 +16,18 @@ import base64
|
|
|
14
16
|
import logging
|
|
15
17
|
import mimetypes
|
|
16
18
|
from typing import Final, NamedTuple
|
|
19
|
+
from urllib.parse import quote
|
|
17
20
|
|
|
18
21
|
import httpx
|
|
19
22
|
from tai42_contract.app import tai42_app
|
|
20
|
-
from tai42_contract.storage import
|
|
23
|
+
from tai42_contract.storage import (
|
|
24
|
+
ObjectStat,
|
|
25
|
+
Storage,
|
|
26
|
+
StoragePathConflictError,
|
|
27
|
+
StoredObject,
|
|
28
|
+
assert_not_root,
|
|
29
|
+
directory_prefix,
|
|
30
|
+
)
|
|
21
31
|
|
|
22
32
|
from tai42_storage_github.client import GithubHttpxClient
|
|
23
33
|
from tai42_storage_github.settings import GithubStorageSettings, github_storage_settings
|
|
@@ -42,17 +52,22 @@ class _Blob(NamedTuple):
|
|
|
42
52
|
sha: str
|
|
43
53
|
|
|
44
54
|
|
|
45
|
-
def
|
|
46
|
-
"""
|
|
55
|
+
def _path_segments(path: str) -> list[str]:
|
|
56
|
+
"""The segments of storage ``path``, dropping empty and stray-slash segments.
|
|
47
57
|
|
|
48
58
|
A ``..`` segment is rejected loudly: preserved, it would climb out of the
|
|
49
59
|
configured repo/branch base and let a caller-supplied path reach an arbitrary
|
|
50
|
-
raw/contents URL (path traversal).
|
|
60
|
+
raw/contents/tree URL (path traversal).
|
|
51
61
|
"""
|
|
52
62
|
segments = [segment for segment in path.split("/") if segment]
|
|
53
63
|
if ".." in segments:
|
|
54
64
|
raise ValueError(f"path traversal ('..') is not allowed in storage path: {path!r}")
|
|
55
|
-
|
|
65
|
+
return segments
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _join(base: str, path: str) -> str:
|
|
69
|
+
"""Join the :func:`_path_segments` of ``path`` onto ``base``."""
|
|
70
|
+
joined = "/".join(_path_segments(path))
|
|
56
71
|
base = base.rstrip("/")
|
|
57
72
|
return f"{base}/{joined}" if joined else base
|
|
58
73
|
|
|
@@ -159,6 +174,12 @@ class GithubStorage(Storage):
|
|
|
159
174
|
raise FileNotFoundError(f"Object not found: {path}")
|
|
160
175
|
raise RuntimeError(f"GitHub answered an unexpected Content-Type {media_type!r} loading {path!r} at {url}.")
|
|
161
176
|
|
|
177
|
+
async def load_object(self, path: str) -> StoredObject:
|
|
178
|
+
"""Every byte of ``path`` (as :meth:`load_bytes` reads them) with the content type :meth:`stat` infers."""
|
|
179
|
+
data = await self.load_bytes(path)
|
|
180
|
+
content_type, _ = mimetypes.guess_type(path)
|
|
181
|
+
return StoredObject(data=data, content_type=content_type)
|
|
182
|
+
|
|
162
183
|
async def stat(self, path: str) -> ObjectStat:
|
|
163
184
|
"""The byte size from the Contents API object form and the path-inferred content type.
|
|
164
185
|
|
|
@@ -239,34 +260,50 @@ class GithubStorage(Storage):
|
|
|
239
260
|
raise RuntimeError(f"GitHub answered no head commit sha for the branch at {url}.")
|
|
240
261
|
return sha
|
|
241
262
|
|
|
242
|
-
async def list(self) -> list[str]:
|
|
243
|
-
"""Every blob path in the repo.
|
|
244
|
-
|
|
263
|
+
async def list(self, directory: str | None = None) -> list[str]:
|
|
264
|
+
"""Every blob path in the repo, or only those beneath ``directory``, read from that directory's tree.
|
|
265
|
+
|
|
266
|
+
A directory is listed through the Git Trees API at ``branch:directory``, so only
|
|
267
|
+
its subtree is read. A directory that does not exist on the branch, or names a
|
|
268
|
+
file, answers ``[]``; the branch itself is confirmed first, so a missing branch
|
|
269
|
+
or repo still raises.
|
|
270
|
+
"""
|
|
271
|
+
return [blob.path for blob in await self._blobs(None if directory is None else directory_prefix(directory))]
|
|
245
272
|
|
|
246
|
-
async def _blobs(self, prefix: str) -> list[_Blob]:
|
|
247
|
-
"""Every blob in the repo
|
|
273
|
+
async def _blobs(self, prefix: str | None) -> list[_Blob]:
|
|
274
|
+
"""Every blob in the repo, or in the directory ``prefix`` names (``"a/b/"``), with its blob sha.
|
|
248
275
|
|
|
249
|
-
Uses the recursive Git Trees API (one request)
|
|
250
|
-
|
|
251
|
-
|
|
276
|
+
Uses the recursive Git Trees API (one request) at the branch, or at
|
|
277
|
+
``branch:directory`` for a ``prefix``, whose paths are relative to the directory
|
|
278
|
+
and are answered with ``prefix`` in front; the ``truncated`` flag is raised on
|
|
279
|
+
loudly rather than acting on a partial listing, and so is a blob entry carrying
|
|
280
|
+
no sha.
|
|
252
281
|
"""
|
|
253
282
|
settings = _configured_settings()
|
|
254
|
-
|
|
283
|
+
segments = [] if prefix is None else _path_segments(prefix)
|
|
284
|
+
tree = settings.branch if prefix is None else f"{settings.branch}:{quote('/'.join(segments), safe='/')}"
|
|
285
|
+
url = settings.trees_api_url.format(username=settings.username, repo=settings.repo, branch=tree)
|
|
255
286
|
async with tai42_app.clients.client_ctx(GithubHttpxClient) as client:
|
|
256
287
|
resp = await client.get(url, headers=_api_headers(settings), params={"recursive": "1"})
|
|
288
|
+
if prefix is not None and resp.status_code in (404, 422):
|
|
289
|
+
# 404: no such path on the branch; 422: the path names a blob, not a tree.
|
|
290
|
+
# A 404 also answers a missing branch or repo, so the branch is resolved
|
|
291
|
+
# first and its own failure raises.
|
|
292
|
+
if resp.status_code == 404:
|
|
293
|
+
await self._head_commit(client, settings, prefix)
|
|
294
|
+
return []
|
|
257
295
|
_raise_for_status(resp, "listing", prefix or "/", url)
|
|
258
296
|
data = resp.json()
|
|
259
297
|
if not isinstance(data, dict):
|
|
260
298
|
raise RuntimeError(f"Expected a tree object from {url}, got {type(data).__name__}") # noqa: TRY004 raised type is intentional (invariant/state/validation taxonomy); TypeError would change behaviour
|
|
261
299
|
if data.get("truncated"):
|
|
262
300
|
raise RuntimeError(f"GitHub tree listing was truncated; the tree is too large to list safely: {url}")
|
|
301
|
+
base = "".join(f"{segment}/" for segment in segments)
|
|
263
302
|
blobs: list[_Blob] = []
|
|
264
303
|
for item in data.get("tree", []):
|
|
265
304
|
if not isinstance(item, dict) or item.get("type") != "blob":
|
|
266
305
|
continue
|
|
267
|
-
path = item[
|
|
268
|
-
if prefix and not path.startswith(prefix):
|
|
269
|
-
continue
|
|
306
|
+
path = f"{base}{item['path']}"
|
|
270
307
|
sha = item.get("sha")
|
|
271
308
|
if not isinstance(sha, str) or not sha:
|
|
272
309
|
raise RuntimeError(f"GitHub tree listing at {url} carries no blob sha for {path}.")
|
|
@@ -350,8 +387,7 @@ class GithubStorage(Storage):
|
|
|
350
387
|
deletion surfaces loudly.
|
|
351
388
|
"""
|
|
352
389
|
assert_not_root(path)
|
|
353
|
-
|
|
354
|
-
blobs = await self._blobs(prefix)
|
|
390
|
+
blobs = await self._blobs(directory_prefix(path))
|
|
355
391
|
if not blobs:
|
|
356
392
|
raise FileNotFoundError(f"Directory not found or empty: {path}")
|
|
357
393
|
|
{tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0/src/tai42_storage_github.egg-info}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tai42-storage-github
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.0.0
|
|
4
4
|
Summary: GitHub-backed Storage provider (text + binary/media) for the TAI ecosystem.
|
|
5
5
|
Author-email: tai42 <oss@tai42.ai>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -34,7 +34,7 @@ A GitHub-backed `Storage` provider for the TAI ecosystem. It stores content as
|
|
|
34
34
|
files in a GitHub repository and serves the full `tai42_contract.storage.Storage`
|
|
35
35
|
surface — the text methods (`load` / `list` / `upload` / `delete` / `delete_dir`)
|
|
36
36
|
plus the metadata read `stat`, the bounded read `load_range` and the binary
|
|
37
|
-
methods (`load_bytes` / `upload_bytes`).
|
|
37
|
+
methods (`load_bytes` / `upload_bytes` / `load_object`).
|
|
38
38
|
|
|
39
39
|
## The TAI ecosystem
|
|
40
40
|
|
|
@@ -100,11 +100,12 @@ a log line, repr, or error message.
|
|
|
100
100
|
|
|
101
101
|
## Reads and writes
|
|
102
102
|
|
|
103
|
-
- **Reads** (`load`, `load_bytes`) are one Contents API call each at the
|
|
103
|
+
- **Reads** (`load`, `load_bytes`, `load_object`) are one Contents API call each at the
|
|
104
104
|
configured branch, asking for the raw form
|
|
105
105
|
(`Accept: application/vnd.github.raw+json`), which serves every byte of a file
|
|
106
106
|
of any size. A directory answers its JSON listing and reads as
|
|
107
|
-
`FileNotFoundError`.
|
|
107
|
+
`FileNotFoundError`. `load_object` infers the content type from the path suffix,
|
|
108
|
+
as `stat` does.
|
|
108
109
|
- **Ranged reads** (`load_range`) resolve the branch's head commit through the
|
|
109
110
|
Git refs API, then read `raw.githubusercontent.com` at that commit with a
|
|
110
111
|
`Range` header and `Accept-Encoding: identity` (the raw host compresses, and a
|
|
@@ -120,7 +121,11 @@ a log line, repr, or error message.
|
|
|
120
121
|
are capped conservatively at 1 MiB. A larger payload raises `ValueError` before
|
|
121
122
|
the request rather than truncating; the API's own `422` is the backstop.
|
|
122
123
|
- **Listing** uses the recursive Git Trees API (one request) and raises loudly on
|
|
123
|
-
a `truncated` response rather than acting on a partial listing.
|
|
124
|
+
a `truncated` response rather than acting on a partial listing. `list(directory)`
|
|
125
|
+
reads the tree of that directory alone (`<branch>:<directory>`), so it grows with
|
|
126
|
+
the directory, not the repository; a directory that does not exist answers `[]`
|
|
127
|
+
once the branch is confirmed to exist. `delete_dir` lists the directory the same
|
|
128
|
+
way.
|
|
124
129
|
- **`stat`** is one Contents API call asking for the object form
|
|
125
130
|
(`Accept: application/vnd.github.object+json`): it answers the file's `size`,
|
|
126
131
|
and infers `content_type` from the path suffix — GitHub stores no per-object
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Binary/media surface: load_bytes / upload_bytes / stat / load_range, plus the size caps.
|
|
1
|
+
"""Binary/media surface: load_bytes / load_object / upload_bytes / stat / load_range, plus the size caps.
|
|
2
2
|
|
|
3
3
|
Whole reads and ``stat`` are one Contents API call at the branch (the raw form serves
|
|
4
4
|
every byte, the object form the size); a ranged read resolves the branch head and
|
|
@@ -11,7 +11,7 @@ import base64
|
|
|
11
11
|
|
|
12
12
|
import pytest
|
|
13
13
|
from httpx import HTTPStatusError
|
|
14
|
-
from tai42_contract.storage import ObjectStat
|
|
14
|
+
from tai42_contract.storage import ObjectStat, StoredObject
|
|
15
15
|
|
|
16
16
|
from tai42_storage_github import GithubStorage
|
|
17
17
|
from tai42_storage_github.storage import MAX_UPLOAD_BYTES
|
|
@@ -69,6 +69,23 @@ async def test_load_bytes_large_file_is_read_whole(client):
|
|
|
69
69
|
assert "raw.githubusercontent.com" not in call.args[0]
|
|
70
70
|
|
|
71
71
|
|
|
72
|
+
async def test_load_object_is_one_raw_read_with_the_path_type(client):
|
|
73
|
+
client.get.return_value = make_response(status=200, content=b"\x89PNG", headers=RAW_FORM_HEADERS)
|
|
74
|
+
|
|
75
|
+
stored = await GithubStorage().load_object("img/a.png")
|
|
76
|
+
|
|
77
|
+
assert stored == StoredObject(data=b"\x89PNG", content_type="image/png")
|
|
78
|
+
client.get.assert_awaited_once()
|
|
79
|
+
assert client.get.call_args.kwargs["headers"]["Accept"] == RAW_FORM
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
async def test_load_object_missing_raises_filenotfound(client):
|
|
83
|
+
client.get.return_value = make_response(status=404)
|
|
84
|
+
|
|
85
|
+
with pytest.raises(FileNotFoundError):
|
|
86
|
+
await GithubStorage().load_object("missing.bin")
|
|
87
|
+
|
|
88
|
+
|
|
72
89
|
async def test_load_bytes_missing_raises_filenotfound(client):
|
|
73
90
|
client.get.return_value = make_response(status=404)
|
|
74
91
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""``delete_dir``: the root guard, one tree listing, a DELETE per blob with its tree sha, and
|
|
1
|
+
"""``delete_dir``: the root guard, one directory-tree listing, a DELETE per blob with its tree sha, and errors."""
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
@@ -9,7 +9,7 @@ from tai42_storage_github import GithubStorage
|
|
|
9
9
|
|
|
10
10
|
from .conftest import make_response
|
|
11
11
|
|
|
12
|
-
|
|
12
|
+
DIR_TREE_URL = "https://api.github.com/repos/acme/content/git/trees/dev:d"
|
|
13
13
|
CONTENTS_URL = "https://api.github.com/repos/acme/content/contents"
|
|
14
14
|
|
|
15
15
|
|
|
@@ -35,26 +35,35 @@ async def test_delete_dir_root_escape_refused(root):
|
|
|
35
35
|
@pytest.mark.parametrize("path", ["d", "d/"])
|
|
36
36
|
async def test_delete_dir_deletes_each_blob_under_it_with_its_tree_sha(client, path):
|
|
37
37
|
client.get.return_value = _tree(
|
|
38
|
-
{"path": "
|
|
39
|
-
{"path": "
|
|
40
|
-
{"path": "
|
|
41
|
-
{"path": "dd/c.j2", "type": "blob", "sha": "s3"},
|
|
42
|
-
{"path": "e/d.j2", "type": "blob", "sha": "s4"},
|
|
38
|
+
{"path": "a.j2", "type": "blob", "sha": "s1"},
|
|
39
|
+
{"path": "sub", "type": "tree", "sha": "t1"},
|
|
40
|
+
{"path": "sub/b.j2", "type": "blob", "sha": "s2"},
|
|
43
41
|
)
|
|
44
42
|
client.request.return_value = make_response(status=200)
|
|
45
43
|
|
|
46
44
|
await GithubStorage().delete_dir(path)
|
|
47
45
|
|
|
48
46
|
client.get.assert_awaited_once()
|
|
49
|
-
assert client.get.call_args.args[0] ==
|
|
47
|
+
assert client.get.call_args.args[0] == DIR_TREE_URL
|
|
50
48
|
assert client.get.call_args.kwargs["params"] == {"recursive": "1"}
|
|
51
49
|
assert _deleted(client) == [(f"{CONTENTS_URL}/d/a.j2", "s1"), (f"{CONTENTS_URL}/d/sub/b.j2", "s2")]
|
|
52
50
|
payload = client.request.call_args.kwargs["json"]
|
|
53
51
|
assert payload == {"message": "Delete d/sub/b.j2", "sha": "s2", "branch": "dev"}
|
|
54
52
|
|
|
55
53
|
|
|
54
|
+
async def test_delete_dir_lists_only_the_directory_tree(client):
|
|
55
|
+
client.get.return_value = _tree({"path": "a.j2", "type": "blob", "sha": "s1"})
|
|
56
|
+
client.request.return_value = make_response(status=200)
|
|
57
|
+
|
|
58
|
+
await GithubStorage().delete_dir("d/")
|
|
59
|
+
|
|
60
|
+
# The listing reads the tree at "dev:d", so "d" never reaches sibling "dd/...".
|
|
61
|
+
assert client.get.call_args.args[0].endswith("/git/trees/dev:d")
|
|
62
|
+
assert _deleted(client) == [(f"{CONTENTS_URL}/d/a.j2", "s1")]
|
|
63
|
+
|
|
64
|
+
|
|
56
65
|
async def test_delete_dir_of_an_empty_listing_raises_filenotfound_and_deletes_nothing(client):
|
|
57
|
-
client.get.return_value = _tree(
|
|
66
|
+
client.get.return_value = _tree()
|
|
58
67
|
|
|
59
68
|
with pytest.raises(FileNotFoundError):
|
|
60
69
|
await GithubStorage().delete_dir("d")
|
|
@@ -65,8 +74,8 @@ async def test_delete_dir_of_an_empty_listing_raises_filenotfound_and_deletes_no
|
|
|
65
74
|
|
|
66
75
|
async def test_delete_dir_skips_a_file_already_gone_and_deletes_the_rest(client):
|
|
67
76
|
client.get.return_value = _tree(
|
|
68
|
-
{"path": "
|
|
69
|
-
{"path": "
|
|
77
|
+
{"path": "a.j2", "type": "blob", "sha": "s1"},
|
|
78
|
+
{"path": "b.j2", "type": "blob", "sha": "s2"},
|
|
70
79
|
)
|
|
71
80
|
client.request.side_effect = [make_response(status=404), make_response(status=200)]
|
|
72
81
|
|
|
@@ -77,8 +86,8 @@ async def test_delete_dir_skips_a_file_already_gone_and_deletes_the_rest(client)
|
|
|
77
86
|
|
|
78
87
|
async def test_delete_dir_raises_on_a_file_changed_since_the_listing(client):
|
|
79
88
|
client.get.return_value = _tree(
|
|
80
|
-
{"path": "
|
|
81
|
-
{"path": "
|
|
89
|
+
{"path": "a.j2", "type": "blob", "sha": "s1"},
|
|
90
|
+
{"path": "b.j2", "type": "blob", "sha": "s2"},
|
|
82
91
|
)
|
|
83
92
|
client.request.return_value = make_response(status=409)
|
|
84
93
|
|
|
@@ -90,8 +99,8 @@ async def test_delete_dir_raises_on_a_file_changed_since_the_listing(client):
|
|
|
90
99
|
|
|
91
100
|
async def test_delete_dir_raises_on_a_blob_without_a_sha_before_deleting_anything(client):
|
|
92
101
|
client.get.return_value = _tree(
|
|
93
|
-
{"path": "
|
|
94
|
-
{"path": "
|
|
102
|
+
{"path": "a.j2", "type": "blob", "sha": "s1"},
|
|
103
|
+
{"path": "b.j2", "type": "blob"},
|
|
95
104
|
)
|
|
96
105
|
|
|
97
106
|
with pytest.raises(RuntimeError, match="no blob sha"):
|
|
@@ -123,6 +123,76 @@ async def test_list_server_error_propagates(client):
|
|
|
123
123
|
await GithubStorage().list()
|
|
124
124
|
|
|
125
125
|
|
|
126
|
+
TREES = "https://api.github.com/repos/acme/content/git/trees"
|
|
127
|
+
REF_URL = "https://api.github.com/repos/acme/content/git/ref/heads/dev"
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@pytest.mark.parametrize("directory", ["d/sub", "d/sub/", "/d/sub"])
|
|
131
|
+
async def test_list_of_a_directory_reads_only_its_tree_and_answers_full_paths(client, directory):
|
|
132
|
+
client.get.return_value = make_response(
|
|
133
|
+
status=200,
|
|
134
|
+
json_body={
|
|
135
|
+
"truncated": False,
|
|
136
|
+
"tree": [
|
|
137
|
+
{"path": "a.j2", "type": "blob", "sha": "s1"},
|
|
138
|
+
{"path": "e", "type": "tree", "sha": "t1"},
|
|
139
|
+
{"path": "e/b.j2", "type": "blob", "sha": "s2"},
|
|
140
|
+
],
|
|
141
|
+
},
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
files = await GithubStorage().list(directory)
|
|
145
|
+
|
|
146
|
+
assert files == ["d/sub/a.j2", "d/sub/e/b.j2"]
|
|
147
|
+
assert client.get.await_count == 1
|
|
148
|
+
assert client.get.call_args.args[0] == f"{TREES}/dev:d/sub"
|
|
149
|
+
assert client.get.call_args.kwargs["params"] == {"recursive": "1"}
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
async def test_list_of_a_directory_quotes_its_path_into_the_tree(client):
|
|
153
|
+
client.get.return_value = make_response(status=200, json_body={"truncated": False, "tree": []})
|
|
154
|
+
|
|
155
|
+
await GithubStorage().list("my dir/#1")
|
|
156
|
+
|
|
157
|
+
assert client.get.call_args.args[0] == f"{TREES}/dev:my%20dir/%231"
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
async def test_list_of_a_missing_directory_confirms_the_branch_and_is_empty(client):
|
|
161
|
+
client.get.side_effect = [
|
|
162
|
+
make_response(status=404),
|
|
163
|
+
make_response(status=200, json_body={"object": {"sha": "abc123"}}),
|
|
164
|
+
]
|
|
165
|
+
|
|
166
|
+
assert await GithubStorage().list("absent") == []
|
|
167
|
+
assert [call.args[0] for call in client.get.call_args_list] == [f"{TREES}/dev:absent", REF_URL]
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
async def test_list_of_a_directory_on_a_missing_branch_raises(client):
|
|
171
|
+
client.get.side_effect = [make_response(status=404), make_response(status=404)]
|
|
172
|
+
|
|
173
|
+
with pytest.raises(HTTPStatusError):
|
|
174
|
+
await GithubStorage().list("d")
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
async def test_list_of_a_path_naming_a_file_is_empty(client):
|
|
178
|
+
client.get.return_value = make_response(status=422)
|
|
179
|
+
|
|
180
|
+
assert await GithubStorage().list("a.j2") == []
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
async def test_list_of_a_directory_server_error_propagates(client):
|
|
184
|
+
client.get.return_value = make_response(status=500)
|
|
185
|
+
|
|
186
|
+
with pytest.raises(HTTPStatusError):
|
|
187
|
+
await GithubStorage().list("d")
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
async def test_list_of_a_directory_with_a_parent_segment_is_refused_before_any_call(client):
|
|
191
|
+
with pytest.raises(ValueError, match="path traversal"):
|
|
192
|
+
await GithubStorage().list("d/../e")
|
|
193
|
+
client.get.assert_not_called()
|
|
194
|
+
|
|
195
|
+
|
|
126
196
|
async def test_upload_create_puts_without_sha(client):
|
|
127
197
|
client.get.return_value = make_response(status=404)
|
|
128
198
|
client.put.return_value = make_response(status=200)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/src/tai42_storage_github/__init__.py
RENAMED
|
File without changes
|
{tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/src/tai42_storage_github/client.py
RENAMED
|
File without changes
|
{tai42_storage_github-2.0.2 → tai42_storage_github-3.0.0}/src/tai42_storage_github/docs/index.mdx
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|