tai42-storage-github 1.5.0__tar.gz → 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tai42_storage_github-1.5.0/src/tai42_storage_github.egg-info → tai42_storage_github-2.0.0}/PKG-INFO +25 -12
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/README.md +22 -9
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/pyproject.toml +3 -3
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github/docs/index.mdx +9 -4
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github/settings.py +8 -5
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github/storage.py +133 -29
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github/tai-plugin.yml +2 -2
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0/src/tai42_storage_github.egg-info}/PKG-INFO +25 -12
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github.egg-info/requires.txt +2 -2
- tai42_storage_github-2.0.0/tests/test_binary.py +343 -0
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/tests/test_security.py +32 -5
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/tests/test_text.py +45 -7
- tai42_storage_github-1.5.0/tests/test_binary.py +0 -98
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/LICENSE +0 -0
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/NOTICE +0 -0
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/setup.cfg +0 -0
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github/__init__.py +0 -0
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github/client.py +0 -0
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github/py.typed +0 -0
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github.egg-info/SOURCES.txt +0 -0
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github.egg-info/dependency_links.txt +0 -0
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github.egg-info/top_level.txt +0 -0
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/tests/test_delete_dir.py +0 -0
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/tests/test_import_graph.py +0 -0
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/tests/test_plugin_spec.py +0 -0
- {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/tests/test_registration.py +0 -0
{tai42_storage_github-1.5.0/src/tai42_storage_github.egg-info → tai42_storage_github-2.0.0}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tai42-storage-github
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.0
|
|
4
4
|
Summary: GitHub-backed Storage provider (text + binary/media) for the TAI ecosystem.
|
|
5
5
|
Author-email: tai42 <oss@tai42.ai>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -19,8 +19,8 @@ Requires-Python: >=3.13
|
|
|
19
19
|
Description-Content-Type: text/markdown
|
|
20
20
|
License-File: LICENSE
|
|
21
21
|
License-File: NOTICE
|
|
22
|
-
Requires-Dist: tai42-contract<
|
|
23
|
-
Requires-Dist: tai42-kit<
|
|
22
|
+
Requires-Dist: tai42-contract<25,>=24.0
|
|
23
|
+
Requires-Dist: tai42-kit<14,>=13.0
|
|
24
24
|
Requires-Dist: httpx>=0.28
|
|
25
25
|
Requires-Dist: pydantic>=2.12
|
|
26
26
|
Requires-Dist: pydantic-settings>=2.15.0
|
|
@@ -33,7 +33,8 @@ Dynamic: license-file
|
|
|
33
33
|
A GitHub-backed `Storage` provider for the TAI ecosystem. It stores content as
|
|
34
34
|
files in a GitHub repository and serves the full `tai42_contract.storage.Storage`
|
|
35
35
|
surface — the text methods (`load` / `list` / `upload` / `delete` / `delete_dir`)
|
|
36
|
-
plus the
|
|
36
|
+
plus the metadata read `stat`, the bounded read `load_range` and the binary
|
|
37
|
+
methods (`load_bytes` / `upload_bytes`).
|
|
37
38
|
|
|
38
39
|
## The TAI ecosystem
|
|
39
40
|
|
|
@@ -99,19 +100,31 @@ a log line, repr, or error message.
|
|
|
99
100
|
|
|
100
101
|
## Reads and writes
|
|
101
102
|
|
|
102
|
-
- **Reads** (`load`, `load_bytes`)
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
103
|
+
- **Reads** (`load`, `load_bytes`) are one Contents API call each at the
|
|
104
|
+
configured branch, asking for the raw form
|
|
105
|
+
(`Accept: application/vnd.github.raw+json`), which serves every byte of a file
|
|
106
|
+
of any size. A directory answers its JSON listing and reads as
|
|
107
|
+
`FileNotFoundError`.
|
|
108
|
+
- **Ranged reads** (`load_range`) resolve the branch's head commit through the
|
|
109
|
+
Git refs API, then read `raw.githubusercontent.com` at that commit with a
|
|
110
|
+
`Range` header and `Accept-Encoding: identity` (the raw host compresses, and a
|
|
111
|
+
compressed answer ranges over the encoded bytes): two calls, one of them REST.
|
|
112
|
+
Reading at the commit, never at the branch name, means a ranged read answers the
|
|
113
|
+
branch's current bytes, not a cached copy. An answer that ignores the range or
|
|
114
|
+
is still encoded raises; there is no whole-object read in its place. A
|
|
115
|
+
zero-length range sends no request.
|
|
116
|
+
- **Rate limit**: GitHub allows 60 REST calls an hour without a token and 5000
|
|
117
|
+
with one; `stat`, `load` and `load_bytes` cost one REST call each and
|
|
118
|
+
`load_range` one.
|
|
108
119
|
- **Writes** (`upload`, `upload_bytes`) go through the Contents API (base64) and
|
|
109
120
|
are capped conservatively at 1 MiB. A larger payload raises `ValueError` before
|
|
110
121
|
the request rather than truncating; the API's own `422` is the backstop.
|
|
111
122
|
- **Listing** uses the recursive Git Trees API (one request) and raises loudly on
|
|
112
123
|
a `truncated` response rather than acting on a partial listing.
|
|
113
|
-
- **`stat`**
|
|
114
|
-
|
|
124
|
+
- **`stat`** is one Contents API call asking for the object form
|
|
125
|
+
(`Accept: application/vnd.github.object+json`): it answers the file's `size`,
|
|
126
|
+
and infers `content_type` from the path suffix — GitHub stores no per-object
|
|
127
|
+
content-type. A missing path and a directory raise `FileNotFoundError`.
|
|
115
128
|
|
|
116
129
|
## Development
|
|
117
130
|
|
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
A GitHub-backed `Storage` provider for the TAI ecosystem. It stores content as
|
|
6
6
|
files in a GitHub repository and serves the full `tai42_contract.storage.Storage`
|
|
7
7
|
surface — the text methods (`load` / `list` / `upload` / `delete` / `delete_dir`)
|
|
8
|
-
plus the
|
|
8
|
+
plus the metadata read `stat`, the bounded read `load_range` and the binary
|
|
9
|
+
methods (`load_bytes` / `upload_bytes`).
|
|
9
10
|
|
|
10
11
|
## The TAI ecosystem
|
|
11
12
|
|
|
@@ -71,19 +72,31 @@ a log line, repr, or error message.
|
|
|
71
72
|
|
|
72
73
|
## Reads and writes
|
|
73
74
|
|
|
74
|
-
- **Reads** (`load`, `load_bytes`)
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
75
|
+
- **Reads** (`load`, `load_bytes`) are one Contents API call each at the
|
|
76
|
+
configured branch, asking for the raw form
|
|
77
|
+
(`Accept: application/vnd.github.raw+json`), which serves every byte of a file
|
|
78
|
+
of any size. A directory answers its JSON listing and reads as
|
|
79
|
+
`FileNotFoundError`.
|
|
80
|
+
- **Ranged reads** (`load_range`) resolve the branch's head commit through the
|
|
81
|
+
Git refs API, then read `raw.githubusercontent.com` at that commit with a
|
|
82
|
+
`Range` header and `Accept-Encoding: identity` (the raw host compresses, and a
|
|
83
|
+
compressed answer ranges over the encoded bytes): two calls, one of them REST.
|
|
84
|
+
Reading at the commit, never at the branch name, means a ranged read answers the
|
|
85
|
+
branch's current bytes, not a cached copy. An answer that ignores the range or
|
|
86
|
+
is still encoded raises; there is no whole-object read in its place. A
|
|
87
|
+
zero-length range sends no request.
|
|
88
|
+
- **Rate limit**: GitHub allows 60 REST calls an hour without a token and 5000
|
|
89
|
+
with one; `stat`, `load` and `load_bytes` cost one REST call each and
|
|
90
|
+
`load_range` one.
|
|
80
91
|
- **Writes** (`upload`, `upload_bytes`) go through the Contents API (base64) and
|
|
81
92
|
are capped conservatively at 1 MiB. A larger payload raises `ValueError` before
|
|
82
93
|
the request rather than truncating; the API's own `422` is the backstop.
|
|
83
94
|
- **Listing** uses the recursive Git Trees API (one request) and raises loudly on
|
|
84
95
|
a `truncated` response rather than acting on a partial listing.
|
|
85
|
-
- **`stat`**
|
|
86
|
-
|
|
96
|
+
- **`stat`** is one Contents API call asking for the object form
|
|
97
|
+
(`Accept: application/vnd.github.object+json`): it answers the file's `size`,
|
|
98
|
+
and infers `content_type` from the path suffix — GitHub stores no per-object
|
|
99
|
+
content-type. A missing path and a directory raise `FileNotFoundError`.
|
|
87
100
|
|
|
88
101
|
## Development
|
|
89
102
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "tai42-storage-github"
|
|
7
|
-
version = "
|
|
7
|
+
version = "2.0.0"
|
|
8
8
|
description = "GitHub-backed Storage provider (text + binary/media) for the TAI ecosystem."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.13"
|
|
@@ -26,9 +26,9 @@ dependencies = [
|
|
|
26
26
|
# The pure interface contract the backend registers through — Storage,
|
|
27
27
|
# ObjectStat, assert_not_root, and the tai42_app handle.
|
|
28
28
|
# The contract floor is rewritten by scripts/range_sync.py at release.
|
|
29
|
-
"tai42-contract>=
|
|
29
|
+
"tai42-contract>=24.0,<25",
|
|
30
30
|
# Pooled clients + settings primitives (PooledClient, TaiBaseSettings, cache).
|
|
31
|
-
"tai42-kit>=
|
|
31
|
+
"tai42-kit>=13.0,<14",
|
|
32
32
|
"httpx>=0.28",
|
|
33
33
|
"pydantic>=2.12",
|
|
34
34
|
"pydantic-settings>=2.15.0",
|
{tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github/docs/index.mdx
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
title: "GitHub storage"
|
|
3
|
-
description: "Configure the GitHub-repository storage provider, its token scopes, and the
|
|
3
|
+
description: "Configure the GitHub-repository storage provider, its token scopes, the 1 MiB write cap, and the API calls each read costs."
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
Stores content as files on a branch of a GitHub repository, which makes every write
|
|
@@ -33,11 +33,16 @@ is ignored, as are proxy environment variables.
|
|
|
33
33
|
|
|
34
34
|
<Warning>
|
|
35
35
|
Two limits shape what this provider suits. Writes are capped at 1 MiB and a larger
|
|
36
|
-
payload raises before the request. Reads
|
|
37
|
-
|
|
38
|
-
content a tool writes and immediately reads back.
|
|
36
|
+
payload raises before the request. Reads are GitHub REST API calls, and GitHub
|
|
37
|
+
allows 60 of them an hour without a token and 5000 with one.
|
|
39
38
|
</Warning>
|
|
40
39
|
|
|
40
|
+
A whole read and a stat are one REST call each, made at the configured branch, so
|
|
41
|
+
they answer the branch as it is right after a write. A ranged read resolves the
|
|
42
|
+
branch's head commit (one REST call) and then reads the raw content host at that
|
|
43
|
+
commit with a `Range` header; reading at the commit, never at the branch name,
|
|
44
|
+
means it never serves a cached copy of an older version.
|
|
45
|
+
|
|
41
46
|
Listing uses the recursive tree API and raises on a truncated response rather than
|
|
42
47
|
acting on a partial listing. The API host is fixed, so GitHub Enterprise Server is
|
|
43
48
|
not supported.
|
{tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github/settings.py
RENAMED
|
@@ -20,12 +20,15 @@ class GithubStorageSettings(TaiBaseSettings):
|
|
|
20
20
|
repo: str | None = None
|
|
21
21
|
branch: str = "main"
|
|
22
22
|
token: SecretStr | None = None
|
|
23
|
-
# Base-URL templates for the
|
|
24
|
-
# the real GitHub hosts. Overridable (like ``S3Settings.endpoint``)
|
|
25
|
-
# point the plugin at a hermetic stand-in
|
|
26
|
-
# ``{
|
|
27
|
-
|
|
23
|
+
# Base-URL templates for the four REST and raw surfaces the provider speaks,
|
|
24
|
+
# defaulting to the real GitHub hosts. Overridable (like ``S3Settings.endpoint``)
|
|
25
|
+
# so a test can point the plugin at a hermetic stand-in. Each carries
|
|
26
|
+
# ``{username}``/``{repo}`` placeholders the provider fills per request; the ref
|
|
27
|
+
# and trees templates also ``{branch}``, and the raw template ``{commit}``, the
|
|
28
|
+
# branch head commit a ranged read resolves first.
|
|
29
|
+
raw_base_url: str = "https://raw.githubusercontent.com/{username}/{repo}/{commit}"
|
|
28
30
|
contents_api_url: str = "https://api.github.com/repos/{username}/{repo}/contents"
|
|
31
|
+
ref_api_url: str = "https://api.github.com/repos/{username}/{repo}/git/ref/heads/{branch}"
|
|
29
32
|
trees_api_url: str = "https://api.github.com/repos/{username}/{repo}/git/trees/{branch}"
|
|
30
33
|
timeout_total: float = 15.0
|
|
31
34
|
max_connections: int = 200
|
{tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github/storage.py
RENAMED
|
@@ -1,20 +1,23 @@
|
|
|
1
1
|
"""GitHub-backed :class:`~tai42_contract.storage.Storage` provider.
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
the
|
|
3
|
+
``stat``, ``load`` and ``load_bytes`` are one Contents API call each at the
|
|
4
|
+
configured branch: the object form answers the size, the raw form every byte of
|
|
5
|
+
the file. ``load_range`` resolves the branch's head commit through the Git refs API
|
|
6
|
+
and reads the raw host at that commit with a ``Range`` header, so a ranged read
|
|
7
|
+
answers the branch's current bytes. Writes use the Contents API (base64) and are
|
|
8
|
+
capped below the API limit, raising before the request rather than truncating.
|
|
8
9
|
"""
|
|
9
10
|
|
|
10
11
|
from __future__ import annotations
|
|
11
12
|
|
|
12
13
|
import base64
|
|
13
14
|
import logging
|
|
15
|
+
import mimetypes
|
|
16
|
+
from typing import Final
|
|
14
17
|
|
|
15
18
|
import httpx
|
|
16
19
|
from tai42_contract.app import tai42_app
|
|
17
|
-
from tai42_contract.storage import Storage, StoragePathConflictError, assert_not_root
|
|
20
|
+
from tai42_contract.storage import ObjectStat, Storage, StoragePathConflictError, assert_not_root
|
|
18
21
|
|
|
19
22
|
from tai42_storage_github.client import GithubHttpxClient
|
|
20
23
|
from tai42_storage_github.settings import GithubStorageSettings, github_storage_settings
|
|
@@ -22,9 +25,15 @@ from tai42_storage_github.settings import GithubStorageSettings, github_storage_
|
|
|
22
25
|
logger = logging.getLogger(__name__)
|
|
23
26
|
|
|
24
27
|
# Conservative upload cap below GitHub's Contents API limit; an oversize file
|
|
25
|
-
# raises before the request. Reads are uncapped (raw
|
|
28
|
+
# raises before the request. Reads are uncapped (the raw form serves every byte).
|
|
26
29
|
MAX_UPLOAD_BYTES = 1024 * 1024
|
|
27
30
|
|
|
31
|
+
# The Contents API answer forms: the object form carries ``type`` and ``size``; the
|
|
32
|
+
# raw form is the file's bytes, and a directory answers its JSON listing instead.
|
|
33
|
+
_OBJECT_FORM: Final = "application/vnd.github.object+json"
|
|
34
|
+
_RAW_FORM: Final = "application/vnd.github.raw+json"
|
|
35
|
+
_JSON: Final = "application/json"
|
|
36
|
+
|
|
28
37
|
|
|
29
38
|
def _join(base: str, path: str) -> str:
|
|
30
39
|
"""Join ``path`` onto ``base``, dropping empty and stray-slash segments.
|
|
@@ -41,9 +50,9 @@ def _join(base: str, path: str) -> str:
|
|
|
41
50
|
return f"{base}/{joined}" if joined else base
|
|
42
51
|
|
|
43
52
|
|
|
44
|
-
def _raw_url(settings: GithubStorageSettings, path: str) -> str:
|
|
45
|
-
"""The
|
|
46
|
-
base = settings.raw_base_url.format(username=settings.username, repo=settings.repo,
|
|
53
|
+
def _raw_url(settings: GithubStorageSettings, commit: str, path: str) -> str:
|
|
54
|
+
"""The raw-host URL for ``path`` at ``commit``, from the configured raw base-URL template."""
|
|
55
|
+
base = settings.raw_base_url.format(username=settings.username, repo=settings.repo, commit=commit)
|
|
47
56
|
return _join(base, path)
|
|
48
57
|
|
|
49
58
|
|
|
@@ -79,14 +88,19 @@ def _auth_headers(settings: GithubStorageSettings) -> dict[str, str]:
|
|
|
79
88
|
return {"Authorization": f"Bearer {settings.token.get_secret_value()}"}
|
|
80
89
|
|
|
81
90
|
|
|
82
|
-
def _api_headers(settings: GithubStorageSettings) -> dict[str, str]:
|
|
91
|
+
def _api_headers(settings: GithubStorageSettings, accept: str = "application/vnd.github+json") -> dict[str, str]:
|
|
83
92
|
return {
|
|
84
93
|
**_auth_headers(settings),
|
|
85
|
-
"Accept":
|
|
94
|
+
"Accept": accept,
|
|
86
95
|
"X-GitHub-Api-Version": "2022-11-28",
|
|
87
96
|
}
|
|
88
97
|
|
|
89
98
|
|
|
99
|
+
def _media_type(resp: httpx.Response) -> str:
|
|
100
|
+
"""The ``Content-Type`` media type, without its parameters (``; charset=utf-8``)."""
|
|
101
|
+
return resp.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
|
102
|
+
|
|
103
|
+
|
|
90
104
|
def _raise_for_status(resp: httpx.Response, action: str, path: str, url: str) -> None:
|
|
91
105
|
"""Surface an HTTP error loudly; the log names the URL/status/body, never the token."""
|
|
92
106
|
try:
|
|
@@ -98,35 +112,125 @@ def _raise_for_status(resp: httpx.Response, action: str, path: str, url: str) ->
|
|
|
98
112
|
raise
|
|
99
113
|
|
|
100
114
|
|
|
115
|
+
def _raise_unless_ok(resp: httpx.Response, expected: int, action: str, path: str, url: str) -> None:
|
|
116
|
+
"""Raise on any status other than ``expected``: an HTTP error loudly, an unexpected success as a fault."""
|
|
117
|
+
if resp.status_code == expected:
|
|
118
|
+
return
|
|
119
|
+
_raise_for_status(resp, action, path, url)
|
|
120
|
+
raise RuntimeError(f"GitHub answered {resp.status_code} {action} {path!r} at {url}; expected {expected}.")
|
|
121
|
+
|
|
122
|
+
|
|
101
123
|
# Importing this module registers GithubStorage as the active storage provider.
|
|
102
124
|
@tai42_app.storage.register_storage
|
|
103
125
|
class GithubStorage(Storage):
|
|
104
|
-
"""GitHub-backed storage: reads via the
|
|
126
|
+
"""GitHub-backed storage: reads and writes via the Contents API at the branch, ranged reads via the raw host."""
|
|
105
127
|
|
|
106
128
|
async def load(self, path: str) -> str:
|
|
107
|
-
"""Read ``path`` as text
|
|
129
|
+
"""Read ``path`` as UTF-8 text; a missing object raises ``FileNotFoundError``.
|
|
130
|
+
|
|
131
|
+
The raw bytes are strictly decoded, so an object that is not UTF-8 text raises
|
|
132
|
+
``UnicodeDecodeError`` rather than reading back with substituted characters.
|
|
133
|
+
"""
|
|
134
|
+
return (await self.load_bytes(path)).decode("utf-8")
|
|
135
|
+
|
|
136
|
+
async def load_bytes(self, path: str) -> bytes:
|
|
137
|
+
"""Read every byte of ``path`` through the Contents API raw form; a missing object raises ``FileNotFoundError``.
|
|
138
|
+
|
|
139
|
+
A directory answers its JSON listing instead of the raw form, and is not an object.
|
|
140
|
+
"""
|
|
108
141
|
settings = _configured_settings()
|
|
109
|
-
url =
|
|
142
|
+
url = _contents_url(settings, path)
|
|
110
143
|
async with tai42_app.clients.client_ctx(GithubHttpxClient) as client:
|
|
111
|
-
resp = await client.get(url, headers=
|
|
112
|
-
|
|
113
|
-
|
|
144
|
+
resp = await client.get(url, headers=_api_headers(settings, _RAW_FORM), params={"ref": settings.branch})
|
|
145
|
+
if resp.status_code == 404:
|
|
146
|
+
raise FileNotFoundError(f"Object not found: {path}")
|
|
147
|
+
_raise_unless_ok(resp, 200, "loading", path, url)
|
|
148
|
+
media_type = _media_type(resp)
|
|
149
|
+
if media_type == _RAW_FORM:
|
|
150
|
+
return resp.content
|
|
151
|
+
if media_type == _JSON:
|
|
152
|
+
raise FileNotFoundError(f"Object not found: {path}")
|
|
153
|
+
raise RuntimeError(f"GitHub answered an unexpected Content-Type {media_type!r} loading {path!r} at {url}.")
|
|
114
154
|
|
|
115
|
-
async def
|
|
116
|
-
"""
|
|
155
|
+
async def stat(self, path: str) -> ObjectStat:
|
|
156
|
+
"""The byte size from the Contents API object form and the path-inferred content type.
|
|
157
|
+
|
|
158
|
+
GitHub stores no content type. A missing object, and a directory (not an object),
|
|
159
|
+
raise ``FileNotFoundError``; a file answer without an integer size raises ``RuntimeError``.
|
|
160
|
+
"""
|
|
161
|
+
settings = _configured_settings()
|
|
162
|
+
url = _contents_url(settings, path)
|
|
163
|
+
async with tai42_app.clients.client_ctx(GithubHttpxClient) as client:
|
|
164
|
+
resp = await client.get(url, headers=_api_headers(settings, _OBJECT_FORM), params={"ref": settings.branch})
|
|
165
|
+
if resp.status_code == 404:
|
|
166
|
+
raise FileNotFoundError(f"Object not found: {path}")
|
|
167
|
+
_raise_unless_ok(resp, 200, "reading the size of", path, url)
|
|
168
|
+
data = resp.json()
|
|
169
|
+
if not isinstance(data, dict) or data.get("type") != "file":
|
|
170
|
+
raise FileNotFoundError(f"Object not found: {path}")
|
|
171
|
+
size = data.get("size")
|
|
172
|
+
if type(size) is not int:
|
|
173
|
+
raise RuntimeError(f"GitHub Contents API answered a file without an integer size at {url}.")
|
|
174
|
+
content_type, _ = mimetypes.guess_type(path)
|
|
175
|
+
return ObjectStat(content_type=content_type, size=size)
|
|
176
|
+
|
|
177
|
+
async def load_range(self, path: str, start: int, length: int) -> bytes:
|
|
178
|
+
"""At most ``length`` bytes of ``path`` from ``start``, read at the branch's current head commit.
|
|
179
|
+
|
|
180
|
+
Resolves the head commit, then reads the raw host at that commit with a ``Range``
|
|
181
|
+
header and the identity encoding, since a compressed answer ranges over the encoded
|
|
182
|
+
bytes. ``b""`` for ``length == 0`` (no request) and for a start at or past the end
|
|
183
|
+
(416); ``FileNotFoundError`` for a missing object or a directory (404). An answer
|
|
184
|
+
that ignores the range (200), is still encoded, or runs longer than ``length``
|
|
185
|
+
raises ``RuntimeError``.
|
|
186
|
+
"""
|
|
187
|
+
if length == 0:
|
|
188
|
+
return b""
|
|
117
189
|
settings = _configured_settings()
|
|
118
|
-
url = _raw_url(settings, path)
|
|
119
190
|
async with tai42_app.clients.client_ctx(GithubHttpxClient) as client:
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
191
|
+
commit = await self._head_commit(client, settings, path)
|
|
192
|
+
url = _raw_url(settings, commit, path)
|
|
193
|
+
headers = {
|
|
194
|
+
**_auth_headers(settings),
|
|
195
|
+
"Accept-Encoding": "identity",
|
|
196
|
+
"Range": f"bytes={start}-{start + length - 1}",
|
|
197
|
+
}
|
|
198
|
+
async with client.stream("GET", url, headers=headers) as resp:
|
|
199
|
+
if resp.status_code == 404:
|
|
200
|
+
raise FileNotFoundError(f"Object not found: {path}")
|
|
201
|
+
if resp.status_code == 416:
|
|
202
|
+
return b""
|
|
203
|
+
if resp.status_code == 200:
|
|
204
|
+
raise RuntimeError(f"GitHub answered 200, the whole object, to a ranged read of {url}.")
|
|
205
|
+
if resp.status_code != 206:
|
|
206
|
+
await resp.aread()
|
|
207
|
+
_raise_unless_ok(resp, 206, "loading a range of", path, url)
|
|
208
|
+
encoding = resp.headers.get("content-encoding", "identity").strip().lower()
|
|
209
|
+
if encoding != "identity":
|
|
210
|
+
raise RuntimeError(
|
|
211
|
+
f"GitHub answered a {encoding}-encoded range of {url}; the identity encoding was asked."
|
|
212
|
+
)
|
|
213
|
+
chunks: list[bytes] = []
|
|
214
|
+
received = 0
|
|
215
|
+
async for chunk in resp.aiter_bytes():
|
|
216
|
+
received += len(chunk)
|
|
217
|
+
if received > length:
|
|
218
|
+
raise RuntimeError(f"GitHub answered more than the {length} bytes asked of {url}.")
|
|
219
|
+
chunks.append(chunk)
|
|
220
|
+
return b"".join(chunks)
|
|
124
221
|
|
|
125
222
|
@staticmethod
|
|
126
|
-
def
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
223
|
+
async def _head_commit(client: httpx.AsyncClient, settings: GithubStorageSettings, path: str) -> str:
|
|
224
|
+
"""The branch's current head commit sha, from the Git refs API."""
|
|
225
|
+
url = settings.ref_api_url.format(username=settings.username, repo=settings.repo, branch=settings.branch)
|
|
226
|
+
resp = await client.get(url, headers=_api_headers(settings))
|
|
227
|
+
_raise_unless_ok(resp, 200, "resolving the branch head for", path, url)
|
|
228
|
+
data = resp.json()
|
|
229
|
+
target = data.get("object") if isinstance(data, dict) else None
|
|
230
|
+
sha = target.get("sha") if isinstance(target, dict) else None
|
|
231
|
+
if not isinstance(sha, str) or not sha:
|
|
232
|
+
raise RuntimeError(f"GitHub answered no head commit sha for the branch at {url}.")
|
|
233
|
+
return sha
|
|
130
234
|
|
|
131
235
|
async def list(self) -> list[str]:
|
|
132
236
|
"""Every blob path in the repo."""
|
{tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github/tai-plugin.yml
RENAMED
|
@@ -2,12 +2,12 @@ spec_version: 1
|
|
|
2
2
|
namespace: tai42
|
|
3
3
|
name: storage-github
|
|
4
4
|
package: tai42-storage-github
|
|
5
|
-
version:
|
|
5
|
+
version: 2.0.0
|
|
6
6
|
description: GitHub-backed Storage provider (text + binary/media) for the TAI ecosystem.
|
|
7
7
|
license: Apache-2.0
|
|
8
8
|
homepage: https://tai42.ai
|
|
9
9
|
repository: https://github.com/tai42ai/tai42/tree/main/plugins/storage-github
|
|
10
|
-
contract: '>=
|
|
10
|
+
contract: '>=24.0,<25'
|
|
11
11
|
categories:
|
|
12
12
|
- storage
|
|
13
13
|
tags:
|
{tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0/src/tai42_storage_github.egg-info}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tai42-storage-github
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.0
|
|
4
4
|
Summary: GitHub-backed Storage provider (text + binary/media) for the TAI ecosystem.
|
|
5
5
|
Author-email: tai42 <oss@tai42.ai>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -19,8 +19,8 @@ Requires-Python: >=3.13
|
|
|
19
19
|
Description-Content-Type: text/markdown
|
|
20
20
|
License-File: LICENSE
|
|
21
21
|
License-File: NOTICE
|
|
22
|
-
Requires-Dist: tai42-contract<
|
|
23
|
-
Requires-Dist: tai42-kit<
|
|
22
|
+
Requires-Dist: tai42-contract<25,>=24.0
|
|
23
|
+
Requires-Dist: tai42-kit<14,>=13.0
|
|
24
24
|
Requires-Dist: httpx>=0.28
|
|
25
25
|
Requires-Dist: pydantic>=2.12
|
|
26
26
|
Requires-Dist: pydantic-settings>=2.15.0
|
|
@@ -33,7 +33,8 @@ Dynamic: license-file
|
|
|
33
33
|
A GitHub-backed `Storage` provider for the TAI ecosystem. It stores content as
|
|
34
34
|
files in a GitHub repository and serves the full `tai42_contract.storage.Storage`
|
|
35
35
|
surface — the text methods (`load` / `list` / `upload` / `delete` / `delete_dir`)
|
|
36
|
-
plus the
|
|
36
|
+
plus the metadata read `stat`, the bounded read `load_range` and the binary
|
|
37
|
+
methods (`load_bytes` / `upload_bytes`).
|
|
37
38
|
|
|
38
39
|
## The TAI ecosystem
|
|
39
40
|
|
|
@@ -99,19 +100,31 @@ a log line, repr, or error message.
|
|
|
99
100
|
|
|
100
101
|
## Reads and writes
|
|
101
102
|
|
|
102
|
-
- **Reads** (`load`, `load_bytes`)
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
103
|
+
- **Reads** (`load`, `load_bytes`) are one Contents API call each at the
|
|
104
|
+
configured branch, asking for the raw form
|
|
105
|
+
(`Accept: application/vnd.github.raw+json`), which serves every byte of a file
|
|
106
|
+
of any size. A directory answers its JSON listing and reads as
|
|
107
|
+
`FileNotFoundError`.
|
|
108
|
+
- **Ranged reads** (`load_range`) resolve the branch's head commit through the
|
|
109
|
+
Git refs API, then read `raw.githubusercontent.com` at that commit with a
|
|
110
|
+
`Range` header and `Accept-Encoding: identity` (the raw host compresses, and a
|
|
111
|
+
compressed answer ranges over the encoded bytes): two calls, one of them REST.
|
|
112
|
+
Reading at the commit, never at the branch name, means a ranged read answers the
|
|
113
|
+
branch's current bytes, not a cached copy. An answer that ignores the range or
|
|
114
|
+
is still encoded raises; there is no whole-object read in its place. A
|
|
115
|
+
zero-length range sends no request.
|
|
116
|
+
- **Rate limit**: GitHub allows 60 REST calls an hour without a token and 5000
|
|
117
|
+
with one; `stat`, `load` and `load_bytes` cost one REST call each and
|
|
118
|
+
`load_range` one.
|
|
108
119
|
- **Writes** (`upload`, `upload_bytes`) go through the Contents API (base64) and
|
|
109
120
|
are capped conservatively at 1 MiB. A larger payload raises `ValueError` before
|
|
110
121
|
the request rather than truncating; the API's own `422` is the backstop.
|
|
111
122
|
- **Listing** uses the recursive Git Trees API (one request) and raises loudly on
|
|
112
123
|
a `truncated` response rather than acting on a partial listing.
|
|
113
|
-
- **`stat`**
|
|
114
|
-
|
|
124
|
+
- **`stat`** is one Contents API call asking for the object form
|
|
125
|
+
(`Accept: application/vnd.github.object+json`): it answers the file's `size`,
|
|
126
|
+
and infers `content_type` from the path suffix — GitHub stores no per-object
|
|
127
|
+
content-type. A missing path and a directory raise `FileNotFoundError`.
|
|
115
128
|
|
|
116
129
|
## Development
|
|
117
130
|
|
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
"""Binary/media surface: load_bytes / upload_bytes / stat / load_range, plus the size caps.
|
|
2
|
+
|
|
3
|
+
Whole reads and ``stat`` are one Contents API call at the branch (the raw form serves
|
|
4
|
+
every byte, the object form the size); a ranged read resolves the branch head and
|
|
5
|
+
reads the raw host at that commit; uploads are capped loudly.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import base64
|
|
11
|
+
|
|
12
|
+
import pytest
|
|
13
|
+
from httpx import HTTPStatusError
|
|
14
|
+
from tai42_contract.storage import ObjectStat
|
|
15
|
+
|
|
16
|
+
from tai42_storage_github import GithubStorage
|
|
17
|
+
from tai42_storage_github.storage import MAX_UPLOAD_BYTES
|
|
18
|
+
|
|
19
|
+
from .conftest import make_response, set_stream
|
|
20
|
+
|
|
21
|
+
pytestmark = pytest.mark.usefixtures("client")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
CONTENTS = "https://api.github.com/repos/acme/content/contents"
|
|
25
|
+
RAW_FORM = "application/vnd.github.raw+json"
|
|
26
|
+
RAW_FORM_HEADERS = {"Content-Type": f"{RAW_FORM}; charset=utf-8"}
|
|
27
|
+
OBJECT_FORM = "application/vnd.github.object+json"
|
|
28
|
+
HEAD_SHA = "0123456789abcdef0123456789abcdef01234567"
|
|
29
|
+
REF_URL = "https://api.github.com/repos/acme/content/git/ref/heads/dev"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _ref_answer(sha: object = HEAD_SHA) -> object:
|
|
33
|
+
return make_response(status=200, json_body={"ref": "refs/heads/dev", "object": {"type": "commit", "sha": sha}})
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
async def test_binary_round_trip_is_byte_identical(client):
|
|
37
|
+
# Every byte value, including bytes that are not valid UTF-8.
|
|
38
|
+
data = bytes(range(256)) * 8
|
|
39
|
+
|
|
40
|
+
client.get.return_value = make_response(status=404) # a create
|
|
41
|
+
client.put.return_value = make_response(status=200)
|
|
42
|
+
await GithubStorage().upload_bytes("media/blob.bin", data)
|
|
43
|
+
|
|
44
|
+
encoded = client.put.call_args.kwargs["json"]["content"]
|
|
45
|
+
assert base64.b64decode(encoded) == data
|
|
46
|
+
|
|
47
|
+
client.get.reset_mock()
|
|
48
|
+
client.get.return_value = make_response(status=200, content=base64.b64decode(encoded), headers=RAW_FORM_HEADERS)
|
|
49
|
+
loaded = await GithubStorage().load_bytes("media/blob.bin")
|
|
50
|
+
|
|
51
|
+
assert loaded == data
|
|
52
|
+
client.get.assert_awaited_once()
|
|
53
|
+
args, kwargs = client.get.call_args
|
|
54
|
+
assert args[0] == f"{CONTENTS}/media/blob.bin"
|
|
55
|
+
assert kwargs["params"] == {"ref": "dev"}
|
|
56
|
+
assert kwargs["headers"]["Accept"] == RAW_FORM
|
|
57
|
+
assert "Range" not in kwargs["headers"]
|
|
58
|
+
client.stream.assert_not_called()
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
async def test_load_bytes_large_file_is_read_whole(client):
|
|
62
|
+
big = b"\x00\xff" * (1024 * 1024) # 2 MiB
|
|
63
|
+
client.get.return_value = make_response(status=200, content=big, headers=RAW_FORM_HEADERS)
|
|
64
|
+
|
|
65
|
+
loaded = await GithubStorage().load_bytes("big.bin")
|
|
66
|
+
|
|
67
|
+
assert loaded == big
|
|
68
|
+
for call in client.get.call_args_list:
|
|
69
|
+
assert "raw.githubusercontent.com" not in call.args[0]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
async def test_load_bytes_missing_raises_filenotfound(client):
|
|
73
|
+
client.get.return_value = make_response(status=404)
|
|
74
|
+
|
|
75
|
+
with pytest.raises(FileNotFoundError):
|
|
76
|
+
await GithubStorage().load_bytes("missing.bin")
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
async def test_load_bytes_of_a_directory_listing_raises_filenotfound(client):
|
|
80
|
+
client.get.return_value = make_response(
|
|
81
|
+
status=200, content=b"[]", headers={"Content-Type": "application/json; charset=utf-8"}
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
with pytest.raises(FileNotFoundError, match="Object not found: media"):
|
|
85
|
+
await GithubStorage().load_bytes("media")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
async def test_load_bytes_unexpected_success_status_raises(client):
|
|
89
|
+
client.get.return_value = make_response(status=204, headers=RAW_FORM_HEADERS)
|
|
90
|
+
|
|
91
|
+
with pytest.raises(RuntimeError, match="204"):
|
|
92
|
+
await GithubStorage().load_bytes("x.bin")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
async def test_upload_bytes_over_cap_raises_before_request(client):
|
|
96
|
+
oversize = b"\x00" * (MAX_UPLOAD_BYTES + 1)
|
|
97
|
+
|
|
98
|
+
with pytest.raises(ValueError, match="upload cap"):
|
|
99
|
+
await GithubStorage().upload_bytes("big.bin", oversize)
|
|
100
|
+
|
|
101
|
+
# The cap raises BEFORE any network call — no partial/truncated upload.
|
|
102
|
+
client.get.assert_not_called()
|
|
103
|
+
client.put.assert_not_called()
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
async def test_upload_bytes_at_cap_is_allowed(client):
|
|
107
|
+
at_cap = b"\x00" * MAX_UPLOAD_BYTES
|
|
108
|
+
client.get.return_value = make_response(status=404)
|
|
109
|
+
client.put.return_value = make_response(status=200)
|
|
110
|
+
|
|
111
|
+
await GithubStorage().upload_bytes("edge.bin", at_cap)
|
|
112
|
+
|
|
113
|
+
client.put.assert_awaited_once()
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
async def test_upload_bytes_422_backstop_raises(client):
|
|
117
|
+
client.get.return_value = make_response(status=404)
|
|
118
|
+
client.put.return_value = make_response(status=422)
|
|
119
|
+
|
|
120
|
+
with pytest.raises(HTTPStatusError):
|
|
121
|
+
await GithubStorage().upload_bytes("x.bin", b"within-cap")
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
# --- stat ---------------------------------------------------------------------
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
async def test_stat_reads_the_object_form_size_at_the_branch(client):
|
|
128
|
+
client.get.return_value = make_response(
|
|
129
|
+
status=200, json_body={"type": "file", "size": 26214400, "path": "a/b/logo.png"}
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
assert await GithubStorage().stat("a/b/logo.png") == ObjectStat(content_type="image/png", size=26214400)
|
|
133
|
+
client.get.assert_awaited_once()
|
|
134
|
+
args, kwargs = client.get.call_args
|
|
135
|
+
assert args[0] == f"{CONTENTS}/a/b/logo.png"
|
|
136
|
+
assert kwargs["params"] == {"ref": "dev"}
|
|
137
|
+
assert kwargs["headers"]["Accept"] == OBJECT_FORM
|
|
138
|
+
assert kwargs["headers"]["X-GitHub-Api-Version"] == "2022-11-28"
|
|
139
|
+
assert "Range" not in kwargs["headers"]
|
|
140
|
+
client.stream.assert_not_called()
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
async def test_stat_unknown_suffix_is_none(client):
|
|
144
|
+
client.get.return_value = make_response(status=200, json_body={"type": "file", "size": 3})
|
|
145
|
+
|
|
146
|
+
assert await GithubStorage().stat("mystery.unknownext") == ObjectStat(content_type=None, size=3)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
async def test_stat_missing_raises_filenotfound(client):
|
|
150
|
+
client.get.return_value = make_response(status=404)
|
|
151
|
+
|
|
152
|
+
with pytest.raises(FileNotFoundError, match=r"Object not found: gone\.png"):
|
|
153
|
+
await GithubStorage().stat("gone.png")
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
@pytest.mark.parametrize(
|
|
157
|
+
"answer",
|
|
158
|
+
[[{"path": "d/a.j2", "type": "file"}], {"type": "dir", "entries": []}],
|
|
159
|
+
ids=["default-form-list", "object-form-dir"],
|
|
160
|
+
)
|
|
161
|
+
async def test_stat_of_a_directory_raises_filenotfound(client, answer):
|
|
162
|
+
client.get.return_value = make_response(status=200, json_body=answer)
|
|
163
|
+
|
|
164
|
+
with pytest.raises(FileNotFoundError, match="Object not found: d"):
|
|
165
|
+
await GithubStorage().stat("d")
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
@pytest.mark.parametrize("answer", [{"type": "file"}, {"type": "file", "size": "12"}, {"type": "file", "size": True}])
|
|
169
|
+
async def test_stat_of_a_file_without_an_integer_size_raises(client, answer):
|
|
170
|
+
client.get.return_value = make_response(status=200, json_body=answer)
|
|
171
|
+
|
|
172
|
+
with pytest.raises(RuntimeError, match=r"contents/x\.bin"):
|
|
173
|
+
await GithubStorage().stat("x.bin")
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
async def test_stat_server_error_propagates(client):
|
|
177
|
+
client.get.return_value = make_response(status=500)
|
|
178
|
+
|
|
179
|
+
with pytest.raises(HTTPStatusError):
|
|
180
|
+
await GithubStorage().stat("x.bin")
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
# --- load_range -----------------------------------------------------------------
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
async def test_load_range_resolves_the_head_then_reads_the_raw_host_at_that_commit(client, settings):
|
|
187
|
+
client.get.return_value = _ref_answer()
|
|
188
|
+
resp = make_response(status=206, content=b"\x89PNG", headers={"Content-Range": "bytes 0-3/100"})
|
|
189
|
+
set_stream(client, resp)
|
|
190
|
+
|
|
191
|
+
assert await GithubStorage().load_range("p", 0, 4) == b"\x89PNG"
|
|
192
|
+
|
|
193
|
+
assert [c[0] for c in client.mock_calls if c[0] in ("get", "stream")] == ["get", "stream"]
|
|
194
|
+
ref_args, ref_kwargs = client.get.call_args
|
|
195
|
+
assert ref_args[0] == REF_URL
|
|
196
|
+
assert ref_kwargs["headers"]["Accept"] == "application/vnd.github+json"
|
|
197
|
+
assert ref_kwargs["headers"]["X-GitHub-Api-Version"] == "2022-11-28"
|
|
198
|
+
assert ref_kwargs["headers"]["Authorization"] == f"Bearer {settings.token.get_secret_value()}"
|
|
199
|
+
(method, url), raw_kwargs = client.stream.call_args
|
|
200
|
+
assert method == "GET"
|
|
201
|
+
assert url == f"https://raw.githubusercontent.com/acme/content/{HEAD_SHA}/p"
|
|
202
|
+
assert raw_kwargs["headers"] == {
|
|
203
|
+
"Authorization": f"Bearer {settings.token.get_secret_value()}",
|
|
204
|
+
"Accept-Encoding": "identity",
|
|
205
|
+
"Range": "bytes=0-3",
|
|
206
|
+
}
|
|
207
|
+
for call in [*client.get.call_args_list, *client.stream.call_args_list]:
|
|
208
|
+
assert "refs/heads/dev" not in str(call)
|
|
209
|
+
assert "/dev/" not in str(call)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
async def test_load_range_of_the_tail_sends_the_tail_range(client):
|
|
213
|
+
client.get.return_value = _ref_answer()
|
|
214
|
+
set_stream(client, make_response(status=206, content=b"tail"))
|
|
215
|
+
|
|
216
|
+
assert await GithubStorage().load_range("blob.bin", 96, 4) == b"tail"
|
|
217
|
+
assert client.stream.call_args.kwargs["headers"]["Range"] == "bytes=96-99"
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
async def test_load_range_running_past_the_end_answers_the_tail_only(client):
|
|
221
|
+
# The raw host clamps a range whose end is past the object to the bytes it has.
|
|
222
|
+
client.get.return_value = _ref_answer()
|
|
223
|
+
set_stream(client, make_response(status=206, content=b"ab", headers={"Content-Range": "bytes 98-99/100"}))
|
|
224
|
+
|
|
225
|
+
assert await GithubStorage().load_range("blob.bin", 98, 10) == b"ab"
|
|
226
|
+
assert client.stream.call_args.kwargs["headers"]["Range"] == "bytes=98-107"
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
async def test_load_range_past_the_end_answers_empty(client):
|
|
230
|
+
client.get.return_value = _ref_answer()
|
|
231
|
+
set_stream(client, make_response(status=416, headers={"Content-Range": "bytes */100"}))
|
|
232
|
+
|
|
233
|
+
assert await GithubStorage().load_range("blob.bin", 100, 4) == b""
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
async def test_load_range_missing_or_directory_raises_filenotfound(client):
|
|
237
|
+
client.get.return_value = _ref_answer()
|
|
238
|
+
set_stream(client, make_response(status=404))
|
|
239
|
+
|
|
240
|
+
with pytest.raises(FileNotFoundError, match="Object not found: jq"):
|
|
241
|
+
await GithubStorage().load_range("jq", 0, 4)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
async def test_load_range_whole_object_answer_raises_without_reading_the_body(client):
|
|
245
|
+
client.get.return_value = _ref_answer()
|
|
246
|
+
resp = make_response(status=200, content=b"x" * 1000)
|
|
247
|
+
set_stream(client, resp)
|
|
248
|
+
|
|
249
|
+
with pytest.raises(RuntimeError, match="200") as exc:
|
|
250
|
+
await GithubStorage().load_range("blob.bin", 0, 4)
|
|
251
|
+
|
|
252
|
+
assert f"raw.githubusercontent.com/acme/content/{HEAD_SHA}/blob.bin" in str(exc.value)
|
|
253
|
+
assert resp.body_read is False
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
async def test_load_range_encoded_answer_raises_without_reading_the_body(client):
|
|
257
|
+
# A compressed answer ranges over the encoded bytes, not the object's.
|
|
258
|
+
client.get.return_value = _ref_answer()
|
|
259
|
+
resp = make_response(
|
|
260
|
+
status=206,
|
|
261
|
+
content=b"\x1f\x8b\x08\x00\x00",
|
|
262
|
+
headers={"Content-Range": "bytes 0-4/47", "Content-Encoding": "gzip"},
|
|
263
|
+
)
|
|
264
|
+
set_stream(client, resp)
|
|
265
|
+
|
|
266
|
+
with pytest.raises(RuntimeError, match="gzip") as exc:
|
|
267
|
+
await GithubStorage().load_range("blob.bin", 0, 5)
|
|
268
|
+
|
|
269
|
+
assert f"raw.githubusercontent.com/acme/content/{HEAD_SHA}/blob.bin" in str(exc.value)
|
|
270
|
+
assert resp.body_read is False
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
async def test_load_range_longer_answer_than_asked_raises(client):
|
|
274
|
+
client.get.return_value = _ref_answer()
|
|
275
|
+
set_stream(client, make_response(status=206, chunks=[b"abc", b"def"]))
|
|
276
|
+
|
|
277
|
+
with pytest.raises(RuntimeError, match="more than the 4 bytes"):
|
|
278
|
+
await GithubStorage().load_range("blob.bin", 0, 4)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
async def test_load_range_server_error_propagates(client):
|
|
282
|
+
client.get.return_value = _ref_answer()
|
|
283
|
+
set_stream(client, make_response(status=500, text="boom"))
|
|
284
|
+
|
|
285
|
+
with pytest.raises(HTTPStatusError):
|
|
286
|
+
await GithubStorage().load_range("blob.bin", 0, 4)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
async def test_load_range_unexpected_success_status_raises(client):
|
|
290
|
+
client.get.return_value = _ref_answer()
|
|
291
|
+
set_stream(client, make_response(status=204))
|
|
292
|
+
|
|
293
|
+
with pytest.raises(RuntimeError, match="204"):
|
|
294
|
+
await GithubStorage().load_range("blob.bin", 0, 4)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
async def test_load_range_failed_head_lookup_raises_before_the_read(client):
|
|
298
|
+
client.get.return_value = make_response(status=404)
|
|
299
|
+
|
|
300
|
+
with pytest.raises(HTTPStatusError):
|
|
301
|
+
await GithubStorage().load_range("blob.bin", 0, 4)
|
|
302
|
+
|
|
303
|
+
client.stream.assert_not_called()
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
@pytest.mark.parametrize(
|
|
307
|
+
"body",
|
|
308
|
+
[
|
|
309
|
+
{"ref": "refs/heads/dev"},
|
|
310
|
+
{"object": {"type": "commit"}},
|
|
311
|
+
{"object": {"sha": ""}},
|
|
312
|
+
{"object": {"sha": 7}},
|
|
313
|
+
{"object": "abc"},
|
|
314
|
+
[{"ref": "refs/heads/dev-a"}],
|
|
315
|
+
],
|
|
316
|
+
ids=["no-object", "no-sha", "empty-sha", "int-sha", "object-not-a-dict", "list"],
|
|
317
|
+
)
|
|
318
|
+
async def test_load_range_head_answer_without_a_commit_sha_raises(client, body):
|
|
319
|
+
client.get.return_value = make_response(status=200, json_body=body)
|
|
320
|
+
|
|
321
|
+
with pytest.raises(RuntimeError, match="git/ref/heads/dev"):
|
|
322
|
+
await GithubStorage().load_range("blob.bin", 0, 4)
|
|
323
|
+
|
|
324
|
+
client.stream.assert_not_called()
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
async def test_load_range_zero_length_sends_no_request(client):
|
|
328
|
+
assert await GithubStorage().load_range("blob.bin", 0, 0) == b""
|
|
329
|
+
|
|
330
|
+
client.get.assert_not_called()
|
|
331
|
+
client.stream.assert_not_called()
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
async def test_load_range_resolves_the_head_on_every_call(client):
|
|
335
|
+
shas = ["a" * 40, "b" * 40]
|
|
336
|
+
client.get.side_effect = [_ref_answer(shas[0]), _ref_answer(shas[1])]
|
|
337
|
+
set_stream(client, make_response(status=206, content=b"x"))
|
|
338
|
+
|
|
339
|
+
await GithubStorage().load_range("p", 0, 1)
|
|
340
|
+
await GithubStorage().load_range("p", 0, 1)
|
|
341
|
+
|
|
342
|
+
urls = [c.args[1] for c in client.stream.call_args_list]
|
|
343
|
+
assert urls == [f"https://raw.githubusercontent.com/acme/content/{sha}/p" for sha in shas]
|
|
@@ -10,7 +10,7 @@ from httpx import HTTPStatusError
|
|
|
10
10
|
|
|
11
11
|
from tai42_storage_github import GithubStorage
|
|
12
12
|
|
|
13
|
-
from .conftest import make_response
|
|
13
|
+
from .conftest import make_response, set_stream
|
|
14
14
|
|
|
15
15
|
pytestmark = pytest.mark.usefixtures("client")
|
|
16
16
|
|
|
@@ -29,13 +29,40 @@ async def test_token_sent_on_wire_but_not_leaked(client, settings, caplog):
|
|
|
29
29
|
assert secret not in caplog.text
|
|
30
30
|
|
|
31
31
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
32
|
+
_RAW_FORM_HEADERS = {"Content-Type": "application/vnd.github.raw+json; charset=utf-8"}
|
|
33
|
+
_REF_ANSWER = {"object": {"type": "commit", "sha": "c" * 40}}
|
|
34
|
+
|
|
35
35
|
|
|
36
|
+
async def _read_every_way(client) -> None:
|
|
37
|
+
client.get.return_value = make_response(status=200, text="content", headers=_RAW_FORM_HEADERS)
|
|
36
38
|
await GithubStorage().load("x.j2")
|
|
39
|
+
await GithubStorage().load_bytes("x.bin")
|
|
40
|
+
client.get.return_value = make_response(status=200, json_body={"type": "file", "size": 7})
|
|
41
|
+
await GithubStorage().stat("x.bin")
|
|
42
|
+
client.get.return_value = make_response(status=200, json_body=_REF_ANSWER)
|
|
43
|
+
set_stream(client, make_response(status=206, content=b"con"))
|
|
44
|
+
await GithubStorage().load_range("x.bin", 0, 3)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
async def test_token_rides_every_read_on_both_hosts(client, settings):
|
|
48
|
+
await _read_every_way(client)
|
|
49
|
+
|
|
50
|
+
expected = f"Bearer {settings.token.get_secret_value()}"
|
|
51
|
+
calls = [*client.get.call_args_list, *client.stream.call_args_list]
|
|
52
|
+
assert len(calls) == 5
|
|
53
|
+
for call in calls:
|
|
54
|
+
assert call.kwargs["headers"]["Authorization"] == expected
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
async def test_public_repo_omits_authorization_header_on_both_hosts(client, settings):
|
|
58
|
+
settings.token = None
|
|
59
|
+
|
|
60
|
+
await _read_every_way(client)
|
|
37
61
|
|
|
38
|
-
|
|
62
|
+
calls = [*client.get.call_args_list, *client.stream.call_args_list]
|
|
63
|
+
assert len(calls) == 5
|
|
64
|
+
for call in calls:
|
|
65
|
+
assert "Authorization" not in call.kwargs["headers"]
|
|
39
66
|
|
|
40
67
|
|
|
41
68
|
async def test_settings_secretstr_hides_token_in_repr(monkeypatch):
|
|
@@ -14,14 +14,36 @@ from .conftest import make_response
|
|
|
14
14
|
pytestmark = pytest.mark.usefixtures("client")
|
|
15
15
|
|
|
16
16
|
|
|
17
|
-
|
|
18
|
-
|
|
17
|
+
RAW_FORM = "application/vnd.github.raw+json"
|
|
18
|
+
RAW_FORM_HEADERS = {"Content-Type": f"{RAW_FORM}; charset=utf-8"}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
async def test_load_reads_the_raw_form_from_the_contents_api_at_the_branch(client):
|
|
22
|
+
client.get.return_value = make_response(status=200, text="hello world", headers=RAW_FORM_HEADERS)
|
|
19
23
|
|
|
20
24
|
result = await GithubStorage().load("greetings/hi.j2")
|
|
21
25
|
|
|
22
26
|
assert result == "hello world"
|
|
23
|
-
|
|
24
|
-
|
|
27
|
+
client.get.assert_awaited_once()
|
|
28
|
+
args, kwargs = client.get.call_args
|
|
29
|
+
assert args[0] == "https://api.github.com/repos/acme/content/contents/greetings/hi.j2"
|
|
30
|
+
assert kwargs["params"] == {"ref": "dev"}
|
|
31
|
+
assert kwargs["headers"]["Accept"] == RAW_FORM
|
|
32
|
+
assert kwargs["headers"]["X-GitHub-Api-Version"] == "2022-11-28"
|
|
33
|
+
assert "Range" not in kwargs["headers"]
|
|
34
|
+
client.stream.assert_not_called()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
async def test_load_non_utf8_object_raises_unicode_decode_error(client):
|
|
38
|
+
# httpx's ``Response.text`` would substitute U+FFFD for the invalid bytes; ``load``
|
|
39
|
+
# strictly decodes the raw bytes so a binary object is refused, never garbled.
|
|
40
|
+
png = b"\x89PNG\r\n\x1a\n"
|
|
41
|
+
client.get.return_value = make_response(
|
|
42
|
+
status=200, text=png.decode("utf-8", errors="replace"), content=png, headers=RAW_FORM_HEADERS
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
with pytest.raises(UnicodeDecodeError):
|
|
46
|
+
await GithubStorage().load("images/logo.png")
|
|
25
47
|
|
|
26
48
|
|
|
27
49
|
async def test_load_missing_raises_filenotfound(client):
|
|
@@ -31,6 +53,22 @@ async def test_load_missing_raises_filenotfound(client):
|
|
|
31
53
|
await GithubStorage().load("missing.j2")
|
|
32
54
|
|
|
33
55
|
|
|
56
|
+
async def test_load_of_a_directory_listing_raises_filenotfound(client):
|
|
57
|
+
client.get.return_value = make_response(
|
|
58
|
+
status=200, text="[]", headers={"Content-Type": "application/json; charset=utf-8"}
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
with pytest.raises(FileNotFoundError, match="Object not found: d"):
|
|
62
|
+
await GithubStorage().load("d")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
async def test_load_of_another_content_type_raises_naming_it(client):
|
|
66
|
+
client.get.return_value = make_response(status=200, text="<html>", headers={"Content-Type": "text/html"})
|
|
67
|
+
|
|
68
|
+
with pytest.raises(RuntimeError, match="text/html"):
|
|
69
|
+
await GithubStorage().load("x.j2")
|
|
70
|
+
|
|
71
|
+
|
|
34
72
|
async def test_load_server_error_propagates(client):
|
|
35
73
|
client.get.return_value = make_response(status=500)
|
|
36
74
|
|
|
@@ -55,7 +93,7 @@ async def test_list_returns_blob_paths_via_trees(client):
|
|
|
55
93
|
|
|
56
94
|
assert files == ["a.j2", "d/b.j2"]
|
|
57
95
|
assert client.get.call_args.kwargs["params"] == {"recursive": "1"}
|
|
58
|
-
assert client.get.call_args.args[0].endswith("/git/trees/
|
|
96
|
+
assert client.get.call_args.args[0].endswith("/git/trees/dev")
|
|
59
97
|
|
|
60
98
|
|
|
61
99
|
async def test_list_empty_tree_returns_empty(client):
|
|
@@ -113,7 +151,7 @@ async def test_upload_create_puts_without_sha(client):
|
|
|
113
151
|
payload = client.put.call_args.kwargs["json"]
|
|
114
152
|
assert "sha" not in payload
|
|
115
153
|
assert base64.b64decode(payload["content"]) == b"hello"
|
|
116
|
-
assert payload["branch"] == "
|
|
154
|
+
assert payload["branch"] == "dev"
|
|
117
155
|
|
|
118
156
|
|
|
119
157
|
async def test_upload_update_reuses_existing_sha(client):
|
|
@@ -172,7 +210,7 @@ async def test_delete_resolves_sha_then_deletes(client):
|
|
|
172
210
|
assert args[0] == "DELETE"
|
|
173
211
|
assert args[1].endswith("a/b.j2")
|
|
174
212
|
assert kwargs["json"]["sha"] == "abc123"
|
|
175
|
-
assert kwargs["json"]["branch"] == "
|
|
213
|
+
assert kwargs["json"]["branch"] == "dev"
|
|
176
214
|
|
|
177
215
|
|
|
178
216
|
async def test_delete_missing_raises_filenotfound(client):
|
|
@@ -1,98 +0,0 @@
|
|
|
1
|
-
"""Binary/media surface: load_bytes / upload_bytes / stat, plus the size caps.
|
|
2
|
-
|
|
3
|
-
The two data-integrity invariants live here: reads use the raw endpoint (no size
|
|
4
|
-
cap, no >1 MiB silent-empty Contents-API trap) and uploads are capped loudly.
|
|
5
|
-
"""
|
|
6
|
-
|
|
7
|
-
from __future__ import annotations
|
|
8
|
-
|
|
9
|
-
import base64
|
|
10
|
-
|
|
11
|
-
import pytest
|
|
12
|
-
from httpx import HTTPStatusError
|
|
13
|
-
|
|
14
|
-
from tai42_storage_github import GithubStorage
|
|
15
|
-
from tai42_storage_github.storage import MAX_UPLOAD_BYTES
|
|
16
|
-
|
|
17
|
-
from .conftest import make_response
|
|
18
|
-
|
|
19
|
-
pytestmark = pytest.mark.usefixtures("client")
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
async def test_binary_round_trip_is_byte_identical(client):
|
|
23
|
-
# Every byte value, including bytes that are not valid UTF-8.
|
|
24
|
-
data = bytes(range(256)) * 8
|
|
25
|
-
|
|
26
|
-
client.get.return_value = make_response(status=404) # a create
|
|
27
|
-
client.put.return_value = make_response(status=200)
|
|
28
|
-
await GithubStorage().upload_bytes("media/blob.bin", data)
|
|
29
|
-
|
|
30
|
-
encoded = client.put.call_args.kwargs["json"]["content"]
|
|
31
|
-
assert base64.b64decode(encoded) == data
|
|
32
|
-
|
|
33
|
-
# Read it back through the raw endpoint.
|
|
34
|
-
client.get.return_value = make_response(status=200, content=base64.b64decode(encoded))
|
|
35
|
-
loaded = await GithubStorage().load_bytes("media/blob.bin")
|
|
36
|
-
|
|
37
|
-
assert loaded == data
|
|
38
|
-
assert client.get.call_args.args[0].startswith("https://raw.githubusercontent.com/")
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
async def test_load_bytes_large_file_succeeds_via_raw(client):
|
|
42
|
-
# A 2 MiB file — the Contents API would return content:"" for this; the raw
|
|
43
|
-
# endpoint serves it in full.
|
|
44
|
-
big = b"\x00\xff" * (1024 * 1024) # 2 MiB
|
|
45
|
-
client.get.return_value = make_response(status=200, content=big)
|
|
46
|
-
|
|
47
|
-
loaded = await GithubStorage().load_bytes("big.bin")
|
|
48
|
-
|
|
49
|
-
assert len(loaded) == len(big)
|
|
50
|
-
assert loaded == big
|
|
51
|
-
# Reads never touch the Contents API (that is the >1 MiB silent-empty trap).
|
|
52
|
-
for call in client.get.call_args_list:
|
|
53
|
-
assert "api.github.com" not in call.args[0]
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
async def test_load_bytes_missing_raises_filenotfound(client):
|
|
57
|
-
client.get.return_value = make_response(status=404)
|
|
58
|
-
|
|
59
|
-
with pytest.raises(FileNotFoundError):
|
|
60
|
-
await GithubStorage().load_bytes("missing.bin")
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
async def test_upload_bytes_over_cap_raises_before_request(client):
|
|
64
|
-
oversize = b"\x00" * (MAX_UPLOAD_BYTES + 1)
|
|
65
|
-
|
|
66
|
-
with pytest.raises(ValueError, match="upload cap"):
|
|
67
|
-
await GithubStorage().upload_bytes("big.bin", oversize)
|
|
68
|
-
|
|
69
|
-
# The cap raises BEFORE any network call — no partial/truncated upload.
|
|
70
|
-
client.get.assert_not_called()
|
|
71
|
-
client.put.assert_not_called()
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
async def test_upload_bytes_at_cap_is_allowed(client):
|
|
75
|
-
at_cap = b"\x00" * MAX_UPLOAD_BYTES
|
|
76
|
-
client.get.return_value = make_response(status=404)
|
|
77
|
-
client.put.return_value = make_response(status=200)
|
|
78
|
-
|
|
79
|
-
await GithubStorage().upload_bytes("edge.bin", at_cap)
|
|
80
|
-
|
|
81
|
-
client.put.assert_awaited_once()
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
async def test_upload_bytes_422_backstop_raises(client):
|
|
85
|
-
client.get.return_value = make_response(status=404)
|
|
86
|
-
client.put.return_value = make_response(status=422)
|
|
87
|
-
|
|
88
|
-
with pytest.raises(HTTPStatusError):
|
|
89
|
-
await GithubStorage().upload_bytes("x.bin", b"within-cap")
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
async def test_stat_infers_mime_from_path_suffix():
|
|
93
|
-
assert (await GithubStorage().stat("a/b/logo.png")).content_type == "image/png"
|
|
94
|
-
assert (await GithubStorage().stat("clip.mp3")).content_type == "audio/mpeg"
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
async def test_stat_unknown_suffix_is_none():
|
|
98
|
-
assert (await GithubStorage().stat("mystery.unknownext")).content_type is None
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github/__init__.py
RENAMED
|
File without changes
|
{tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github/client.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|