tai42-storage-github 1.5.0__tar.gz → 2.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. {tai42_storage_github-1.5.0/src/tai42_storage_github.egg-info → tai42_storage_github-2.0.0}/PKG-INFO +25 -12
  2. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/README.md +22 -9
  3. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/pyproject.toml +3 -3
  4. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github/docs/index.mdx +9 -4
  5. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github/settings.py +8 -5
  6. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github/storage.py +133 -29
  7. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github/tai-plugin.yml +2 -2
  8. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0/src/tai42_storage_github.egg-info}/PKG-INFO +25 -12
  9. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github.egg-info/requires.txt +2 -2
  10. tai42_storage_github-2.0.0/tests/test_binary.py +343 -0
  11. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/tests/test_security.py +32 -5
  12. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/tests/test_text.py +45 -7
  13. tai42_storage_github-1.5.0/tests/test_binary.py +0 -98
  14. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/LICENSE +0 -0
  15. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/NOTICE +0 -0
  16. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/setup.cfg +0 -0
  17. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github/__init__.py +0 -0
  18. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github/client.py +0 -0
  19. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github/py.typed +0 -0
  20. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github.egg-info/SOURCES.txt +0 -0
  21. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github.egg-info/dependency_links.txt +0 -0
  22. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/src/tai42_storage_github.egg-info/top_level.txt +0 -0
  23. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/tests/test_delete_dir.py +0 -0
  24. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/tests/test_import_graph.py +0 -0
  25. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/tests/test_plugin_spec.py +0 -0
  26. {tai42_storage_github-1.5.0 → tai42_storage_github-2.0.0}/tests/test_registration.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tai42-storage-github
3
- Version: 1.5.0
3
+ Version: 2.0.0
4
4
  Summary: GitHub-backed Storage provider (text + binary/media) for the TAI ecosystem.
5
5
  Author-email: tai42 <oss@tai42.ai>
6
6
  License-Expression: Apache-2.0
@@ -19,8 +19,8 @@ Requires-Python: >=3.13
19
19
  Description-Content-Type: text/markdown
20
20
  License-File: LICENSE
21
21
  License-File: NOTICE
22
- Requires-Dist: tai42-contract<23,>=22.0
23
- Requires-Dist: tai42-kit<12,>=11.3
22
+ Requires-Dist: tai42-contract<25,>=24.0
23
+ Requires-Dist: tai42-kit<14,>=13.0
24
24
  Requires-Dist: httpx>=0.28
25
25
  Requires-Dist: pydantic>=2.12
26
26
  Requires-Dist: pydantic-settings>=2.15.0
@@ -33,7 +33,8 @@ Dynamic: license-file
33
33
  A GitHub-backed `Storage` provider for the TAI ecosystem. It stores content as
34
34
  files in a GitHub repository and serves the full `tai42_contract.storage.Storage`
35
35
  surface — the text methods (`load` / `list` / `upload` / `delete` / `delete_dir`)
36
- plus the binary/media methods (`load_bytes` / `upload_bytes` / `stat`).
36
+ plus the metadata read `stat`, the bounded read `load_range` and the binary
37
+ methods (`load_bytes` / `upload_bytes`).
37
38
 
38
39
  ## The TAI ecosystem
39
40
 
@@ -99,19 +100,31 @@ a log line, repr, or error message.
99
100
 
100
101
  ## Reads and writes
101
102
 
102
- - **Reads** (`load`, `load_bytes`) go through the raw endpoint
103
- (`raw.githubusercontent.com`), which serves files of any size. The Contents API
104
- is deliberately **not** used for reads: it silently returns an empty body for a
105
- file over 1 MiB, so a large object would read as empty bytes. The raw endpoint
106
- is CDN-cached, so a read shortly after a write may return the previous content
107
- for up to ~5 minutes.
103
+ - **Reads** (`load`, `load_bytes`) are one Contents API call each at the
104
+ configured branch, asking for the raw form
105
+ (`Accept: application/vnd.github.raw+json`), which serves every byte of a file
106
+ of any size. A directory answers its JSON listing and reads as
107
+ `FileNotFoundError`.
108
+ - **Ranged reads** (`load_range`) resolve the branch's head commit through the
109
+ Git refs API, then read `raw.githubusercontent.com` at that commit with a
110
+ `Range` header and `Accept-Encoding: identity` (the raw host compresses, and a
111
+ compressed answer ranges over the encoded bytes): two calls, one of them REST.
112
+ Reading at the commit, never at the branch name, means a ranged read answers the
113
+ branch's current bytes, not a cached copy. An answer that ignores the range or
114
+ is still encoded raises; there is no whole-object read in its place. A
115
+ zero-length range sends no request.
116
+ - **Rate limit**: GitHub allows 60 REST calls an hour without a token and 5000
117
+ with one; `stat`, `load` and `load_bytes` cost one REST call each and
118
+ `load_range` one.
108
119
  - **Writes** (`upload`, `upload_bytes`) go through the Contents API (base64) and
109
120
  are capped conservatively at 1 MiB. A larger payload raises `ValueError` before
110
121
  the request rather than truncating; the API's own `422` is the backstop.
111
122
  - **Listing** uses the recursive Git Trees API (one request) and raises loudly on
112
123
  a `truncated` response rather than acting on a partial listing.
113
- - **`stat`** infers `content_type` from the path suffix — GitHub stores no
114
- per-object content-type.
124
+ - **`stat`** is one Contents API call asking for the object form
125
+ (`Accept: application/vnd.github.object+json`): it answers the file's `size`,
126
+ and infers `content_type` from the path suffix — GitHub stores no per-object
127
+ content-type. A missing path and a directory raise `FileNotFoundError`.
115
128
 
116
129
  ## Development
117
130
 
@@ -5,7 +5,8 @@
5
5
  A GitHub-backed `Storage` provider for the TAI ecosystem. It stores content as
6
6
  files in a GitHub repository and serves the full `tai42_contract.storage.Storage`
7
7
  surface — the text methods (`load` / `list` / `upload` / `delete` / `delete_dir`)
8
- plus the binary/media methods (`load_bytes` / `upload_bytes` / `stat`).
8
+ plus the metadata read `stat`, the bounded read `load_range` and the binary
9
+ methods (`load_bytes` / `upload_bytes`).
9
10
 
10
11
  ## The TAI ecosystem
11
12
 
@@ -71,19 +72,31 @@ a log line, repr, or error message.
71
72
 
72
73
  ## Reads and writes
73
74
 
74
- - **Reads** (`load`, `load_bytes`) go through the raw endpoint
75
- (`raw.githubusercontent.com`), which serves files of any size. The Contents API
76
- is deliberately **not** used for reads: it silently returns an empty body for a
77
- file over 1 MiB, so a large object would read as empty bytes. The raw endpoint
78
- is CDN-cached, so a read shortly after a write may return the previous content
79
- for up to ~5 minutes.
75
+ - **Reads** (`load`, `load_bytes`) are one Contents API call each at the
76
+ configured branch, asking for the raw form
77
+ (`Accept: application/vnd.github.raw+json`), which serves every byte of a file
78
+ of any size. A directory answers its JSON listing and reads as
79
+ `FileNotFoundError`.
80
+ - **Ranged reads** (`load_range`) resolve the branch's head commit through the
81
+ Git refs API, then read `raw.githubusercontent.com` at that commit with a
82
+ `Range` header and `Accept-Encoding: identity` (the raw host compresses, and a
83
+ compressed answer ranges over the encoded bytes): two calls, one of them REST.
84
+ Reading at the commit, never at the branch name, means a ranged read answers the
85
+ branch's current bytes, not a cached copy. An answer that ignores the range or
86
+ is still encoded raises; there is no whole-object read in its place. A
87
+ zero-length range sends no request.
88
+ - **Rate limit**: GitHub allows 60 REST calls an hour without a token and 5000
89
+ with one; `stat`, `load` and `load_bytes` cost one REST call each and
90
+ `load_range` one.
80
91
  - **Writes** (`upload`, `upload_bytes`) go through the Contents API (base64) and
81
92
  are capped conservatively at 1 MiB. A larger payload raises `ValueError` before
82
93
  the request rather than truncating; the API's own `422` is the backstop.
83
94
  - **Listing** uses the recursive Git Trees API (one request) and raises loudly on
84
95
  a `truncated` response rather than acting on a partial listing.
85
- - **`stat`** infers `content_type` from the path suffix — GitHub stores no
86
- per-object content-type.
96
+ - **`stat`** is one Contents API call asking for the object form
97
+ (`Accept: application/vnd.github.object+json`): it answers the file's `size`,
98
+ and infers `content_type` from the path suffix — GitHub stores no per-object
99
+ content-type. A missing path and a directory raise `FileNotFoundError`.
87
100
 
88
101
  ## Development
89
102
 
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "tai42-storage-github"
7
- version = "1.5.0"
7
+ version = "2.0.0"
8
8
  description = "GitHub-backed Storage provider (text + binary/media) for the TAI ecosystem."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.13"
@@ -26,9 +26,9 @@ dependencies = [
26
26
  # The pure interface contract the backend registers through — Storage,
27
27
  # ObjectStat, assert_not_root, and the tai42_app handle.
28
28
  # The contract floor is rewritten by scripts/range_sync.py at release.
29
- "tai42-contract>=22.0,<23",
29
+ "tai42-contract>=24.0,<25",
30
30
  # Pooled clients + settings primitives (PooledClient, TaiBaseSettings, cache).
31
- "tai42-kit>=11.3,<12",
31
+ "tai42-kit>=13.0,<14",
32
32
  "httpx>=0.28",
33
33
  "pydantic>=2.12",
34
34
  "pydantic-settings>=2.15.0",
@@ -1,6 +1,6 @@
1
1
  ---
2
2
  title: "GitHub storage"
3
- description: "Configure the GitHub-repository storage provider, its token scopes, and the size and CDN caveats."
3
+ description: "Configure the GitHub-repository storage provider, its token scopes, the 1 MiB write cap, and the API calls each read costs."
4
4
  ---
5
5
 
6
6
  Stores content as files on a branch of a GitHub repository, which makes every write
@@ -33,11 +33,16 @@ is ignored, as are proxy environment variables.
33
33
 
34
34
  <Warning>
35
35
  Two limits shape what this provider suits. Writes are capped at 1 MiB and a larger
36
- payload raises before the request. Reads come from the raw content CDN, which can
37
- serve a stale copy for a few minutes after a write — do not use GitHub storage for
38
- content a tool writes and immediately reads back.
36
+ payload raises before the request. Reads are GitHub REST API calls, and GitHub
37
+ allows 60 of them an hour without a token and 5000 with one.
39
38
  </Warning>
40
39
 
40
+ A whole read and a stat are one REST call each, made at the configured branch, so
41
+ they answer the branch as it is right after a write. A ranged read resolves the
42
+ branch's head commit (one REST call) and then reads the raw content host at that
43
+ commit with a `Range` header; reading at the commit, never at the branch name,
44
+ means it never serves a cached copy of an older version.
45
+
41
46
  Listing uses the recursive tree API and raises on a truncated response rather than
42
47
  acting on a partial listing. The API host is fixed, so GitHub Enterprise Server is
43
48
  not supported.
@@ -20,12 +20,15 @@ class GithubStorageSettings(TaiBaseSettings):
20
20
  repo: str | None = None
21
21
  branch: str = "main"
22
22
  token: SecretStr | None = None
23
- # Base-URL templates for the three REST surfaces the provider speaks, default to
24
- # the real GitHub hosts. Overridable (like ``S3Settings.endpoint``) so a test can
25
- # point the plugin at a hermetic stand-in; each carries ``{username}``/``{repo}``/
26
- # ``{branch}`` placeholders the provider fills per request.
27
- raw_base_url: str = "https://raw.githubusercontent.com/{username}/{repo}/refs/heads/{branch}"
23
+ # Base-URL templates for the four REST and raw surfaces the provider speaks,
24
+ # defaulting to the real GitHub hosts. Overridable (like ``S3Settings.endpoint``)
25
+ # so a test can point the plugin at a hermetic stand-in. Each carries
26
+ # ``{username}``/``{repo}`` placeholders the provider fills per request; the ref
27
+ # and trees templates also ``{branch}``, and the raw template ``{commit}``, the
28
+ # branch head commit a ranged read resolves first.
29
+ raw_base_url: str = "https://raw.githubusercontent.com/{username}/{repo}/{commit}"
28
30
  contents_api_url: str = "https://api.github.com/repos/{username}/{repo}/contents"
31
+ ref_api_url: str = "https://api.github.com/repos/{username}/{repo}/git/ref/heads/{branch}"
29
32
  trees_api_url: str = "https://api.github.com/repos/{username}/{repo}/git/trees/{branch}"
30
33
  timeout_total: float = 15.0
31
34
  max_connections: int = 200
@@ -1,20 +1,23 @@
1
1
  """GitHub-backed :class:`~tai42_contract.storage.Storage` provider.
2
2
 
3
- Reads use the RAW endpoint (``raw.githubusercontent.com``): the Contents API
4
- returns empty content for files over 1 MiB. The raw endpoint is CDN-cached, so a
5
- read shortly after a write may serve stale content for up to ~5 minutes. Writes
6
- use the Contents API (base64) and are capped below the API limit, raising before
7
- the request rather than truncating.
3
+ ``stat``, ``load`` and ``load_bytes`` are one Contents API call each at the
4
+ configured branch: the object form answers the size, the raw form every byte of
5
+ the file. ``load_range`` resolves the branch's head commit through the Git refs API
6
+ and reads the raw host at that commit with a ``Range`` header, so a ranged read
7
+ answers the branch's current bytes. Writes use the Contents API (base64) and are
8
+ capped below the API limit, raising before the request rather than truncating.
8
9
  """
9
10
 
10
11
  from __future__ import annotations
11
12
 
12
13
  import base64
13
14
  import logging
15
+ import mimetypes
16
+ from typing import Final
14
17
 
15
18
  import httpx
16
19
  from tai42_contract.app import tai42_app
17
- from tai42_contract.storage import Storage, StoragePathConflictError, assert_not_root
20
+ from tai42_contract.storage import ObjectStat, Storage, StoragePathConflictError, assert_not_root
18
21
 
19
22
  from tai42_storage_github.client import GithubHttpxClient
20
23
  from tai42_storage_github.settings import GithubStorageSettings, github_storage_settings
@@ -22,9 +25,15 @@ from tai42_storage_github.settings import GithubStorageSettings, github_storage_
22
25
  logger = logging.getLogger(__name__)
23
26
 
24
27
  # Conservative upload cap below GitHub's Contents API limit; an oversize file
25
- # raises before the request. Reads are uncapped (raw endpoint).
28
+ # raises before the request. Reads are uncapped (the raw form serves every byte).
26
29
  MAX_UPLOAD_BYTES = 1024 * 1024
27
30
 
31
+ # The Contents API answer forms: the object form carries ``type`` and ``size``; the
32
+ # raw form is the file's bytes, and a directory answers its JSON listing instead.
33
+ _OBJECT_FORM: Final = "application/vnd.github.object+json"
34
+ _RAW_FORM: Final = "application/vnd.github.raw+json"
35
+ _JSON: Final = "application/json"
36
+
28
37
 
29
38
  def _join(base: str, path: str) -> str:
30
39
  """Join ``path`` onto ``base``, dropping empty and stray-slash segments.
@@ -41,9 +50,9 @@ def _join(base: str, path: str) -> str:
41
50
  return f"{base}/{joined}" if joined else base
42
51
 
43
52
 
44
- def _raw_url(settings: GithubStorageSettings, path: str) -> str:
45
- """The RAW-endpoint URL for ``path``, from the configured raw base-URL template."""
46
- base = settings.raw_base_url.format(username=settings.username, repo=settings.repo, branch=settings.branch)
53
+ def _raw_url(settings: GithubStorageSettings, commit: str, path: str) -> str:
54
+ """The raw-host URL for ``path`` at ``commit``, from the configured raw base-URL template."""
55
+ base = settings.raw_base_url.format(username=settings.username, repo=settings.repo, commit=commit)
47
56
  return _join(base, path)
48
57
 
49
58
 
@@ -79,14 +88,19 @@ def _auth_headers(settings: GithubStorageSettings) -> dict[str, str]:
79
88
  return {"Authorization": f"Bearer {settings.token.get_secret_value()}"}
80
89
 
81
90
 
82
- def _api_headers(settings: GithubStorageSettings) -> dict[str, str]:
91
+ def _api_headers(settings: GithubStorageSettings, accept: str = "application/vnd.github+json") -> dict[str, str]:
83
92
  return {
84
93
  **_auth_headers(settings),
85
- "Accept": "application/vnd.github+json",
94
+ "Accept": accept,
86
95
  "X-GitHub-Api-Version": "2022-11-28",
87
96
  }
88
97
 
89
98
 
99
+ def _media_type(resp: httpx.Response) -> str:
100
+ """The ``Content-Type`` media type, without its parameters (``; charset=utf-8``)."""
101
+ return resp.headers.get("content-type", "").split(";", 1)[0].strip().lower()
102
+
103
+
90
104
  def _raise_for_status(resp: httpx.Response, action: str, path: str, url: str) -> None:
91
105
  """Surface an HTTP error loudly; the log names the URL/status/body, never the token."""
92
106
  try:
@@ -98,35 +112,125 @@ def _raise_for_status(resp: httpx.Response, action: str, path: str, url: str) ->
98
112
  raise
99
113
 
100
114
 
115
+ def _raise_unless_ok(resp: httpx.Response, expected: int, action: str, path: str, url: str) -> None:
116
+ """Raise on any status other than ``expected``: an HTTP error loudly, an unexpected success as a fault."""
117
+ if resp.status_code == expected:
118
+ return
119
+ _raise_for_status(resp, action, path, url)
120
+ raise RuntimeError(f"GitHub answered {resp.status_code} {action} {path!r} at {url}; expected {expected}.")
121
+
122
+
101
123
  # Importing this module registers GithubStorage as the active storage provider.
102
124
  @tai42_app.storage.register_storage
103
125
  class GithubStorage(Storage):
104
- """GitHub-backed storage: reads via the raw CDN endpoint, writes via the Contents API."""
126
+ """GitHub-backed storage: reads and writes via the Contents API at the branch, ranged reads via the raw host."""
105
127
 
106
128
  async def load(self, path: str) -> str:
107
- """Read ``path`` as text via the raw endpoint; a missing object raises ``FileNotFoundError``."""
129
+ """Read ``path`` as UTF-8 text; a missing object raises ``FileNotFoundError``.
130
+
131
+ The raw bytes are strictly decoded, so an object that is not UTF-8 text raises
132
+ ``UnicodeDecodeError`` rather than reading back with substituted characters.
133
+ """
134
+ return (await self.load_bytes(path)).decode("utf-8")
135
+
136
+ async def load_bytes(self, path: str) -> bytes:
137
+ """Read every byte of ``path`` through the Contents API raw form; a missing object raises ``FileNotFoundError``.
138
+
139
+ A directory answers its JSON listing instead of the raw form, and is not an object.
140
+ """
108
141
  settings = _configured_settings()
109
- url = _raw_url(settings, path)
142
+ url = _contents_url(settings, path)
110
143
  async with tai42_app.clients.client_ctx(GithubHttpxClient) as client:
111
- resp = await client.get(url, headers=_auth_headers(settings))
112
- self._guard_read(resp, path, url)
113
- return resp.text
144
+ resp = await client.get(url, headers=_api_headers(settings, _RAW_FORM), params={"ref": settings.branch})
145
+ if resp.status_code == 404:
146
+ raise FileNotFoundError(f"Object not found: {path}")
147
+ _raise_unless_ok(resp, 200, "loading", path, url)
148
+ media_type = _media_type(resp)
149
+ if media_type == _RAW_FORM:
150
+ return resp.content
151
+ if media_type == _JSON:
152
+ raise FileNotFoundError(f"Object not found: {path}")
153
+ raise RuntimeError(f"GitHub answered an unexpected Content-Type {media_type!r} loading {path!r} at {url}.")
114
154
 
115
- async def load_bytes(self, path: str) -> bytes:
116
- """Read ``path`` as bytes via the raw endpoint (uncapped); a missing object raises ``FileNotFoundError``."""
155
+ async def stat(self, path: str) -> ObjectStat:
156
+ """The byte size from the Contents API object form and the path-inferred content type.
157
+
158
+ GitHub stores no content type. A missing object, and a directory (not an object),
159
+ raise ``FileNotFoundError``; a file answer without an integer size raises ``RuntimeError``.
160
+ """
161
+ settings = _configured_settings()
162
+ url = _contents_url(settings, path)
163
+ async with tai42_app.clients.client_ctx(GithubHttpxClient) as client:
164
+ resp = await client.get(url, headers=_api_headers(settings, _OBJECT_FORM), params={"ref": settings.branch})
165
+ if resp.status_code == 404:
166
+ raise FileNotFoundError(f"Object not found: {path}")
167
+ _raise_unless_ok(resp, 200, "reading the size of", path, url)
168
+ data = resp.json()
169
+ if not isinstance(data, dict) or data.get("type") != "file":
170
+ raise FileNotFoundError(f"Object not found: {path}")
171
+ size = data.get("size")
172
+ if type(size) is not int:
173
+ raise RuntimeError(f"GitHub Contents API answered a file without an integer size at {url}.")
174
+ content_type, _ = mimetypes.guess_type(path)
175
+ return ObjectStat(content_type=content_type, size=size)
176
+
177
+ async def load_range(self, path: str, start: int, length: int) -> bytes:
178
+ """At most ``length`` bytes of ``path`` from ``start``, read at the branch's current head commit.
179
+
180
+ Resolves the head commit, then reads the raw host at that commit with a ``Range``
181
+ header and the identity encoding, since a compressed answer ranges over the encoded
182
+ bytes. ``b""`` for ``length == 0`` (no request) and for a start at or past the end
183
+ (416); ``FileNotFoundError`` for a missing object or a directory (404). An answer
184
+ that ignores the range (200), is still encoded, or runs longer than ``length``
185
+ raises ``RuntimeError``.
186
+ """
187
+ if length == 0:
188
+ return b""
117
189
  settings = _configured_settings()
118
- url = _raw_url(settings, path)
119
190
  async with tai42_app.clients.client_ctx(GithubHttpxClient) as client:
120
- resp = await client.get(url, headers=_auth_headers(settings))
121
- self._guard_read(resp, path, url)
122
- # Raw endpoint has no size cap, unlike the Contents API.
123
- return resp.content
191
+ commit = await self._head_commit(client, settings, path)
192
+ url = _raw_url(settings, commit, path)
193
+ headers = {
194
+ **_auth_headers(settings),
195
+ "Accept-Encoding": "identity",
196
+ "Range": f"bytes={start}-{start + length - 1}",
197
+ }
198
+ async with client.stream("GET", url, headers=headers) as resp:
199
+ if resp.status_code == 404:
200
+ raise FileNotFoundError(f"Object not found: {path}")
201
+ if resp.status_code == 416:
202
+ return b""
203
+ if resp.status_code == 200:
204
+ raise RuntimeError(f"GitHub answered 200, the whole object, to a ranged read of {url}.")
205
+ if resp.status_code != 206:
206
+ await resp.aread()
207
+ _raise_unless_ok(resp, 206, "loading a range of", path, url)
208
+ encoding = resp.headers.get("content-encoding", "identity").strip().lower()
209
+ if encoding != "identity":
210
+ raise RuntimeError(
211
+ f"GitHub answered a {encoding}-encoded range of {url}; the identity encoding was asked."
212
+ )
213
+ chunks: list[bytes] = []
214
+ received = 0
215
+ async for chunk in resp.aiter_bytes():
216
+ received += len(chunk)
217
+ if received > length:
218
+ raise RuntimeError(f"GitHub answered more than the {length} bytes asked of {url}.")
219
+ chunks.append(chunk)
220
+ return b"".join(chunks)
124
221
 
125
222
  @staticmethod
126
- def _guard_read(resp: httpx.Response, path: str, url: str) -> None:
127
- if resp.status_code == 404:
128
- raise FileNotFoundError(f"Object not found: {path}")
129
- _raise_for_status(resp, "loading", path, url)
223
+ async def _head_commit(client: httpx.AsyncClient, settings: GithubStorageSettings, path: str) -> str:
224
+ """The branch's current head commit sha, from the Git refs API."""
225
+ url = settings.ref_api_url.format(username=settings.username, repo=settings.repo, branch=settings.branch)
226
+ resp = await client.get(url, headers=_api_headers(settings))
227
+ _raise_unless_ok(resp, 200, "resolving the branch head for", path, url)
228
+ data = resp.json()
229
+ target = data.get("object") if isinstance(data, dict) else None
230
+ sha = target.get("sha") if isinstance(target, dict) else None
231
+ if not isinstance(sha, str) or not sha:
232
+ raise RuntimeError(f"GitHub answered no head commit sha for the branch at {url}.")
233
+ return sha
130
234
 
131
235
  async def list(self) -> list[str]:
132
236
  """Every blob path in the repo."""
@@ -2,12 +2,12 @@ spec_version: 1
2
2
  namespace: tai42
3
3
  name: storage-github
4
4
  package: tai42-storage-github
5
- version: 1.5.0
5
+ version: 2.0.0
6
6
  description: GitHub-backed Storage provider (text + binary/media) for the TAI ecosystem.
7
7
  license: Apache-2.0
8
8
  homepage: https://tai42.ai
9
9
  repository: https://github.com/tai42ai/tai42/tree/main/plugins/storage-github
10
- contract: '>=22.0,<23'
10
+ contract: '>=24.0,<25'
11
11
  categories:
12
12
  - storage
13
13
  tags:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tai42-storage-github
3
- Version: 1.5.0
3
+ Version: 2.0.0
4
4
  Summary: GitHub-backed Storage provider (text + binary/media) for the TAI ecosystem.
5
5
  Author-email: tai42 <oss@tai42.ai>
6
6
  License-Expression: Apache-2.0
@@ -19,8 +19,8 @@ Requires-Python: >=3.13
19
19
  Description-Content-Type: text/markdown
20
20
  License-File: LICENSE
21
21
  License-File: NOTICE
22
- Requires-Dist: tai42-contract<23,>=22.0
23
- Requires-Dist: tai42-kit<12,>=11.3
22
+ Requires-Dist: tai42-contract<25,>=24.0
23
+ Requires-Dist: tai42-kit<14,>=13.0
24
24
  Requires-Dist: httpx>=0.28
25
25
  Requires-Dist: pydantic>=2.12
26
26
  Requires-Dist: pydantic-settings>=2.15.0
@@ -33,7 +33,8 @@ Dynamic: license-file
33
33
  A GitHub-backed `Storage` provider for the TAI ecosystem. It stores content as
34
34
  files in a GitHub repository and serves the full `tai42_contract.storage.Storage`
35
35
  surface — the text methods (`load` / `list` / `upload` / `delete` / `delete_dir`)
36
- plus the binary/media methods (`load_bytes` / `upload_bytes` / `stat`).
36
+ plus the metadata read `stat`, the bounded read `load_range` and the binary
37
+ methods (`load_bytes` / `upload_bytes`).
37
38
 
38
39
  ## The TAI ecosystem
39
40
 
@@ -99,19 +100,31 @@ a log line, repr, or error message.
99
100
 
100
101
  ## Reads and writes
101
102
 
102
- - **Reads** (`load`, `load_bytes`) go through the raw endpoint
103
- (`raw.githubusercontent.com`), which serves files of any size. The Contents API
104
- is deliberately **not** used for reads: it silently returns an empty body for a
105
- file over 1 MiB, so a large object would read as empty bytes. The raw endpoint
106
- is CDN-cached, so a read shortly after a write may return the previous content
107
- for up to ~5 minutes.
103
+ - **Reads** (`load`, `load_bytes`) are one Contents API call each at the
104
+ configured branch, asking for the raw form
105
+ (`Accept: application/vnd.github.raw+json`), which serves every byte of a file
106
+ of any size. A directory answers its JSON listing and reads as
107
+ `FileNotFoundError`.
108
+ - **Ranged reads** (`load_range`) resolve the branch's head commit through the
109
+ Git refs API, then read `raw.githubusercontent.com` at that commit with a
110
+ `Range` header and `Accept-Encoding: identity` (the raw host compresses, and a
111
+ compressed answer ranges over the encoded bytes): two calls, one of them REST.
112
+ Reading at the commit, never at the branch name, means a ranged read answers the
113
+ branch's current bytes, not a cached copy. An answer that ignores the range or
114
+ is still encoded raises; there is no whole-object read in its place. A
115
+ zero-length range sends no request.
116
+ - **Rate limit**: GitHub allows 60 REST calls an hour without a token and 5000
117
+ with one; `stat`, `load` and `load_bytes` cost one REST call each and
118
+ `load_range` one.
108
119
  - **Writes** (`upload`, `upload_bytes`) go through the Contents API (base64) and
109
120
  are capped conservatively at 1 MiB. A larger payload raises `ValueError` before
110
121
  the request rather than truncating; the API's own `422` is the backstop.
111
122
  - **Listing** uses the recursive Git Trees API (one request) and raises loudly on
112
123
  a `truncated` response rather than acting on a partial listing.
113
- - **`stat`** infers `content_type` from the path suffix — GitHub stores no
114
- per-object content-type.
124
+ - **`stat`** is one Contents API call asking for the object form
125
+ (`Accept: application/vnd.github.object+json`): it answers the file's `size`,
126
+ and infers `content_type` from the path suffix — GitHub stores no per-object
127
+ content-type. A missing path and a directory raise `FileNotFoundError`.
115
128
 
116
129
  ## Development
117
130
 
@@ -1,5 +1,5 @@
1
- tai42-contract<23,>=22.0
2
- tai42-kit<12,>=11.3
1
+ tai42-contract<25,>=24.0
2
+ tai42-kit<14,>=13.0
3
3
  httpx>=0.28
4
4
  pydantic>=2.12
5
5
  pydantic-settings>=2.15.0
@@ -0,0 +1,343 @@
1
+ """Binary/media surface: load_bytes / upload_bytes / stat / load_range, plus the size caps.
2
+
3
+ Whole reads and ``stat`` are one Contents API call at the branch (the raw form serves
4
+ every byte, the object form the size); a ranged read resolves the branch head and
5
+ reads the raw host at that commit; uploads are capped loudly.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import base64
11
+
12
+ import pytest
13
+ from httpx import HTTPStatusError
14
+ from tai42_contract.storage import ObjectStat
15
+
16
+ from tai42_storage_github import GithubStorage
17
+ from tai42_storage_github.storage import MAX_UPLOAD_BYTES
18
+
19
+ from .conftest import make_response, set_stream
20
+
21
+ pytestmark = pytest.mark.usefixtures("client")
22
+
23
+
24
+ CONTENTS = "https://api.github.com/repos/acme/content/contents"
25
+ RAW_FORM = "application/vnd.github.raw+json"
26
+ RAW_FORM_HEADERS = {"Content-Type": f"{RAW_FORM}; charset=utf-8"}
27
+ OBJECT_FORM = "application/vnd.github.object+json"
28
+ HEAD_SHA = "0123456789abcdef0123456789abcdef01234567"
29
+ REF_URL = "https://api.github.com/repos/acme/content/git/ref/heads/dev"
30
+
31
+
32
+ def _ref_answer(sha: object = HEAD_SHA) -> object:
33
+ return make_response(status=200, json_body={"ref": "refs/heads/dev", "object": {"type": "commit", "sha": sha}})
34
+
35
+
36
+ async def test_binary_round_trip_is_byte_identical(client):
37
+ # Every byte value, including bytes that are not valid UTF-8.
38
+ data = bytes(range(256)) * 8
39
+
40
+ client.get.return_value = make_response(status=404) # a create
41
+ client.put.return_value = make_response(status=200)
42
+ await GithubStorage().upload_bytes("media/blob.bin", data)
43
+
44
+ encoded = client.put.call_args.kwargs["json"]["content"]
45
+ assert base64.b64decode(encoded) == data
46
+
47
+ client.get.reset_mock()
48
+ client.get.return_value = make_response(status=200, content=base64.b64decode(encoded), headers=RAW_FORM_HEADERS)
49
+ loaded = await GithubStorage().load_bytes("media/blob.bin")
50
+
51
+ assert loaded == data
52
+ client.get.assert_awaited_once()
53
+ args, kwargs = client.get.call_args
54
+ assert args[0] == f"{CONTENTS}/media/blob.bin"
55
+ assert kwargs["params"] == {"ref": "dev"}
56
+ assert kwargs["headers"]["Accept"] == RAW_FORM
57
+ assert "Range" not in kwargs["headers"]
58
+ client.stream.assert_not_called()
59
+
60
+
61
+ async def test_load_bytes_large_file_is_read_whole(client):
62
+ big = b"\x00\xff" * (1024 * 1024) # 2 MiB
63
+ client.get.return_value = make_response(status=200, content=big, headers=RAW_FORM_HEADERS)
64
+
65
+ loaded = await GithubStorage().load_bytes("big.bin")
66
+
67
+ assert loaded == big
68
+ for call in client.get.call_args_list:
69
+ assert "raw.githubusercontent.com" not in call.args[0]
70
+
71
+
72
+ async def test_load_bytes_missing_raises_filenotfound(client):
73
+ client.get.return_value = make_response(status=404)
74
+
75
+ with pytest.raises(FileNotFoundError):
76
+ await GithubStorage().load_bytes("missing.bin")
77
+
78
+
79
+ async def test_load_bytes_of_a_directory_listing_raises_filenotfound(client):
80
+ client.get.return_value = make_response(
81
+ status=200, content=b"[]", headers={"Content-Type": "application/json; charset=utf-8"}
82
+ )
83
+
84
+ with pytest.raises(FileNotFoundError, match="Object not found: media"):
85
+ await GithubStorage().load_bytes("media")
86
+
87
+
88
+ async def test_load_bytes_unexpected_success_status_raises(client):
89
+ client.get.return_value = make_response(status=204, headers=RAW_FORM_HEADERS)
90
+
91
+ with pytest.raises(RuntimeError, match="204"):
92
+ await GithubStorage().load_bytes("x.bin")
93
+
94
+
95
+ async def test_upload_bytes_over_cap_raises_before_request(client):
96
+ oversize = b"\x00" * (MAX_UPLOAD_BYTES + 1)
97
+
98
+ with pytest.raises(ValueError, match="upload cap"):
99
+ await GithubStorage().upload_bytes("big.bin", oversize)
100
+
101
+ # The cap raises BEFORE any network call — no partial/truncated upload.
102
+ client.get.assert_not_called()
103
+ client.put.assert_not_called()
104
+
105
+
106
+ async def test_upload_bytes_at_cap_is_allowed(client):
107
+ at_cap = b"\x00" * MAX_UPLOAD_BYTES
108
+ client.get.return_value = make_response(status=404)
109
+ client.put.return_value = make_response(status=200)
110
+
111
+ await GithubStorage().upload_bytes("edge.bin", at_cap)
112
+
113
+ client.put.assert_awaited_once()
114
+
115
+
116
+ async def test_upload_bytes_422_backstop_raises(client):
117
+ client.get.return_value = make_response(status=404)
118
+ client.put.return_value = make_response(status=422)
119
+
120
+ with pytest.raises(HTTPStatusError):
121
+ await GithubStorage().upload_bytes("x.bin", b"within-cap")
122
+
123
+
124
+ # --- stat ---------------------------------------------------------------------
125
+
126
+
127
+ async def test_stat_reads_the_object_form_size_at_the_branch(client):
128
+ client.get.return_value = make_response(
129
+ status=200, json_body={"type": "file", "size": 26214400, "path": "a/b/logo.png"}
130
+ )
131
+
132
+ assert await GithubStorage().stat("a/b/logo.png") == ObjectStat(content_type="image/png", size=26214400)
133
+ client.get.assert_awaited_once()
134
+ args, kwargs = client.get.call_args
135
+ assert args[0] == f"{CONTENTS}/a/b/logo.png"
136
+ assert kwargs["params"] == {"ref": "dev"}
137
+ assert kwargs["headers"]["Accept"] == OBJECT_FORM
138
+ assert kwargs["headers"]["X-GitHub-Api-Version"] == "2022-11-28"
139
+ assert "Range" not in kwargs["headers"]
140
+ client.stream.assert_not_called()
141
+
142
+
143
+ async def test_stat_unknown_suffix_is_none(client):
144
+ client.get.return_value = make_response(status=200, json_body={"type": "file", "size": 3})
145
+
146
+ assert await GithubStorage().stat("mystery.unknownext") == ObjectStat(content_type=None, size=3)
147
+
148
+
149
+ async def test_stat_missing_raises_filenotfound(client):
150
+ client.get.return_value = make_response(status=404)
151
+
152
+ with pytest.raises(FileNotFoundError, match=r"Object not found: gone\.png"):
153
+ await GithubStorage().stat("gone.png")
154
+
155
+
156
+ @pytest.mark.parametrize(
157
+ "answer",
158
+ [[{"path": "d/a.j2", "type": "file"}], {"type": "dir", "entries": []}],
159
+ ids=["default-form-list", "object-form-dir"],
160
+ )
161
+ async def test_stat_of_a_directory_raises_filenotfound(client, answer):
162
+ client.get.return_value = make_response(status=200, json_body=answer)
163
+
164
+ with pytest.raises(FileNotFoundError, match="Object not found: d"):
165
+ await GithubStorage().stat("d")
166
+
167
+
168
+ @pytest.mark.parametrize("answer", [{"type": "file"}, {"type": "file", "size": "12"}, {"type": "file", "size": True}])
169
+ async def test_stat_of_a_file_without_an_integer_size_raises(client, answer):
170
+ client.get.return_value = make_response(status=200, json_body=answer)
171
+
172
+ with pytest.raises(RuntimeError, match=r"contents/x\.bin"):
173
+ await GithubStorage().stat("x.bin")
174
+
175
+
176
+ async def test_stat_server_error_propagates(client):
177
+ client.get.return_value = make_response(status=500)
178
+
179
+ with pytest.raises(HTTPStatusError):
180
+ await GithubStorage().stat("x.bin")
181
+
182
+
183
+ # --- load_range -----------------------------------------------------------------
184
+
185
+
186
+ async def test_load_range_resolves_the_head_then_reads_the_raw_host_at_that_commit(client, settings):
187
+ client.get.return_value = _ref_answer()
188
+ resp = make_response(status=206, content=b"\x89PNG", headers={"Content-Range": "bytes 0-3/100"})
189
+ set_stream(client, resp)
190
+
191
+ assert await GithubStorage().load_range("p", 0, 4) == b"\x89PNG"
192
+
193
+ assert [c[0] for c in client.mock_calls if c[0] in ("get", "stream")] == ["get", "stream"]
194
+ ref_args, ref_kwargs = client.get.call_args
195
+ assert ref_args[0] == REF_URL
196
+ assert ref_kwargs["headers"]["Accept"] == "application/vnd.github+json"
197
+ assert ref_kwargs["headers"]["X-GitHub-Api-Version"] == "2022-11-28"
198
+ assert ref_kwargs["headers"]["Authorization"] == f"Bearer {settings.token.get_secret_value()}"
199
+ (method, url), raw_kwargs = client.stream.call_args
200
+ assert method == "GET"
201
+ assert url == f"https://raw.githubusercontent.com/acme/content/{HEAD_SHA}/p"
202
+ assert raw_kwargs["headers"] == {
203
+ "Authorization": f"Bearer {settings.token.get_secret_value()}",
204
+ "Accept-Encoding": "identity",
205
+ "Range": "bytes=0-3",
206
+ }
207
+ for call in [*client.get.call_args_list, *client.stream.call_args_list]:
208
+ assert "refs/heads/dev" not in str(call)
209
+ assert "/dev/" not in str(call)
210
+
211
+
212
+ async def test_load_range_of_the_tail_sends_the_tail_range(client):
213
+ client.get.return_value = _ref_answer()
214
+ set_stream(client, make_response(status=206, content=b"tail"))
215
+
216
+ assert await GithubStorage().load_range("blob.bin", 96, 4) == b"tail"
217
+ assert client.stream.call_args.kwargs["headers"]["Range"] == "bytes=96-99"
218
+
219
+
220
+ async def test_load_range_running_past_the_end_answers_the_tail_only(client):
221
+ # The raw host clamps a range whose end is past the object to the bytes it has.
222
+ client.get.return_value = _ref_answer()
223
+ set_stream(client, make_response(status=206, content=b"ab", headers={"Content-Range": "bytes 98-99/100"}))
224
+
225
+ assert await GithubStorage().load_range("blob.bin", 98, 10) == b"ab"
226
+ assert client.stream.call_args.kwargs["headers"]["Range"] == "bytes=98-107"
227
+
228
+
229
+ async def test_load_range_past_the_end_answers_empty(client):
230
+ client.get.return_value = _ref_answer()
231
+ set_stream(client, make_response(status=416, headers={"Content-Range": "bytes */100"}))
232
+
233
+ assert await GithubStorage().load_range("blob.bin", 100, 4) == b""
234
+
235
+
236
+ async def test_load_range_missing_or_directory_raises_filenotfound(client):
237
+ client.get.return_value = _ref_answer()
238
+ set_stream(client, make_response(status=404))
239
+
240
+ with pytest.raises(FileNotFoundError, match="Object not found: jq"):
241
+ await GithubStorage().load_range("jq", 0, 4)
242
+
243
+
244
+ async def test_load_range_whole_object_answer_raises_without_reading_the_body(client):
245
+ client.get.return_value = _ref_answer()
246
+ resp = make_response(status=200, content=b"x" * 1000)
247
+ set_stream(client, resp)
248
+
249
+ with pytest.raises(RuntimeError, match="200") as exc:
250
+ await GithubStorage().load_range("blob.bin", 0, 4)
251
+
252
+ assert f"raw.githubusercontent.com/acme/content/{HEAD_SHA}/blob.bin" in str(exc.value)
253
+ assert resp.body_read is False
254
+
255
+
256
+ async def test_load_range_encoded_answer_raises_without_reading_the_body(client):
257
+ # A compressed answer ranges over the encoded bytes, not the object's.
258
+ client.get.return_value = _ref_answer()
259
+ resp = make_response(
260
+ status=206,
261
+ content=b"\x1f\x8b\x08\x00\x00",
262
+ headers={"Content-Range": "bytes 0-4/47", "Content-Encoding": "gzip"},
263
+ )
264
+ set_stream(client, resp)
265
+
266
+ with pytest.raises(RuntimeError, match="gzip") as exc:
267
+ await GithubStorage().load_range("blob.bin", 0, 5)
268
+
269
+ assert f"raw.githubusercontent.com/acme/content/{HEAD_SHA}/blob.bin" in str(exc.value)
270
+ assert resp.body_read is False
271
+
272
+
273
+ async def test_load_range_longer_answer_than_asked_raises(client):
274
+ client.get.return_value = _ref_answer()
275
+ set_stream(client, make_response(status=206, chunks=[b"abc", b"def"]))
276
+
277
+ with pytest.raises(RuntimeError, match="more than the 4 bytes"):
278
+ await GithubStorage().load_range("blob.bin", 0, 4)
279
+
280
+
281
+ async def test_load_range_server_error_propagates(client):
282
+ client.get.return_value = _ref_answer()
283
+ set_stream(client, make_response(status=500, text="boom"))
284
+
285
+ with pytest.raises(HTTPStatusError):
286
+ await GithubStorage().load_range("blob.bin", 0, 4)
287
+
288
+
289
+ async def test_load_range_unexpected_success_status_raises(client):
290
+ client.get.return_value = _ref_answer()
291
+ set_stream(client, make_response(status=204))
292
+
293
+ with pytest.raises(RuntimeError, match="204"):
294
+ await GithubStorage().load_range("blob.bin", 0, 4)
295
+
296
+
297
+ async def test_load_range_failed_head_lookup_raises_before_the_read(client):
298
+ client.get.return_value = make_response(status=404)
299
+
300
+ with pytest.raises(HTTPStatusError):
301
+ await GithubStorage().load_range("blob.bin", 0, 4)
302
+
303
+ client.stream.assert_not_called()
304
+
305
+
306
+ @pytest.mark.parametrize(
307
+ "body",
308
+ [
309
+ {"ref": "refs/heads/dev"},
310
+ {"object": {"type": "commit"}},
311
+ {"object": {"sha": ""}},
312
+ {"object": {"sha": 7}},
313
+ {"object": "abc"},
314
+ [{"ref": "refs/heads/dev-a"}],
315
+ ],
316
+ ids=["no-object", "no-sha", "empty-sha", "int-sha", "object-not-a-dict", "list"],
317
+ )
318
+ async def test_load_range_head_answer_without_a_commit_sha_raises(client, body):
319
+ client.get.return_value = make_response(status=200, json_body=body)
320
+
321
+ with pytest.raises(RuntimeError, match="git/ref/heads/dev"):
322
+ await GithubStorage().load_range("blob.bin", 0, 4)
323
+
324
+ client.stream.assert_not_called()
325
+
326
+
327
+ async def test_load_range_zero_length_sends_no_request(client):
328
+ assert await GithubStorage().load_range("blob.bin", 0, 0) == b""
329
+
330
+ client.get.assert_not_called()
331
+ client.stream.assert_not_called()
332
+
333
+
334
+ async def test_load_range_resolves_the_head_on_every_call(client):
335
+ shas = ["a" * 40, "b" * 40]
336
+ client.get.side_effect = [_ref_answer(shas[0]), _ref_answer(shas[1])]
337
+ set_stream(client, make_response(status=206, content=b"x"))
338
+
339
+ await GithubStorage().load_range("p", 0, 1)
340
+ await GithubStorage().load_range("p", 0, 1)
341
+
342
+ urls = [c.args[1] for c in client.stream.call_args_list]
343
+ assert urls == [f"https://raw.githubusercontent.com/acme/content/{sha}/p" for sha in shas]
@@ -10,7 +10,7 @@ from httpx import HTTPStatusError
10
10
 
11
11
  from tai42_storage_github import GithubStorage
12
12
 
13
- from .conftest import make_response
13
+ from .conftest import make_response, set_stream
14
14
 
15
15
  pytestmark = pytest.mark.usefixtures("client")
16
16
 
@@ -29,13 +29,40 @@ async def test_token_sent_on_wire_but_not_leaked(client, settings, caplog):
29
29
  assert secret not in caplog.text
30
30
 
31
31
 
32
- async def test_public_repo_omits_authorization_header(client, settings):
33
- settings.token = None
34
- client.get.return_value = make_response(status=200, text="public content")
32
+ _RAW_FORM_HEADERS = {"Content-Type": "application/vnd.github.raw+json; charset=utf-8"}
33
+ _REF_ANSWER = {"object": {"type": "commit", "sha": "c" * 40}}
34
+
35
35
 
36
+ async def _read_every_way(client) -> None:
37
+ client.get.return_value = make_response(status=200, text="content", headers=_RAW_FORM_HEADERS)
36
38
  await GithubStorage().load("x.j2")
39
+ await GithubStorage().load_bytes("x.bin")
40
+ client.get.return_value = make_response(status=200, json_body={"type": "file", "size": 7})
41
+ await GithubStorage().stat("x.bin")
42
+ client.get.return_value = make_response(status=200, json_body=_REF_ANSWER)
43
+ set_stream(client, make_response(status=206, content=b"con"))
44
+ await GithubStorage().load_range("x.bin", 0, 3)
45
+
46
+
47
+ async def test_token_rides_every_read_on_both_hosts(client, settings):
48
+ await _read_every_way(client)
49
+
50
+ expected = f"Bearer {settings.token.get_secret_value()}"
51
+ calls = [*client.get.call_args_list, *client.stream.call_args_list]
52
+ assert len(calls) == 5
53
+ for call in calls:
54
+ assert call.kwargs["headers"]["Authorization"] == expected
55
+
56
+
57
+ async def test_public_repo_omits_authorization_header_on_both_hosts(client, settings):
58
+ settings.token = None
59
+
60
+ await _read_every_way(client)
37
61
 
38
- assert "Authorization" not in client.get.call_args.kwargs["headers"]
62
+ calls = [*client.get.call_args_list, *client.stream.call_args_list]
63
+ assert len(calls) == 5
64
+ for call in calls:
65
+ assert "Authorization" not in call.kwargs["headers"]
39
66
 
40
67
 
41
68
  async def test_settings_secretstr_hides_token_in_repr(monkeypatch):
@@ -14,14 +14,36 @@ from .conftest import make_response
14
14
  pytestmark = pytest.mark.usefixtures("client")
15
15
 
16
16
 
17
- async def test_load_returns_text_from_raw_endpoint(client):
18
- client.get.return_value = make_response(status=200, text="hello world")
17
+ RAW_FORM = "application/vnd.github.raw+json"
18
+ RAW_FORM_HEADERS = {"Content-Type": f"{RAW_FORM}; charset=utf-8"}
19
+
20
+
21
+ async def test_load_reads_the_raw_form_from_the_contents_api_at_the_branch(client):
22
+ client.get.return_value = make_response(status=200, text="hello world", headers=RAW_FORM_HEADERS)
19
23
 
20
24
  result = await GithubStorage().load("greetings/hi.j2")
21
25
 
22
26
  assert result == "hello world"
23
- url = client.get.call_args.args[0]
24
- assert url == "https://raw.githubusercontent.com/acme/content/refs/heads/main/greetings/hi.j2"
27
+ client.get.assert_awaited_once()
28
+ args, kwargs = client.get.call_args
29
+ assert args[0] == "https://api.github.com/repos/acme/content/contents/greetings/hi.j2"
30
+ assert kwargs["params"] == {"ref": "dev"}
31
+ assert kwargs["headers"]["Accept"] == RAW_FORM
32
+ assert kwargs["headers"]["X-GitHub-Api-Version"] == "2022-11-28"
33
+ assert "Range" not in kwargs["headers"]
34
+ client.stream.assert_not_called()
35
+
36
+
37
+ async def test_load_non_utf8_object_raises_unicode_decode_error(client):
38
+ # httpx's ``Response.text`` would substitute U+FFFD for the invalid bytes; ``load``
39
+ # strictly decodes the raw bytes so a binary object is refused, never garbled.
40
+ png = b"\x89PNG\r\n\x1a\n"
41
+ client.get.return_value = make_response(
42
+ status=200, text=png.decode("utf-8", errors="replace"), content=png, headers=RAW_FORM_HEADERS
43
+ )
44
+
45
+ with pytest.raises(UnicodeDecodeError):
46
+ await GithubStorage().load("images/logo.png")
25
47
 
26
48
 
27
49
  async def test_load_missing_raises_filenotfound(client):
@@ -31,6 +53,22 @@ async def test_load_missing_raises_filenotfound(client):
31
53
  await GithubStorage().load("missing.j2")
32
54
 
33
55
 
56
+ async def test_load_of_a_directory_listing_raises_filenotfound(client):
57
+ client.get.return_value = make_response(
58
+ status=200, text="[]", headers={"Content-Type": "application/json; charset=utf-8"}
59
+ )
60
+
61
+ with pytest.raises(FileNotFoundError, match="Object not found: d"):
62
+ await GithubStorage().load("d")
63
+
64
+
65
+ async def test_load_of_another_content_type_raises_naming_it(client):
66
+ client.get.return_value = make_response(status=200, text="<html>", headers={"Content-Type": "text/html"})
67
+
68
+ with pytest.raises(RuntimeError, match="text/html"):
69
+ await GithubStorage().load("x.j2")
70
+
71
+
34
72
  async def test_load_server_error_propagates(client):
35
73
  client.get.return_value = make_response(status=500)
36
74
 
@@ -55,7 +93,7 @@ async def test_list_returns_blob_paths_via_trees(client):
55
93
 
56
94
  assert files == ["a.j2", "d/b.j2"]
57
95
  assert client.get.call_args.kwargs["params"] == {"recursive": "1"}
58
- assert client.get.call_args.args[0].endswith("/git/trees/main")
96
+ assert client.get.call_args.args[0].endswith("/git/trees/dev")
59
97
 
60
98
 
61
99
  async def test_list_empty_tree_returns_empty(client):
@@ -113,7 +151,7 @@ async def test_upload_create_puts_without_sha(client):
113
151
  payload = client.put.call_args.kwargs["json"]
114
152
  assert "sha" not in payload
115
153
  assert base64.b64decode(payload["content"]) == b"hello"
116
- assert payload["branch"] == "main"
154
+ assert payload["branch"] == "dev"
117
155
 
118
156
 
119
157
  async def test_upload_update_reuses_existing_sha(client):
@@ -172,7 +210,7 @@ async def test_delete_resolves_sha_then_deletes(client):
172
210
  assert args[0] == "DELETE"
173
211
  assert args[1].endswith("a/b.j2")
174
212
  assert kwargs["json"]["sha"] == "abc123"
175
- assert kwargs["json"]["branch"] == "main"
213
+ assert kwargs["json"]["branch"] == "dev"
176
214
 
177
215
 
178
216
  async def test_delete_missing_raises_filenotfound(client):
@@ -1,98 +0,0 @@
1
- """Binary/media surface: load_bytes / upload_bytes / stat, plus the size caps.
2
-
3
- The two data-integrity invariants live here: reads use the raw endpoint (no size
4
- cap, no >1 MiB silent-empty Contents-API trap) and uploads are capped loudly.
5
- """
6
-
7
- from __future__ import annotations
8
-
9
- import base64
10
-
11
- import pytest
12
- from httpx import HTTPStatusError
13
-
14
- from tai42_storage_github import GithubStorage
15
- from tai42_storage_github.storage import MAX_UPLOAD_BYTES
16
-
17
- from .conftest import make_response
18
-
19
- pytestmark = pytest.mark.usefixtures("client")
20
-
21
-
22
- async def test_binary_round_trip_is_byte_identical(client):
23
- # Every byte value, including bytes that are not valid UTF-8.
24
- data = bytes(range(256)) * 8
25
-
26
- client.get.return_value = make_response(status=404) # a create
27
- client.put.return_value = make_response(status=200)
28
- await GithubStorage().upload_bytes("media/blob.bin", data)
29
-
30
- encoded = client.put.call_args.kwargs["json"]["content"]
31
- assert base64.b64decode(encoded) == data
32
-
33
- # Read it back through the raw endpoint.
34
- client.get.return_value = make_response(status=200, content=base64.b64decode(encoded))
35
- loaded = await GithubStorage().load_bytes("media/blob.bin")
36
-
37
- assert loaded == data
38
- assert client.get.call_args.args[0].startswith("https://raw.githubusercontent.com/")
39
-
40
-
41
- async def test_load_bytes_large_file_succeeds_via_raw(client):
42
- # A 2 MiB file — the Contents API would return content:"" for this; the raw
43
- # endpoint serves it in full.
44
- big = b"\x00\xff" * (1024 * 1024) # 2 MiB
45
- client.get.return_value = make_response(status=200, content=big)
46
-
47
- loaded = await GithubStorage().load_bytes("big.bin")
48
-
49
- assert len(loaded) == len(big)
50
- assert loaded == big
51
- # Reads never touch the Contents API (that is the >1 MiB silent-empty trap).
52
- for call in client.get.call_args_list:
53
- assert "api.github.com" not in call.args[0]
54
-
55
-
56
- async def test_load_bytes_missing_raises_filenotfound(client):
57
- client.get.return_value = make_response(status=404)
58
-
59
- with pytest.raises(FileNotFoundError):
60
- await GithubStorage().load_bytes("missing.bin")
61
-
62
-
63
- async def test_upload_bytes_over_cap_raises_before_request(client):
64
- oversize = b"\x00" * (MAX_UPLOAD_BYTES + 1)
65
-
66
- with pytest.raises(ValueError, match="upload cap"):
67
- await GithubStorage().upload_bytes("big.bin", oversize)
68
-
69
- # The cap raises BEFORE any network call — no partial/truncated upload.
70
- client.get.assert_not_called()
71
- client.put.assert_not_called()
72
-
73
-
74
- async def test_upload_bytes_at_cap_is_allowed(client):
75
- at_cap = b"\x00" * MAX_UPLOAD_BYTES
76
- client.get.return_value = make_response(status=404)
77
- client.put.return_value = make_response(status=200)
78
-
79
- await GithubStorage().upload_bytes("edge.bin", at_cap)
80
-
81
- client.put.assert_awaited_once()
82
-
83
-
84
- async def test_upload_bytes_422_backstop_raises(client):
85
- client.get.return_value = make_response(status=404)
86
- client.put.return_value = make_response(status=422)
87
-
88
- with pytest.raises(HTTPStatusError):
89
- await GithubStorage().upload_bytes("x.bin", b"within-cap")
90
-
91
-
92
- async def test_stat_infers_mime_from_path_suffix():
93
- assert (await GithubStorage().stat("a/b/logo.png")).content_type == "image/png"
94
- assert (await GithubStorage().stat("clip.mp3")).content_type == "audio/mpeg"
95
-
96
-
97
- async def test_stat_unknown_suffix_is_none():
98
- assert (await GithubStorage().stat("mystery.unknownext")).content_type is None