tai42-storage-github 2.0.1__tar.gz → 3.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. {tai42_storage_github-2.0.1/src/tai42_storage_github.egg-info → tai42_storage_github-3.0.0}/PKG-INFO +12 -7
  2. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/README.md +9 -4
  3. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/pyproject.toml +3 -3
  4. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/src/tai42_storage_github/settings.py +3 -2
  5. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/src/tai42_storage_github/storage.py +103 -41
  6. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/src/tai42_storage_github/tai-plugin.yml +2 -2
  7. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0/src/tai42_storage_github.egg-info}/PKG-INFO +12 -7
  8. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/src/tai42_storage_github.egg-info/requires.txt +2 -2
  9. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/tests/test_binary.py +19 -2
  10. tai42_storage_github-3.0.0/tests/test_delete_dir.py +110 -0
  11. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/tests/test_text.py +69 -10
  12. tai42_storage_github-2.0.1/tests/test_delete_dir.py +0 -80
  13. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/LICENSE +0 -0
  14. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/NOTICE +0 -0
  15. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/setup.cfg +0 -0
  16. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/src/tai42_storage_github/__init__.py +0 -0
  17. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/src/tai42_storage_github/client.py +0 -0
  18. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/src/tai42_storage_github/docs/index.mdx +0 -0
  19. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/src/tai42_storage_github/py.typed +0 -0
  20. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/src/tai42_storage_github.egg-info/SOURCES.txt +0 -0
  21. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/src/tai42_storage_github.egg-info/dependency_links.txt +0 -0
  22. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/src/tai42_storage_github.egg-info/top_level.txt +0 -0
  23. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/tests/test_import_graph.py +0 -0
  24. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/tests/test_plugin_spec.py +0 -0
  25. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/tests/test_registration.py +0 -0
  26. {tai42_storage_github-2.0.1 → tai42_storage_github-3.0.0}/tests/test_security.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tai42-storage-github
3
- Version: 2.0.1
3
+ Version: 3.0.0
4
4
  Summary: GitHub-backed Storage provider (text + binary/media) for the TAI ecosystem.
5
5
  Author-email: tai42 <oss@tai42.ai>
6
6
  License-Expression: Apache-2.0
@@ -19,8 +19,8 @@ Requires-Python: >=3.13
19
19
  Description-Content-Type: text/markdown
20
20
  License-File: LICENSE
21
21
  License-File: NOTICE
22
- Requires-Dist: tai42-contract<26,>=25.0
23
- Requires-Dist: tai42-kit<14,>=13.0
22
+ Requires-Dist: tai42-contract<27,>=26.0
23
+ Requires-Dist: tai42-kit<15,>=14.0
24
24
  Requires-Dist: httpx>=0.28
25
25
  Requires-Dist: pydantic>=2.12
26
26
  Requires-Dist: pydantic-settings>=2.15.0
@@ -34,7 +34,7 @@ A GitHub-backed `Storage` provider for the TAI ecosystem. It stores content as
34
34
  files in a GitHub repository and serves the full `tai42_contract.storage.Storage`
35
35
  surface — the text methods (`load` / `list` / `upload` / `delete` / `delete_dir`)
36
36
  plus the metadata read `stat`, the bounded read `load_range` and the binary
37
- methods (`load_bytes` / `upload_bytes`).
37
+ methods (`load_bytes` / `upload_bytes` / `load_object`).
38
38
 
39
39
  ## The TAI ecosystem
40
40
 
@@ -100,11 +100,12 @@ a log line, repr, or error message.
100
100
 
101
101
  ## Reads and writes
102
102
 
103
- - **Reads** (`load`, `load_bytes`) are one Contents API call each at the
103
+ - **Reads** (`load`, `load_bytes`, `load_object`) are one Contents API call each at the
104
104
  configured branch, asking for the raw form
105
105
  (`Accept: application/vnd.github.raw+json`), which serves every byte of a file
106
106
  of any size. A directory answers its JSON listing and reads as
107
- `FileNotFoundError`.
107
+ `FileNotFoundError`. `load_object` infers the content type from the path suffix,
108
+ as `stat` does.
108
109
  - **Ranged reads** (`load_range`) resolve the branch's head commit through the
109
110
  Git refs API, then read `raw.githubusercontent.com` at that commit with a
110
111
  `Range` header and `Accept-Encoding: identity` (the raw host compresses, and a
@@ -120,7 +121,11 @@ a log line, repr, or error message.
120
121
  are capped conservatively at 1 MiB. A larger payload raises `ValueError` before
121
122
  the request rather than truncating; the API's own `422` is the backstop.
122
123
  - **Listing** uses the recursive Git Trees API (one request) and raises loudly on
123
- a `truncated` response rather than acting on a partial listing.
124
+ a `truncated` response rather than acting on a partial listing. `list(directory)`
125
+ reads the tree of that directory alone (`<branch>:<directory>`), so it grows with
126
+ the directory, not the repository; a directory that does not exist answers `[]`
127
+ once the branch is confirmed to exist. `delete_dir` lists the directory the same
128
+ way.
124
129
  - **`stat`** is one Contents API call asking for the object form
125
130
  (`Accept: application/vnd.github.object+json`): it answers the file's `size`,
126
131
  and infers `content_type` from the path suffix — GitHub stores no per-object
@@ -6,7 +6,7 @@ A GitHub-backed `Storage` provider for the TAI ecosystem. It stores content as
6
6
  files in a GitHub repository and serves the full `tai42_contract.storage.Storage`
7
7
  surface — the text methods (`load` / `list` / `upload` / `delete` / `delete_dir`)
8
8
  plus the metadata read `stat`, the bounded read `load_range` and the binary
9
- methods (`load_bytes` / `upload_bytes`).
9
+ methods (`load_bytes` / `upload_bytes` / `load_object`).
10
10
 
11
11
  ## The TAI ecosystem
12
12
 
@@ -72,11 +72,12 @@ a log line, repr, or error message.
72
72
 
73
73
  ## Reads and writes
74
74
 
75
- - **Reads** (`load`, `load_bytes`) are one Contents API call each at the
75
+ - **Reads** (`load`, `load_bytes`, `load_object`) are one Contents API call each at the
76
76
  configured branch, asking for the raw form
77
77
  (`Accept: application/vnd.github.raw+json`), which serves every byte of a file
78
78
  of any size. A directory answers its JSON listing and reads as
79
- `FileNotFoundError`.
79
+ `FileNotFoundError`. `load_object` infers the content type from the path suffix,
80
+ as `stat` does.
80
81
  - **Ranged reads** (`load_range`) resolve the branch's head commit through the
81
82
  Git refs API, then read `raw.githubusercontent.com` at that commit with a
82
83
  `Range` header and `Accept-Encoding: identity` (the raw host compresses, and a
@@ -92,7 +93,11 @@ a log line, repr, or error message.
92
93
  are capped conservatively at 1 MiB. A larger payload raises `ValueError` before
93
94
  the request rather than truncating; the API's own `422` is the backstop.
94
95
  - **Listing** uses the recursive Git Trees API (one request) and raises loudly on
95
- a `truncated` response rather than acting on a partial listing.
96
+ a `truncated` response rather than acting on a partial listing. `list(directory)`
97
+ reads the tree of that directory alone (`<branch>:<directory>`), so it grows with
98
+ the directory, not the repository; a directory that does not exist answers `[]`
99
+ once the branch is confirmed to exist. `delete_dir` lists the directory the same
100
+ way.
96
101
  - **`stat`** is one Contents API call asking for the object form
97
102
  (`Accept: application/vnd.github.object+json`): it answers the file's `size`,
98
103
  and infers `content_type` from the path suffix — GitHub stores no per-object
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "tai42-storage-github"
7
- version = "2.0.1"
7
+ version = "3.0.0"
8
8
  description = "GitHub-backed Storage provider (text + binary/media) for the TAI ecosystem."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.13"
@@ -26,9 +26,9 @@ dependencies = [
26
26
  # The pure interface contract the backend registers through — Storage,
27
27
  # ObjectStat, assert_not_root, and the tai42_app handle.
28
28
  # The contract floor is rewritten by scripts/range_sync.py at release.
29
- "tai42-contract>=25.0,<26",
29
+ "tai42-contract>=26.0,<27",
30
30
  # Pooled clients + settings primitives (PooledClient, TaiBaseSettings, cache).
31
- "tai42-kit>=13.0,<14",
31
+ "tai42-kit>=14.0,<15",
32
32
  "httpx>=0.28",
33
33
  "pydantic>=2.12",
34
34
  "pydantic-settings>=2.15.0",
@@ -24,8 +24,9 @@ class GithubStorageSettings(TaiBaseSettings):
24
24
  # defaulting to the real GitHub hosts. Overridable (like ``S3Settings.endpoint``)
25
25
  # so a test can point the plugin at a hermetic stand-in. Each carries
26
26
  # ``{username}``/``{repo}`` placeholders the provider fills per request; the ref
27
- # and trees templates also ``{branch}``, and the raw template ``{commit}``, the
28
- # branch head commit a ranged read resolves first.
27
+ # and trees templates also ``{branch}`` (the trees template's is the tree read: the
28
+ # branch, or ``branch:directory`` for a directory listing), and the raw template
29
+ # ``{commit}``, the branch head commit a ranged read resolves first.
29
30
  raw_base_url: str = "https://raw.githubusercontent.com/{username}/{repo}/{commit}"
30
31
  contents_api_url: str = "https://api.github.com/repos/{username}/{repo}/contents"
31
32
  ref_api_url: str = "https://api.github.com/repos/{username}/{repo}/git/ref/heads/{branch}"
@@ -1,11 +1,13 @@
1
1
  """GitHub-backed :class:`~tai42_contract.storage.Storage` provider.
2
2
 
3
- ``stat``, ``load`` and ``load_bytes`` are one Contents API call each at the
4
- configured branch: the object form answers the size, the raw form every byte of
5
- the file. ``load_range`` resolves the branch's head commit through the Git refs API
6
- and reads the raw host at that commit with a ``Range`` header, so a ranged read
7
- answers the branch's current bytes. Writes use the Contents API (base64) and are
8
- capped below the API limit, raising before the request rather than truncating.
3
+ ``stat``, ``load``, ``load_bytes`` and ``load_object`` are one Contents API call each
4
+ at the configured branch: the object form answers the size, the raw form every byte
5
+ of the file (GitHub stores no content type, so it is inferred from the path).
6
+ ``list`` of a directory reads only that directory's tree. ``load_range`` resolves the
7
+ branch's head commit through the Git refs API and reads the raw host at that
8
+ commit with a ``Range`` header, so a ranged read answers the branch's current
9
+ bytes. Writes use the Contents API (base64) and are capped below the API limit,
10
+ raising before the request rather than truncating.
9
11
  """
10
12
 
11
13
  from __future__ import annotations
@@ -13,11 +15,19 @@ from __future__ import annotations
13
15
  import base64
14
16
  import logging
15
17
  import mimetypes
16
- from typing import Final
18
+ from typing import Final, NamedTuple
19
+ from urllib.parse import quote
17
20
 
18
21
  import httpx
19
22
  from tai42_contract.app import tai42_app
20
- from tai42_contract.storage import ObjectStat, Storage, StoragePathConflictError, assert_not_root
23
+ from tai42_contract.storage import (
24
+ ObjectStat,
25
+ Storage,
26
+ StoragePathConflictError,
27
+ StoredObject,
28
+ assert_not_root,
29
+ directory_prefix,
30
+ )
21
31
 
22
32
  from tai42_storage_github.client import GithubHttpxClient
23
33
  from tai42_storage_github.settings import GithubStorageSettings, github_storage_settings
@@ -35,17 +45,29 @@ _RAW_FORM: Final = "application/vnd.github.raw+json"
35
45
  _JSON: Final = "application/json"
36
46
 
37
47
 
38
- def _join(base: str, path: str) -> str:
39
- """Join ``path`` onto ``base``, dropping empty and stray-slash segments.
48
+ class _Blob(NamedTuple):
49
+ """A file in the branch's tree: its path and the blob sha a write or delete names."""
50
+
51
+ path: str
52
+ sha: str
53
+
54
+
55
+ def _path_segments(path: str) -> list[str]:
56
+ """The segments of storage ``path``, dropping empty and stray-slash segments.
40
57
 
41
58
  A ``..`` segment is rejected loudly: preserved, it would climb out of the
42
59
  configured repo/branch base and let a caller-supplied path reach an arbitrary
43
- raw/contents URL (path traversal).
60
+ raw/contents/tree URL (path traversal).
44
61
  """
45
62
  segments = [segment for segment in path.split("/") if segment]
46
63
  if ".." in segments:
47
64
  raise ValueError(f"path traversal ('..') is not allowed in storage path: {path!r}")
48
- joined = "/".join(segments)
65
+ return segments
66
+
67
+
68
+ def _join(base: str, path: str) -> str:
69
+ """Join the :func:`_path_segments` of ``path`` onto ``base``."""
70
+ joined = "/".join(_path_segments(path))
49
71
  base = base.rstrip("/")
50
72
  return f"{base}/{joined}" if joined else base
51
73
 
@@ -152,6 +174,12 @@ class GithubStorage(Storage):
152
174
  raise FileNotFoundError(f"Object not found: {path}")
153
175
  raise RuntimeError(f"GitHub answered an unexpected Content-Type {media_type!r} loading {path!r} at {url}.")
154
176
 
177
+ async def load_object(self, path: str) -> StoredObject:
178
+ """Every byte of ``path`` (as :meth:`load_bytes` reads them) with the content type :meth:`stat` infers."""
179
+ data = await self.load_bytes(path)
180
+ content_type, _ = mimetypes.guess_type(path)
181
+ return StoredObject(data=data, content_type=content_type)
182
+
155
183
  async def stat(self, path: str) -> ObjectStat:
156
184
  """The byte size from the Contents API object form and the path-inferred content type.
157
185
 
@@ -232,33 +260,55 @@ class GithubStorage(Storage):
232
260
  raise RuntimeError(f"GitHub answered no head commit sha for the branch at {url}.")
233
261
  return sha
234
262
 
235
- async def list(self) -> list[str]:
236
- """Every blob path in the repo."""
237
- return await self._list_blobs("")
263
+ async def list(self, directory: str | None = None) -> list[str]:
264
+ """Every blob path in the repo, or only those beneath ``directory``, read from that directory's tree.
238
265
 
239
- async def _list_blobs(self, prefix: str) -> list[str]:
240
- """Every blob path in the repo, optionally filtered to ``prefix``.
266
+ A directory is listed through the Git Trees API at ``branch:directory``, so only
267
+ its subtree is read. A directory that does not exist on the branch, or names a
268
+ file, answers ``[]``; the branch itself is confirmed first, so a missing branch
269
+ or repo still raises.
270
+ """
271
+ return [blob.path for blob in await self._blobs(None if directory is None else directory_prefix(directory))]
272
+
273
+ async def _blobs(self, prefix: str | None) -> list[_Blob]:
274
+ """Every blob in the repo, or in the directory ``prefix`` names (``"a/b/"``), with its blob sha.
241
275
 
242
- Uses the recursive Git Trees API (one request); its ``truncated`` flag is
243
- raised on loudly rather than acting on a partial listing.
276
+ Uses the recursive Git Trees API (one request) at the branch, or at
277
+ ``branch:directory`` for a ``prefix``, whose paths are relative to the directory
278
+ and are answered with ``prefix`` in front; the ``truncated`` flag is raised on
279
+ loudly rather than acting on a partial listing, and so is a blob entry carrying
280
+ no sha.
244
281
  """
245
282
  settings = _configured_settings()
246
- url = settings.trees_api_url.format(username=settings.username, repo=settings.repo, branch=settings.branch)
283
+ segments = [] if prefix is None else _path_segments(prefix)
284
+ tree = settings.branch if prefix is None else f"{settings.branch}:{quote('/'.join(segments), safe='/')}"
285
+ url = settings.trees_api_url.format(username=settings.username, repo=settings.repo, branch=tree)
247
286
  async with tai42_app.clients.client_ctx(GithubHttpxClient) as client:
248
287
  resp = await client.get(url, headers=_api_headers(settings), params={"recursive": "1"})
288
+ if prefix is not None and resp.status_code in (404, 422):
289
+ # 404: no such path on the branch; 422: the path names a blob, not a tree.
290
+ # A 404 also answers a missing branch or repo, so the branch is resolved
291
+ # first and its own failure raises.
292
+ if resp.status_code == 404:
293
+ await self._head_commit(client, settings, prefix)
294
+ return []
249
295
  _raise_for_status(resp, "listing", prefix or "/", url)
250
296
  data = resp.json()
251
297
  if not isinstance(data, dict):
252
298
  raise RuntimeError(f"Expected a tree object from {url}, got {type(data).__name__}") # noqa: TRY004 raised type is intentional (invariant/state/validation taxonomy); TypeError would change behaviour
253
299
  if data.get("truncated"):
254
300
  raise RuntimeError(f"GitHub tree listing was truncated; the tree is too large to list safely: {url}")
255
- return [
256
- item["path"]
257
- for item in data.get("tree", [])
258
- if isinstance(item, dict)
259
- and item.get("type") == "blob"
260
- and (not prefix or item["path"].startswith(prefix))
261
- ]
301
+ base = "".join(f"{segment}/" for segment in segments)
302
+ blobs: list[_Blob] = []
303
+ for item in data.get("tree", []):
304
+ if not isinstance(item, dict) or item.get("type") != "blob":
305
+ continue
306
+ path = f"{base}{item['path']}"
307
+ sha = item.get("sha")
308
+ if not isinstance(sha, str) or not sha:
309
+ raise RuntimeError(f"GitHub tree listing at {url} carries no blob sha for {path}.")
310
+ blobs.append(_Blob(path, sha))
311
+ return blobs
262
312
 
263
313
  async def upload(self, path: str, content: str) -> None:
264
314
  """Write ``content`` (UTF-8) to ``path`` via the Contents API."""
@@ -326,28 +376,40 @@ class GithubStorage(Storage):
326
376
  if not sha:
327
377
  raise RuntimeError(f"GitHub Contents API returned no blob sha for {path}; cannot delete it.")
328
378
 
329
- payload = {"message": f"Delete {path}", "sha": sha, "branch": settings.branch}
330
- resp = await client.request("DELETE", url, headers=headers, json=payload)
331
- _raise_for_status(resp, "deleting", path, url)
379
+ await self._delete_blob(client, settings, path, sha)
332
380
 
333
381
  async def delete_dir(self, path: str) -> None:
334
382
  """Delete every file under ``path`` sequentially (GitHub has no bulk delete).
335
383
 
336
- A failure mid-loop raises immediately; the partial deletion surfaces loudly.
384
+ The one tree listing supplies each file's blob sha, so each file costs one
385
+ DELETE. A file already gone is skipped; a file changed since the listing
386
+ answers 409, which raises like every other failure mid-loop, so a partial
387
+ deletion surfaces loudly.
337
388
  """
338
389
  assert_not_root(path)
339
- prefix = path if path.endswith("/") else f"{path}/"
340
- files = await self._list_blobs(prefix)
341
- if not files:
390
+ blobs = await self._blobs(directory_prefix(path))
391
+ if not blobs:
342
392
  raise FileNotFoundError(f"Directory not found or empty: {path}")
343
393
 
344
- for file_path in files:
345
- try:
346
- await self.delete(file_path)
347
- except FileNotFoundError:
348
- # Already gone (concurrent delete); the delete is idempotent. A
349
- # real DELETE failure raises HTTPStatusError and is not caught here.
350
- logger.info("Object %s already gone during dir delete of %s; skipping", file_path, path)
394
+ settings = _configured_settings()
395
+ async with tai42_app.clients.client_ctx(GithubHttpxClient) as client:
396
+ for blob in blobs:
397
+ try:
398
+ await self._delete_blob(client, settings, blob.path, blob.sha)
399
+ except FileNotFoundError:
400
+ # Already gone (concurrent delete); the delete is idempotent. A
401
+ # real DELETE failure raises HTTPStatusError and is not caught here.
402
+ logger.info("Object %s already gone during dir delete of %s; skipping", blob.path, path)
403
+
404
+ @staticmethod
405
+ async def _delete_blob(client: httpx.AsyncClient, settings: GithubStorageSettings, path: str, sha: str) -> None:
406
+ """Delete the file at ``path`` holding blob ``sha``; a missing object raises ``FileNotFoundError``."""
407
+ url = _contents_url(settings, path)
408
+ payload = {"message": f"Delete {path}", "sha": sha, "branch": settings.branch}
409
+ resp = await client.request("DELETE", url, headers=_api_headers(settings), json=payload)
410
+ if resp.status_code == 404:
411
+ raise FileNotFoundError(f"Object not found: {path}")
412
+ _raise_for_status(resp, "deleting", path, url)
351
413
 
352
414
 
353
415
  __all__ = ["GithubStorage"]
@@ -2,12 +2,12 @@ spec_version: 1
2
2
  namespace: tai42
3
3
  name: storage-github
4
4
  package: tai42-storage-github
5
- version: 2.0.1
5
+ version: 3.0.0
6
6
  description: GitHub-backed Storage provider (text + binary/media) for the TAI ecosystem.
7
7
  license: Apache-2.0
8
8
  homepage: https://tai42.ai
9
9
  repository: https://github.com/tai42ai/tai42/tree/main/plugins/storage-github
10
- contract: '>=25.0,<26'
10
+ contract: '>=26.0,<27'
11
11
  categories:
12
12
  - storage
13
13
  tags:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tai42-storage-github
3
- Version: 2.0.1
3
+ Version: 3.0.0
4
4
  Summary: GitHub-backed Storage provider (text + binary/media) for the TAI ecosystem.
5
5
  Author-email: tai42 <oss@tai42.ai>
6
6
  License-Expression: Apache-2.0
@@ -19,8 +19,8 @@ Requires-Python: >=3.13
19
19
  Description-Content-Type: text/markdown
20
20
  License-File: LICENSE
21
21
  License-File: NOTICE
22
- Requires-Dist: tai42-contract<26,>=25.0
23
- Requires-Dist: tai42-kit<14,>=13.0
22
+ Requires-Dist: tai42-contract<27,>=26.0
23
+ Requires-Dist: tai42-kit<15,>=14.0
24
24
  Requires-Dist: httpx>=0.28
25
25
  Requires-Dist: pydantic>=2.12
26
26
  Requires-Dist: pydantic-settings>=2.15.0
@@ -34,7 +34,7 @@ A GitHub-backed `Storage` provider for the TAI ecosystem. It stores content as
34
34
  files in a GitHub repository and serves the full `tai42_contract.storage.Storage`
35
35
  surface — the text methods (`load` / `list` / `upload` / `delete` / `delete_dir`)
36
36
  plus the metadata read `stat`, the bounded read `load_range` and the binary
37
- methods (`load_bytes` / `upload_bytes`).
37
+ methods (`load_bytes` / `upload_bytes` / `load_object`).
38
38
 
39
39
  ## The TAI ecosystem
40
40
 
@@ -100,11 +100,12 @@ a log line, repr, or error message.
100
100
 
101
101
  ## Reads and writes
102
102
 
103
- - **Reads** (`load`, `load_bytes`) are one Contents API call each at the
103
+ - **Reads** (`load`, `load_bytes`, `load_object`) are one Contents API call each at the
104
104
  configured branch, asking for the raw form
105
105
  (`Accept: application/vnd.github.raw+json`), which serves every byte of a file
106
106
  of any size. A directory answers its JSON listing and reads as
107
- `FileNotFoundError`.
107
+ `FileNotFoundError`. `load_object` infers the content type from the path suffix,
108
+ as `stat` does.
108
109
  - **Ranged reads** (`load_range`) resolve the branch's head commit through the
109
110
  Git refs API, then read `raw.githubusercontent.com` at that commit with a
110
111
  `Range` header and `Accept-Encoding: identity` (the raw host compresses, and a
@@ -120,7 +121,11 @@ a log line, repr, or error message.
120
121
  are capped conservatively at 1 MiB. A larger payload raises `ValueError` before
121
122
  the request rather than truncating; the API's own `422` is the backstop.
122
123
  - **Listing** uses the recursive Git Trees API (one request) and raises loudly on
123
- a `truncated` response rather than acting on a partial listing.
124
+ a `truncated` response rather than acting on a partial listing. `list(directory)`
125
+ reads the tree of that directory alone (`<branch>:<directory>`), so it grows with
126
+ the directory, not the repository; a directory that does not exist answers `[]`
127
+ once the branch is confirmed to exist. `delete_dir` lists the directory the same
128
+ way.
124
129
  - **`stat`** is one Contents API call asking for the object form
125
130
  (`Accept: application/vnd.github.object+json`): it answers the file's `size`,
126
131
  and infers `content_type` from the path suffix — GitHub stores no per-object
@@ -1,5 +1,5 @@
1
- tai42-contract<26,>=25.0
2
- tai42-kit<14,>=13.0
1
+ tai42-contract<27,>=26.0
2
+ tai42-kit<15,>=14.0
3
3
  httpx>=0.28
4
4
  pydantic>=2.12
5
5
  pydantic-settings>=2.15.0
@@ -1,4 +1,4 @@
1
- """Binary/media surface: load_bytes / upload_bytes / stat / load_range, plus the size caps.
1
+ """Binary/media surface: load_bytes / load_object / upload_bytes / stat / load_range, plus the size caps.
2
2
 
3
3
  Whole reads and ``stat`` are one Contents API call at the branch (the raw form serves
4
4
  every byte, the object form the size); a ranged read resolves the branch head and
@@ -11,7 +11,7 @@ import base64
11
11
 
12
12
  import pytest
13
13
  from httpx import HTTPStatusError
14
- from tai42_contract.storage import ObjectStat
14
+ from tai42_contract.storage import ObjectStat, StoredObject
15
15
 
16
16
  from tai42_storage_github import GithubStorage
17
17
  from tai42_storage_github.storage import MAX_UPLOAD_BYTES
@@ -69,6 +69,23 @@ async def test_load_bytes_large_file_is_read_whole(client):
69
69
  assert "raw.githubusercontent.com" not in call.args[0]
70
70
 
71
71
 
72
+ async def test_load_object_is_one_raw_read_with_the_path_type(client):
73
+ client.get.return_value = make_response(status=200, content=b"\x89PNG", headers=RAW_FORM_HEADERS)
74
+
75
+ stored = await GithubStorage().load_object("img/a.png")
76
+
77
+ assert stored == StoredObject(data=b"\x89PNG", content_type="image/png")
78
+ client.get.assert_awaited_once()
79
+ assert client.get.call_args.kwargs["headers"]["Accept"] == RAW_FORM
80
+
81
+
82
+ async def test_load_object_missing_raises_filenotfound(client):
83
+ client.get.return_value = make_response(status=404)
84
+
85
+ with pytest.raises(FileNotFoundError):
86
+ await GithubStorage().load_object("missing.bin")
87
+
88
+
72
89
  async def test_load_bytes_missing_raises_filenotfound(client):
73
90
  client.get.return_value = make_response(status=404)
74
91
 
@@ -0,0 +1,110 @@
1
+ """``delete_dir``: the root guard, one directory-tree listing, a DELETE per blob with its tree sha, and errors."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import pytest
6
+ from httpx import HTTPStatusError
7
+
8
+ from tai42_storage_github import GithubStorage
9
+
10
+ from .conftest import make_response
11
+
12
+ DIR_TREE_URL = "https://api.github.com/repos/acme/content/git/trees/dev:d"
13
+ CONTENTS_URL = "https://api.github.com/repos/acme/content/contents"
14
+
15
+
16
+ def _tree(*entries: dict[str, str]):
17
+ return make_response(status=200, json_body={"truncated": False, "tree": list(entries)})
18
+
19
+
20
+ def _deleted(client) -> list[tuple[str, str]]:
21
+ """The ``(url, sha)`` of every DELETE the client was asked to send."""
22
+ deleted = []
23
+ for call in client.request.call_args_list:
24
+ assert call.args[0] == "DELETE"
25
+ deleted.append((call.args[1], call.kwargs["json"]["sha"]))
26
+ return deleted
27
+
28
+
29
+ @pytest.mark.parametrize("root", ["", "/", ".", " ", "//", "a/..", "./x/../..", "../x", "/.."])
30
+ async def test_delete_dir_root_escape_refused(root):
31
+ with pytest.raises(ValueError, match="storage root"):
32
+ await GithubStorage().delete_dir(root)
33
+
34
+
35
+ @pytest.mark.parametrize("path", ["d", "d/"])
36
+ async def test_delete_dir_deletes_each_blob_under_it_with_its_tree_sha(client, path):
37
+ client.get.return_value = _tree(
38
+ {"path": "a.j2", "type": "blob", "sha": "s1"},
39
+ {"path": "sub", "type": "tree", "sha": "t1"},
40
+ {"path": "sub/b.j2", "type": "blob", "sha": "s2"},
41
+ )
42
+ client.request.return_value = make_response(status=200)
43
+
44
+ await GithubStorage().delete_dir(path)
45
+
46
+ client.get.assert_awaited_once()
47
+ assert client.get.call_args.args[0] == DIR_TREE_URL
48
+ assert client.get.call_args.kwargs["params"] == {"recursive": "1"}
49
+ assert _deleted(client) == [(f"{CONTENTS_URL}/d/a.j2", "s1"), (f"{CONTENTS_URL}/d/sub/b.j2", "s2")]
50
+ payload = client.request.call_args.kwargs["json"]
51
+ assert payload == {"message": "Delete d/sub/b.j2", "sha": "s2", "branch": "dev"}
52
+
53
+
54
+ async def test_delete_dir_lists_only_the_directory_tree(client):
55
+ client.get.return_value = _tree({"path": "a.j2", "type": "blob", "sha": "s1"})
56
+ client.request.return_value = make_response(status=200)
57
+
58
+ await GithubStorage().delete_dir("d/")
59
+
60
+ # The listing reads the tree at "dev:d", so "d" never reaches sibling "dd/...".
61
+ assert client.get.call_args.args[0].endswith("/git/trees/dev:d")
62
+ assert _deleted(client) == [(f"{CONTENTS_URL}/d/a.j2", "s1")]
63
+
64
+
65
+ async def test_delete_dir_of_an_empty_listing_raises_filenotfound_and_deletes_nothing(client):
66
+ client.get.return_value = _tree()
67
+
68
+ with pytest.raises(FileNotFoundError):
69
+ await GithubStorage().delete_dir("d")
70
+
71
+ client.get.assert_awaited_once()
72
+ client.request.assert_not_called()
73
+
74
+
75
+ async def test_delete_dir_skips_a_file_already_gone_and_deletes_the_rest(client):
76
+ client.get.return_value = _tree(
77
+ {"path": "a.j2", "type": "blob", "sha": "s1"},
78
+ {"path": "b.j2", "type": "blob", "sha": "s2"},
79
+ )
80
+ client.request.side_effect = [make_response(status=404), make_response(status=200)]
81
+
82
+ await GithubStorage().delete_dir("d")
83
+
84
+ assert _deleted(client) == [(f"{CONTENTS_URL}/d/a.j2", "s1"), (f"{CONTENTS_URL}/d/b.j2", "s2")]
85
+
86
+
87
+ async def test_delete_dir_raises_on_a_file_changed_since_the_listing(client):
88
+ client.get.return_value = _tree(
89
+ {"path": "a.j2", "type": "blob", "sha": "s1"},
90
+ {"path": "b.j2", "type": "blob", "sha": "s2"},
91
+ )
92
+ client.request.return_value = make_response(status=409)
93
+
94
+ with pytest.raises(HTTPStatusError):
95
+ await GithubStorage().delete_dir("d")
96
+
97
+ assert client.request.await_count == 1
98
+
99
+
100
+ async def test_delete_dir_raises_on_a_blob_without_a_sha_before_deleting_anything(client):
101
+ client.get.return_value = _tree(
102
+ {"path": "a.j2", "type": "blob", "sha": "s1"},
103
+ {"path": "b.j2", "type": "blob"},
104
+ )
105
+
106
+ with pytest.raises(RuntimeError, match="no blob sha"):
107
+ await GithubStorage().delete_dir("d")
108
+
109
+ client.get.assert_awaited_once()
110
+ client.request.assert_not_called()
@@ -82,9 +82,9 @@ async def test_list_returns_blob_paths_via_trees(client):
82
82
  json_body={
83
83
  "truncated": False,
84
84
  "tree": [
85
- {"path": "a.j2", "type": "blob"},
86
- {"path": "d", "type": "tree"},
87
- {"path": "d/b.j2", "type": "blob"},
85
+ {"path": "a.j2", "type": "blob", "sha": "s1"},
86
+ {"path": "d", "type": "tree", "sha": "t1"},
87
+ {"path": "d/b.j2", "type": "blob", "sha": "s2"},
88
88
  ],
89
89
  },
90
90
  )
@@ -123,23 +123,74 @@ async def test_list_server_error_propagates(client):
123
123
  await GithubStorage().list()
124
124
 
125
125
 
126
- async def test_list_blobs_prefix_filters_and_excludes_trees(client):
126
+ TREES = "https://api.github.com/repos/acme/content/git/trees"
127
+ REF_URL = "https://api.github.com/repos/acme/content/git/ref/heads/dev"
128
+
129
+
130
+ @pytest.mark.parametrize("directory", ["d/sub", "d/sub/", "/d/sub"])
131
+ async def test_list_of_a_directory_reads_only_its_tree_and_answers_full_paths(client, directory):
127
132
  client.get.return_value = make_response(
128
133
  status=200,
129
134
  json_body={
130
135
  "truncated": False,
131
136
  "tree": [
132
- {"path": "d/a.j2", "type": "blob"},
133
- {"path": "d/sub", "type": "tree"},
134
- {"path": "d/sub/b.j2", "type": "blob"},
135
- {"path": "dd/c.j2", "type": "blob"},
137
+ {"path": "a.j2", "type": "blob", "sha": "s1"},
138
+ {"path": "e", "type": "tree", "sha": "t1"},
139
+ {"path": "e/b.j2", "type": "blob", "sha": "s2"},
136
140
  ],
137
141
  },
138
142
  )
139
143
 
140
- files = await GithubStorage()._list_blobs("d/")
144
+ files = await GithubStorage().list(directory)
145
+
146
+ assert files == ["d/sub/a.j2", "d/sub/e/b.j2"]
147
+ assert client.get.await_count == 1
148
+ assert client.get.call_args.args[0] == f"{TREES}/dev:d/sub"
149
+ assert client.get.call_args.kwargs["params"] == {"recursive": "1"}
150
+
151
+
152
+ async def test_list_of_a_directory_quotes_its_path_into_the_tree(client):
153
+ client.get.return_value = make_response(status=200, json_body={"truncated": False, "tree": []})
154
+
155
+ await GithubStorage().list("my dir/#1")
156
+
157
+ assert client.get.call_args.args[0] == f"{TREES}/dev:my%20dir/%231"
158
+
159
+
160
+ async def test_list_of_a_missing_directory_confirms_the_branch_and_is_empty(client):
161
+ client.get.side_effect = [
162
+ make_response(status=404),
163
+ make_response(status=200, json_body={"object": {"sha": "abc123"}}),
164
+ ]
165
+
166
+ assert await GithubStorage().list("absent") == []
167
+ assert [call.args[0] for call in client.get.call_args_list] == [f"{TREES}/dev:absent", REF_URL]
168
+
169
+
170
+ async def test_list_of_a_directory_on_a_missing_branch_raises(client):
171
+ client.get.side_effect = [make_response(status=404), make_response(status=404)]
172
+
173
+ with pytest.raises(HTTPStatusError):
174
+ await GithubStorage().list("d")
175
+
176
+
177
+ async def test_list_of_a_path_naming_a_file_is_empty(client):
178
+ client.get.return_value = make_response(status=422)
179
+
180
+ assert await GithubStorage().list("a.j2") == []
181
+
182
+
183
+ async def test_list_of_a_directory_server_error_propagates(client):
184
+ client.get.return_value = make_response(status=500)
185
+
186
+ with pytest.raises(HTTPStatusError):
187
+ await GithubStorage().list("d")
141
188
 
142
- assert files == ["d/a.j2", "d/sub/b.j2"]
189
+
190
+ async def test_list_of_a_directory_with_a_parent_segment_is_refused_before_any_call(client):
191
+ with pytest.raises(ValueError, match="path traversal"):
192
+ await GithubStorage().list("d/../e")
193
+ client.get.assert_not_called()
143
194
 
144
195
 
145
196
  async def test_upload_create_puts_without_sha(client):
@@ -252,6 +303,14 @@ async def test_delete_on_directory_path_raises_filenotfound(client):
252
303
  client.request.assert_not_called()
253
304
 
254
305
 
306
+ async def test_delete_of_a_file_gone_before_its_delete_raises_filenotfound(client):
307
+ client.get.return_value = make_response(status=200, json_body={"sha": "abc"})
308
+ client.request.return_value = make_response(status=404)
309
+
310
+ with pytest.raises(FileNotFoundError):
311
+ await GithubStorage().delete("a/b.j2")
312
+
313
+
255
314
  async def test_delete_call_error_propagates(client):
256
315
  client.get.return_value = make_response(status=200, json_body={"sha": "abc"})
257
316
  client.request.return_value = make_response(status=422)
@@ -1,80 +0,0 @@
1
- """``delete_dir``: the root guard, per-file deletion, and error propagation."""
2
-
3
- from __future__ import annotations
4
-
5
- from unittest.mock import AsyncMock
6
-
7
- import pytest
8
- from httpx import HTTPStatusError, Request
9
-
10
- from tai42_storage_github import GithubStorage
11
-
12
- from .conftest import make_response
13
-
14
-
15
- @pytest.mark.parametrize("root", ["", "/", ".", " ", "//", "a/..", "./x/../..", "../x", "/.."])
16
- async def test_delete_dir_root_escape_refused(root):
17
- with pytest.raises(ValueError, match="storage root"):
18
- await GithubStorage().delete_dir(root)
19
-
20
-
21
- async def test_delete_dir_deletes_each_file(monkeypatch):
22
- store = GithubStorage()
23
- list_blobs = AsyncMock(return_value=["d/a.j2", "d/b.j2"])
24
- monkeypatch.setattr(store, "_list_blobs", list_blobs)
25
- deleted: list[str] = []
26
- monkeypatch.setattr(store, "delete", AsyncMock(side_effect=lambda p: deleted.append(p)))
27
-
28
- await store.delete_dir("d")
29
-
30
- assert deleted == ["d/a.j2", "d/b.j2"]
31
- # The prefix passed to the listing carries a trailing slash so "d" never
32
- # matches sibling "dd/...".
33
- assert list_blobs.call_args.args[0] == "d/"
34
-
35
-
36
- async def test_delete_dir_keeps_existing_trailing_slash(monkeypatch):
37
- store = GithubStorage()
38
- list_blobs = AsyncMock(return_value=["d/a.j2"])
39
- monkeypatch.setattr(store, "_list_blobs", list_blobs)
40
- monkeypatch.setattr(store, "delete", AsyncMock())
41
-
42
- await store.delete_dir("d/")
43
-
44
- assert list_blobs.call_args.args[0] == "d/"
45
-
46
-
47
- async def test_delete_dir_empty_raises_filenotfound(monkeypatch):
48
- store = GithubStorage()
49
- monkeypatch.setattr(store, "_list_blobs", AsyncMock(return_value=[]))
50
-
51
- with pytest.raises(FileNotFoundError):
52
- await store.delete_dir("d")
53
-
54
-
55
- async def test_delete_dir_tolerates_vanished_file(monkeypatch):
56
- store = GithubStorage()
57
- monkeypatch.setattr(store, "_list_blobs", AsyncMock(return_value=["d/a.j2", "d/b.j2"]))
58
- deleted: list[str] = []
59
-
60
- async def _delete(p):
61
- if p == "d/a.j2":
62
- raise FileNotFoundError("vanished")
63
- deleted.append(p)
64
-
65
- monkeypatch.setattr(store, "delete", AsyncMock(side_effect=_delete))
66
-
67
- await store.delete_dir("d")
68
-
69
- assert deleted == ["d/b.j2"]
70
-
71
-
72
- async def test_delete_dir_real_failure_propagates(monkeypatch):
73
- store = GithubStorage()
74
- monkeypatch.setattr(store, "_list_blobs", AsyncMock(return_value=["d/a.j2", "d/b.j2"]))
75
- request = Request("DELETE", "https://example.invalid")
76
- err = HTTPStatusError("boom", request=request, response=make_response(status=500))
77
- monkeypatch.setattr(store, "delete", AsyncMock(side_effect=err))
78
-
79
- with pytest.raises(HTTPStatusError):
80
- await store.delete_dir("d")