gts-agency-python-sdk 0.0.2rc1__tar.gz → 0.0.2rc2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/PKG-INFO +24 -2
  2. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/README.md +23 -1
  3. gts_agency_python_sdk-0.0.2rc2/agency_sdk/delegates/datasets_client.py +395 -0
  4. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/datasets_dto.py +50 -0
  5. gts_agency_python_sdk-0.0.2rc2/agency_sdk/test/test_datasets_client.py +323 -0
  6. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/gts_agency_python_sdk.egg-info/PKG-INFO +24 -2
  7. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/gts_agency_python_sdk.egg-info/SOURCES.txt +1 -0
  8. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/pyproject.toml +1 -1
  9. gts_agency_python_sdk-0.0.2rc1/agency_sdk/delegates/datasets_client.py +0 -109
  10. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/LICENSE +0 -0
  11. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/__init__.py +0 -0
  12. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/auth_hooks.py +0 -0
  13. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/client.py +0 -0
  14. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/credentials.py +0 -0
  15. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/__init__.py +0 -0
  16. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/annotations_client.py +0 -0
  17. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/annotations_dto.py +0 -0
  18. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/base_client.py +0 -0
  19. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/datasource_client.py +0 -0
  20. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/datasource_dto.py +0 -0
  21. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/files_client.py +0 -0
  22. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/files_dto.py +0 -0
  23. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/gateway_client.py +0 -0
  24. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/gateway_dto.py +0 -0
  25. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/ontology_client.py +0 -0
  26. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/ontology_dto.py +0 -0
  27. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/prompts_client.py +0 -0
  28. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/rules_client.py +0 -0
  29. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/rules_dto.py +0 -0
  30. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/session_client.py +0 -0
  31. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/session_dto.py +0 -0
  32. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/session_templates_client.py +0 -0
  33. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/session_templates_dto.py +0 -0
  34. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/session_vault_client.py +0 -0
  35. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/session_vault_dto.py +0 -0
  36. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/work_queue_client.py +0 -0
  37. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/work_queue_dto.py +0 -0
  38. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/domain.py +0 -0
  39. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/observability/__init__.py +0 -0
  40. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/observability/auth.py +0 -0
  41. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/observability/bootstrap.py +0 -0
  42. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/__init__.py +0 -0
  43. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/conftest.py +0 -0
  44. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_annotations_client.py +0 -0
  45. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_annotations_dto.py +0 -0
  46. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_base_client.py +0 -0
  47. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_client_facade.py +0 -0
  48. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_credentials.py +0 -0
  49. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_files_client.py +0 -0
  50. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_files_dto.py +0 -0
  51. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_gateway_dto.py +0 -0
  52. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_gateway_facade.py +0 -0
  53. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_gateway_openai.py +0 -0
  54. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_observability_auth.py +0 -0
  55. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_observability_facade_threadsafety.py +0 -0
  56. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_observability_langfuse.py +0 -0
  57. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_observability_packaging.py +0 -0
  58. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_observability_pipeline.py +0 -0
  59. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_ontology_client.py +0 -0
  60. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_ontology_dto.py +0 -0
  61. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_session_client.py +0 -0
  62. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_session_dto.py +0 -0
  63. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_session_templates_client.py +0 -0
  64. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_session_templates_dto.py +0 -0
  65. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_session_vault_client.py +0 -0
  66. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_session_vault_dto.py +0 -0
  67. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_stub_requests.py +0 -0
  68. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_work_queue_client.py +0 -0
  69. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_work_queue_dto.py +0 -0
  70. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/gts_agency_python_sdk.egg-info/dependency_links.txt +0 -0
  71. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/gts_agency_python_sdk.egg-info/requires.txt +0 -0
  72. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/gts_agency_python_sdk.egg-info/top_level.txt +0 -0
  73. {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: gts-agency_python-sdk
3
- Version: 0.0.2rc1
3
+ Version: 0.0.2rc2
4
4
  Author-email: Ground Control Systems <engineering@groundcontrolsystems.com>
5
5
  Requires-Python: >=3.12
6
6
  Description-Content-Type: text/markdown
@@ -72,7 +72,7 @@ client = AgencyClient(token_supplier=credentials, base_url="https://api.example.
72
72
 
73
73
  Access each API domain through the facade:
74
74
 
75
- - `client.dataset()` — datasets CRUD, filesystem traversal, clone
75
+ - `client.dataset()` — datasets CRUD, filesystem traversal, clone, and the write lifecycle `draft` → `push` → `publish` (`.agency/` workspace shared with `gts-cli`; [docs/datasets.md](docs/datasets.md))
76
76
  - `client.datasource()` — datasource and table introspection
77
77
  - `client.files()` — tenant file storage: list, upload, folder management, delete, signed URLs, `gtsf://` URI resolution, streamed download
78
78
  - `client.ontology()` — list ontologies (name → id, optional `kind` filter), export (JSON snapshot, Turtle/OWL, SHACL, package, package-zip), and entity-datasource mappings
@@ -85,6 +85,28 @@ Access each API domain through the facade:
85
85
  - `client.annotations()` — publish a knowledge graph as annotator work: `push_graph()` does resolve-workflow → create-batch → bind-workflow → multipart upload → read-back in one call (one job per `class: "rule"` vertex), plus the individual legs, `bind_workflow`/`list_workflows` (a batch cannot hold jobs until a workflow is bound to it), batch reads, and the job specifications that seed each job's checklist. Reading the work back is read-only and four calls: `list_jobs` (paged summaries — the payloads live on the single read), `get_job`, `list_job_transitions` (approval evidence; its `note` is PHI-capable and must never be logged), `get_graph`, and `list_batch_members` (who held which role on a batch and when — role-per-row, and the display fields are PII) ([docs/annotations.md](docs/annotations.md))
86
86
  - `client.gateway(org_id=..., gateway_base_url=...)` — OpenAI-compatible LLM calls routed through the org's agentgateway (shared credentials + `x-org` routing header)
87
87
 
88
+ ### Dataset Write Example
89
+
90
+ `clone_dataset` writes an `.agency/` workspace. `push` uploads the tree's diff against it into the
91
+ current draft. A new dataset starts as a draft, so you only need `draft` after a publish.
92
+
93
+ ```python
94
+ datasets = client.dataset()
95
+
96
+ datasets.clone_dataset(dataset_id="ds-id", organisation_id=2, target_path="./samples")
97
+ datasets.draft(dataset_id="ds-id", organisation_id=2, root="./samples") # next mutable version
98
+
99
+ # ... edit files under ./samples (never inside .agency/) ...
100
+
101
+ result = datasets.push(organisation_id=2, root="./samples") # upload changed, delete removed
102
+ print(f"uploaded={result.uploaded} deleted={result.deleted}")
103
+
104
+ datasets.publish(dataset_id="ds-id", organisation_id=2, comment="Added Q1", root="./samples")
105
+ ```
106
+
107
+ A self-checking end-to-end run is in [`examples/quick_dataset_draft_push_publish.py`](examples/quick_dataset_draft_push_publish.py)
108
+ (set `AGENCY_DATASET_ID`; each run publishes one new version).
109
+
88
110
  ### Rules Example
89
111
 
90
112
  ```python
@@ -47,7 +47,7 @@ client = AgencyClient(token_supplier=credentials, base_url="https://api.example.
47
47
 
48
48
  Access each API domain through the facade:
49
49
 
50
- - `client.dataset()` — datasets CRUD, filesystem traversal, clone
50
+ - `client.dataset()` — datasets CRUD, filesystem traversal, clone, and the write lifecycle `draft` → `push` → `publish` (`.agency/` workspace shared with `gts-cli`; [docs/datasets.md](docs/datasets.md))
51
51
  - `client.datasource()` — datasource and table introspection
52
52
  - `client.files()` — tenant file storage: list, upload, folder management, delete, signed URLs, `gtsf://` URI resolution, streamed download
53
53
  - `client.ontology()` — list ontologies (name → id, optional `kind` filter), export (JSON snapshot, Turtle/OWL, SHACL, package, package-zip), and entity-datasource mappings
@@ -60,6 +60,28 @@ Access each API domain through the facade:
60
60
  - `client.annotations()` — publish a knowledge graph as annotator work: `push_graph()` does resolve-workflow → create-batch → bind-workflow → multipart upload → read-back in one call (one job per `class: "rule"` vertex), plus the individual legs, `bind_workflow`/`list_workflows` (a batch cannot hold jobs until a workflow is bound to it), batch reads, and the job specifications that seed each job's checklist. Reading the work back is read-only and four calls: `list_jobs` (paged summaries — the payloads live on the single read), `get_job`, `list_job_transitions` (approval evidence; its `note` is PHI-capable and must never be logged), `get_graph`, and `list_batch_members` (who held which role on a batch and when — role-per-row, and the display fields are PII) ([docs/annotations.md](docs/annotations.md))
61
61
  - `client.gateway(org_id=..., gateway_base_url=...)` — OpenAI-compatible LLM calls routed through the org's agentgateway (shared credentials + `x-org` routing header)
62
62
 
63
+ ### Dataset Write Example
64
+
65
+ `clone_dataset` writes an `.agency/` workspace. `push` uploads the tree's diff against it into the
66
+ current draft. A new dataset starts as a draft, so you only need `draft` after a publish.
67
+
68
+ ```python
69
+ datasets = client.dataset()
70
+
71
+ datasets.clone_dataset(dataset_id="ds-id", organisation_id=2, target_path="./samples")
72
+ datasets.draft(dataset_id="ds-id", organisation_id=2, root="./samples") # next mutable version
73
+
74
+ # ... edit files under ./samples (never inside .agency/) ...
75
+
76
+ result = datasets.push(organisation_id=2, root="./samples") # upload changed, delete removed
77
+ print(f"uploaded={result.uploaded} deleted={result.deleted}")
78
+
79
+ datasets.publish(dataset_id="ds-id", organisation_id=2, comment="Added Q1", root="./samples")
80
+ ```
81
+
82
+ A self-checking end-to-end run is in [`examples/quick_dataset_draft_push_publish.py`](examples/quick_dataset_draft_push_publish.py)
83
+ (set `AGENCY_DATASET_ID`; each run publishes one new version).
84
+
63
85
  ### Rules Example
64
86
 
65
87
  ```python
@@ -0,0 +1,395 @@
1
+ """Datasets delegate: reads, clone, and the draft -> push -> publish write lifecycle.
2
+
3
+ The write lifecycle shares the ``.agency/`` workspace layout with ``gts-cli`` so a tree
4
+ cloned by either tool can be pushed by the other:
5
+
6
+ - ``.agency/dataset.json`` — the version GET body minus ``audit``; ``status`` 0 = draft, 1 = published.
7
+ - ``.agency/files.json`` — ``{relative/path: {hash: <sha256 hex>, last_modified: <unix secs>}}``.
8
+
9
+ The three write endpoints name the organisation differently and that is mirrored, not
10
+ normalised: dataset commands read body ``organisation_id``, upload reads query ``o``, and the
11
+ filesystem ``del`` command reads body ``organisation``. See ``docs/datasets.md``.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import hashlib
17
+ import json
18
+ import os
19
+ import re
20
+ from pathlib import Path
21
+ from typing import Any
22
+
23
+ import requests
24
+
25
+ from agency_sdk.delegates.base_client import BaseDelegateClient
26
+ from agency_sdk.delegates.datasets_dto import (
27
+ DatasetDraftResult,
28
+ DatasetFilesResponse,
29
+ DatasetManifestEntry,
30
+ DatasetPublishResult,
31
+ DatasetPushResult,
32
+ DatasetResponse,
33
+ DatasetsPagedResult,
34
+ SignedUrlResponse,
35
+ )
36
+
37
+ _AGENCY_DIR = ".agency"
38
+ _DATASET_JSON = "dataset.json"
39
+ _FILES_JSON = "files.json"
40
+
41
+ #: The server refuses to delete these (HTTP 200, ``success: false``); refuse locally first.
42
+ _PROTECTED_FILES = frozenset({"metadata.json", "README.md"})
43
+
44
+ #: The upload route rejects bodies over 100 MiB; keep headroom for the multipart framing.
45
+ _MAX_UPLOAD_BYTES = 100 * 1024 * 1024 - 64 * 1024
46
+
47
+ #: The one ``del`` refusal that means "already gone" — a retried push must not wedge on it.
48
+ _ALREADY_DELETED = re.compile(r"^Path '.*' not found$")
49
+
50
+ _DRAFT_STATUS = 0
51
+ _PUBLISHED_STATUS = 1
52
+
53
+ #: Spelled outside the class: its `list` method shadows the builtin inside the class body.
54
+ _RelPaths = list[str]
55
+
56
+
57
+ class DatasetCommandError(RuntimeError):
58
+ """A dataset call returned HTTP 2xx with ``success: false``. HTTP errors still raise ``HTTPError``."""
59
+
60
+
61
+ def _check_success(body: dict[str, Any]) -> dict[str, Any]:
62
+ if body.get("success") is False:
63
+ raise DatasetCommandError(body.get("message") or str(body))
64
+ return body
65
+
66
+
67
+ def _sha256_file(path: Path) -> str:
68
+ digest = hashlib.sha256()
69
+ with path.open("rb") as handle:
70
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
71
+ digest.update(chunk)
72
+ return digest.hexdigest()
73
+
74
+
75
+ def _manifest_entry(path: Path) -> DatasetManifestEntry:
76
+ return DatasetManifestEntry(hash=_sha256_file(path), last_modified=str(int(path.stat().st_mtime)))
77
+
78
+
79
+ def _write_json(path: Path, obj: Any) -> None:
80
+ path.write_text(json.dumps(obj, indent=2))
81
+
82
+
83
+ def _read_json(path: Path) -> Any:
84
+ return json.loads(path.read_text())
85
+
86
+
87
+ def _validate_remote_name(name: str) -> None:
88
+ """Mirror the server's upload filename rule so a bad name fails before any call."""
89
+ if not name or not name[0].isalnum() or ".." in name or "//" in name or "\\" in name:
90
+ raise ValueError(f"Invalid dataset filename {name!r}: must start alphanumeric, no '..', '//' or '\\'")
91
+
92
+
93
+ def _scan_tree(root: Path) -> dict[str, Path]:
94
+ """Map tree-relative POSIX path -> file, skipping the root ``.agency`` dir and symlinks."""
95
+ files: dict[str, Path] = {}
96
+ for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
97
+ current = Path(dirpath)
98
+ if current == root and _AGENCY_DIR in dirnames:
99
+ dirnames.remove(_AGENCY_DIR)
100
+ for filename in filenames:
101
+ path = current / filename
102
+ if path.is_symlink() or not path.is_file():
103
+ continue
104
+ files[path.relative_to(root).as_posix()] = path
105
+ return files
106
+
107
+
108
+ def _update_local_dataset(root: str | Path | None, dataset_id: str, **fields: Any) -> None:
109
+ """Patch ``.agency/dataset.json`` in place when it belongs to ``dataset_id``; otherwise do nothing."""
110
+ if root is None:
111
+ return
112
+ path = Path(root) / _AGENCY_DIR / _DATASET_JSON
113
+ if not path.is_file():
114
+ return
115
+ local = _read_json(path)
116
+ if local.get("id") != dataset_id:
117
+ return
118
+ local.update(fields)
119
+ _write_json(path, local)
120
+
121
+
122
+ class AgencyDatasetsClient(BaseDelegateClient):
123
+ api_path = "/api/datasets"
124
+
125
+ def list(self, organisation_id: int, page: int = 0, size: int = 10) -> DatasetsPagedResult:
126
+ """List all datasets for an organisation."""
127
+ params = {"o": str(organisation_id), "s": str(size), "p": str(page)}
128
+ return DatasetsPagedResult(**self._make_request("GET", "", params=params))
129
+
130
+ def get(self, dataset_id: str, organisation_id: int, version: str = "latest") -> DatasetResponse:
131
+ """Get a specific dataset by ID.
132
+
133
+ ``version`` is ``latest`` (newest *published*), ``draft`` (the status-0 row), ``last``
134
+ (newest of any status) or a version number.
135
+ """
136
+ params = {"o": str(organisation_id), "v": version}
137
+ return DatasetResponse(**self._make_request("GET", f"/{dataset_id}", params=params))
138
+
139
+ def filesystem_list(
140
+ self,
141
+ dataset_id: str,
142
+ organisation_id: int,
143
+ version: str,
144
+ path: str | None = None,
145
+ ) -> DatasetFilesResponse:
146
+ """List files and directories in a dataset filesystem."""
147
+ params: dict[str, str] = {"o": str(organisation_id), "v": version}
148
+ if path is not None:
149
+ params["path"] = path
150
+ return DatasetFilesResponse(**self._make_request("GET", f"/{dataset_id}/filesystem", params=params))
151
+
152
+ def filesystem_signed_url(
153
+ self,
154
+ dataset_id: str,
155
+ organisation_id: int,
156
+ version: str,
157
+ file_path: str,
158
+ ) -> SignedUrlResponse:
159
+ """Get a signed URL for downloading a dataset file."""
160
+ params = {"o": str(organisation_id), "v": version}
161
+ data = {
162
+ "file_path": f"/{file_path}",
163
+ "version": f"{version}",
164
+ }
165
+ return SignedUrlResponse(
166
+ **self._make_request("POST", f"/{dataset_id}/filesystem/_signed-url", data=data, params=params)
167
+ )
168
+
169
+ def clone_dataset(
170
+ self,
171
+ dataset_id: str,
172
+ organisation_id: int,
173
+ target_path: str,
174
+ version: str = "latest",
175
+ ) -> None:
176
+ """Clone a dataset to a given directory.
177
+
178
+ The process is:
179
+ 1. Get a dataset.
180
+ 2. Using the version from the get response call filesystem_list.
181
+ 3. Using that response get signed urls for all files and recursively
182
+ call filesystem_list for directories.
183
+ 4. With all the signed urls download the files to the target directory
184
+ and preserve the directory structure.
185
+ 5. Write the ``.agency/`` workspace (``dataset.json`` + ``files.json``) so the
186
+ clone can be pushed by :meth:`push` or ``gts-cli``.
187
+ """
188
+ dataset = self.get(dataset_id, organisation_id, version)
189
+ actual_version = str(dataset.version)
190
+
191
+ target_dir = Path(target_path)
192
+ target_dir.mkdir(parents=True, exist_ok=True)
193
+
194
+ downloaded: _RelPaths = []
195
+ self._clone_directory(dataset_id, organisation_id, actual_version, "", target_dir, downloaded)
196
+ self._write_workspace(target_dir, dataset, downloaded)
197
+
198
+ def _write_workspace(self, target_dir: Path, dataset: DatasetResponse, downloaded: _RelPaths) -> None:
199
+ """Write ``.agency/`` for a fresh clone.
200
+
201
+ The manifest covers only what was downloaded: files already in the target are left
202
+ out, so a later push sees them as new rather than silently treating them as synced.
203
+ """
204
+ agency_dir = target_dir / _AGENCY_DIR
205
+ agency_dir.mkdir(exist_ok=True)
206
+ _write_json(agency_dir / _DATASET_JSON, dataset.model_dump(mode="json", exclude={"audit"}))
207
+ manifest = {rel: _manifest_entry(target_dir / rel).model_dump() for rel in sorted(downloaded)}
208
+ _write_json(agency_dir / _FILES_JSON, manifest)
209
+
210
+ def _clone_directory(
211
+ self,
212
+ dataset_id: str,
213
+ organisation_id: int,
214
+ version: str,
215
+ path: str,
216
+ target_dir: Path,
217
+ downloaded: _RelPaths,
218
+ ) -> None:
219
+ """Recursively clone a directory and its contents, recording each downloaded relative path."""
220
+ files_response = self.filesystem_list(dataset_id, organisation_id, version, path if path else None)
221
+
222
+ for item in files_response.items:
223
+ if not path and item.name == _AGENCY_DIR:
224
+ continue # never let remote content overwrite the local workspace
225
+ item_target_path = target_dir / item.name
226
+ sub_path = f"{path}/{item.name}" if path else item.name
227
+
228
+ if item.item_type == "directory":
229
+ item_target_path.mkdir(exist_ok=True)
230
+ self._clone_directory(dataset_id, organisation_id, version, sub_path, item_target_path, downloaded)
231
+
232
+ elif item.item_type == "file":
233
+ signed_url_response = self.filesystem_signed_url(dataset_id, organisation_id, version, item.path)
234
+ self._download_file(signed_url_response.signed_url, item_target_path)
235
+ downloaded.append(sub_path)
236
+
237
+ def _download_file(self, signed_url: str, target_path: Path) -> None:
238
+ """Download a file from a signed URL to the target path."""
239
+ response = requests.get(signed_url, timeout=60)
240
+ response.raise_for_status()
241
+ target_path.write_bytes(response.content)
242
+
243
+ # ------------------------------------------------------------------ write lifecycle
244
+
245
+ def draft(self, dataset_id: str, organisation_id: int, root: str | Path | None = None) -> DatasetDraftResult:
246
+ """Open the next mutable version.
247
+
248
+ The server inserts ``MAX(version)+1`` as a draft and copies every object from the
249
+ latest version (of any status) into it. Local files are not uploaded.
250
+
251
+ Args:
252
+ root: Optional clone root. When its ``.agency/dataset.json`` is for this dataset, its
253
+ ``version``/``status``/``comment`` are updated so a following :meth:`push` is allowed.
254
+
255
+ Raises:
256
+ requests.HTTPError: 409 when a draft already exists (one draft at a time).
257
+ DatasetCommandError: HTTP 2xx with ``success: false``.
258
+ """
259
+ body = {"organisation_id": organisation_id, "command": "draft", "payload": {}}
260
+ response = _check_success(
261
+ self._make_request("POST", f"/{dataset_id}/_command", data=body, params={"o": str(organisation_id)})
262
+ )
263
+ result = DatasetDraftResult(**response)
264
+ _update_local_dataset(
265
+ root, dataset_id, version=result.data.version, status=_DRAFT_STATUS, comment="Draft version"
266
+ )
267
+ return result
268
+
269
+ def publish(
270
+ self, dataset_id: str, organisation_id: int, comment: str, root: str | Path | None = None
271
+ ) -> DatasetPublishResult:
272
+ """Freeze the current draft in place (status 0 -> 1). Objects are not copied or hashed.
273
+
274
+ ``comment`` is sent for parity with ``gts-cli`` but **the server does not store it** — the
275
+ version keeps the comment written when the draft was created. Read the version back to
276
+ show a comment; never echo this argument as if it had been saved.
277
+
278
+ Args:
279
+ root: Optional clone root. When it is for this dataset, ``status``/``version`` are updated
280
+ so a later :meth:`push` refuses until the next :meth:`draft`.
281
+
282
+ Raises:
283
+ requests.HTTPError: 400 when the latest version is not a draft (a second publish is an
284
+ error, not a no-op); 404 when the dataset is not visible.
285
+ DatasetCommandError: HTTP 2xx with ``success: false``.
286
+ """
287
+ body = {"organisation_id": organisation_id, "command": "publish", "payload": {"comment": comment}}
288
+ response = _check_success(
289
+ self._make_request("POST", f"/{dataset_id}/_command", data=body, params={"o": str(organisation_id)})
290
+ )
291
+ result = DatasetPublishResult(**response)
292
+ _update_local_dataset(root, dataset_id, version=result.data.version, status=_PUBLISHED_STATUS)
293
+ return result
294
+
295
+ def push(self, organisation_id: int, root: str | Path, dataset_id: str | None = None) -> DatasetPushResult:
296
+ """Upload a clone's changes into the dataset's current draft.
297
+
298
+ Diffs the tree against ``.agency/files.json`` by SHA-256: new and modified files are
299
+ uploaded (overwriting), then files missing from disk are deleted, then the manifest is
300
+ rewritten. Every local check (draft status, filenames, size cap, protected deletes) runs
301
+ before the first call so a batch never stops halfway on a predictable refusal. The
302
+ manifest is only rewritten once every upload and delete succeeded, so a failed push is
303
+ safe to re-run. No changes means no calls and no manifest rewrite.
304
+
305
+ Args:
306
+ root: Clone root containing ``.agency/``.
307
+ dataset_id: Optional guard; must equal ``.agency/dataset.json``'s ``id``.
308
+
309
+ Raises:
310
+ ValueError: no workspace, id mismatch, local status not draft, a bad filename, a file
311
+ over the upload cap, a delete of ``metadata.json``/``README.md``, or no draft on the server.
312
+ requests.HTTPError: an upload/delete failed at the HTTP level.
313
+ DatasetCommandError: an upload/delete returned ``success: false``.
314
+ """
315
+ root_dir = Path(root)
316
+ agency_dir = root_dir / _AGENCY_DIR
317
+ dataset_json = agency_dir / _DATASET_JSON
318
+ if not dataset_json.is_file():
319
+ raise ValueError(f"{dataset_json} not found; push needs a cloned dataset tree")
320
+ local = _read_json(dataset_json)
321
+ local_id = local.get("id")
322
+ if not isinstance(local_id, str) or not local_id:
323
+ raise ValueError(f"{dataset_json} has no dataset id")
324
+ if dataset_id is not None and dataset_id != local_id:
325
+ raise ValueError(f"dataset_id {dataset_id!r} does not match this tree's dataset {local_id!r}")
326
+ if local.get("status") != _DRAFT_STATUS:
327
+ raise ValueError(f"Dataset {local_id} is not a draft locally (status={local.get('status')!r}); run draft")
328
+
329
+ files_json = agency_dir / _FILES_JSON
330
+ previous: dict[str, Any] = _read_json(files_json) if files_json.is_file() else {}
331
+ current = _scan_tree(root_dir)
332
+ entries = {rel: _manifest_entry(path) for rel, path in current.items()}
333
+
334
+ uploads = sorted(rel for rel, entry in entries.items() if (previous.get(rel) or {}).get("hash") != entry.hash)
335
+ deletes = sorted(rel for rel in previous if rel not in current)
336
+ unchanged = sorted(set(entries) - set(uploads))
337
+ if not uploads and not deletes:
338
+ return DatasetPushResult(dataset_id=local_id, version=int(local.get("version", 0)), unchanged=unchanged)
339
+
340
+ for rel in uploads:
341
+ _validate_remote_name(rel)
342
+ if current[rel].stat().st_size >= _MAX_UPLOAD_BYTES:
343
+ raise ValueError(f"{rel} is over the {_MAX_UPLOAD_BYTES} byte upload limit")
344
+ protected = [rel for rel in deletes if rel in _PROTECTED_FILES]
345
+ if protected:
346
+ raise ValueError(f"Cannot delete protected files: {', '.join(protected)}")
347
+
348
+ draft_version = self._require_server_draft(local_id, organisation_id)
349
+
350
+ for rel in uploads:
351
+ self._upload_file(local_id, organisation_id, rel, current[rel], entries[rel].hash)
352
+ for rel in deletes:
353
+ self._delete_path(local_id, organisation_id, rel)
354
+
355
+ _write_json(files_json, {rel: entry.model_dump() for rel, entry in sorted(entries.items())})
356
+ return DatasetPushResult(
357
+ dataset_id=local_id, version=draft_version, uploaded=uploads, deleted=deletes, unchanged=unchanged
358
+ )
359
+
360
+ def _require_server_draft(self, dataset_id: str, organisation_id: int) -> int:
361
+ """Confirm a draft exists server-side; ``del`` with no draft is a server panic (500), not an envelope."""
362
+ try:
363
+ return self.get(dataset_id, organisation_id, "draft").version
364
+ except requests.HTTPError as error:
365
+ if error.response is not None and error.response.status_code == 404:
366
+ raise ValueError(f"Dataset {dataset_id} has no draft version on the server; run draft") from error
367
+ raise
368
+
369
+ def _upload_file(self, dataset_id: str, organisation_id: int, rel: str, path: Path, checksum: str) -> None:
370
+ """PUT one file into the current draft.
371
+
372
+ ``requests`` encodes ``data`` fields before ``files``, so ``filename`` and ``checksum`` reach the
373
+ server before the ``file`` part — required, since it only honours parts parsed before ``file``.
374
+ No ``state`` part: push must overwrite.
375
+ """
376
+ with path.open("rb") as handle:
377
+ response = requests.request(
378
+ "PUT",
379
+ f"{self.base_url}{self.api_path}/{dataset_id}/filesystem/_upload",
380
+ headers={"Authorization": f"Bearer {self.token_supplier.bearer_token()}"},
381
+ params={"o": str(organisation_id)},
382
+ data={"filename": rel, "checksum": checksum},
383
+ files={"file": (rel, handle)},
384
+ timeout=300,
385
+ )
386
+ response.raise_for_status()
387
+ _check_success(response.json() if response.content else {})
388
+
389
+ def _delete_path(self, dataset_id: str, organisation_id: int, rel: str) -> None:
390
+ """Delete one path from the current draft; ``Path '…' not found`` counts as already deleted."""
391
+ body = {"organisation": organisation_id, "command": "del", "payload": {"path": rel, "recurse": False}}
392
+ response = self._make_request("POST", f"/{dataset_id}/filesystem/_command", data=body)
393
+ if response.get("success") is False and _ALREADY_DELETED.match(str(response.get("message", ""))):
394
+ return
395
+ _check_success(response)
@@ -79,3 +79,53 @@ class SignedUrlResponse(BaseModel):
79
79
  class DatasetsPagedResult(BaseModel):
80
80
  page: Page
81
81
  items: list[DatasetSummary]
82
+
83
+
84
+ class DatasetDraftData(BaseModel):
85
+ dataset_id: str
86
+ version: int
87
+ status: str
88
+ copied_from_version: int | None = None
89
+
90
+
91
+ class DatasetDraftResult(BaseModel):
92
+ """``POST /{id}/_command`` ``draft`` envelope. ``data.status`` is the string ``"draft"`` (stored as ``0``)."""
93
+
94
+ success: bool
95
+ message: str
96
+ data: DatasetDraftData
97
+
98
+
99
+ class DatasetPublishData(BaseModel):
100
+ dataset_id: str
101
+ version: int
102
+ status: str
103
+
104
+
105
+ class DatasetPublishResult(BaseModel):
106
+ """``POST /{id}/_command`` ``publish`` envelope. ``data.version`` is the draft just frozen, not a new number."""
107
+
108
+ success: bool
109
+ message: str
110
+ data: DatasetPublishData
111
+
112
+
113
+ class DatasetManifestEntry(BaseModel):
114
+ """One ``.agency/files.json`` entry. Only ``hash`` drives change detection; ``last_modified`` is recorded."""
115
+
116
+ hash: str
117
+ last_modified: str
118
+
119
+
120
+ class DatasetPushResult(BaseModel):
121
+ """What a push did. Paths are tree-relative, forward-slashed."""
122
+
123
+ dataset_id: str
124
+ version: int
125
+ uploaded: list[str] = Field(default_factory=list)
126
+ deleted: list[str] = Field(default_factory=list)
127
+ unchanged: list[str] = Field(default_factory=list)
128
+
129
+ @property
130
+ def is_empty(self) -> bool:
131
+ return not self.uploaded and not self.deleted