gts-agency-python-sdk 0.0.2rc1__tar.gz → 0.0.2rc2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/PKG-INFO +24 -2
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/README.md +23 -1
- gts_agency_python_sdk-0.0.2rc2/agency_sdk/delegates/datasets_client.py +395 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/datasets_dto.py +50 -0
- gts_agency_python_sdk-0.0.2rc2/agency_sdk/test/test_datasets_client.py +323 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/gts_agency_python_sdk.egg-info/PKG-INFO +24 -2
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/gts_agency_python_sdk.egg-info/SOURCES.txt +1 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/pyproject.toml +1 -1
- gts_agency_python_sdk-0.0.2rc1/agency_sdk/delegates/datasets_client.py +0 -109
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/LICENSE +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/__init__.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/auth_hooks.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/client.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/credentials.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/__init__.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/annotations_client.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/annotations_dto.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/base_client.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/datasource_client.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/datasource_dto.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/files_client.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/files_dto.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/gateway_client.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/gateway_dto.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/ontology_client.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/ontology_dto.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/prompts_client.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/rules_client.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/rules_dto.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/session_client.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/session_dto.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/session_templates_client.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/session_templates_dto.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/session_vault_client.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/session_vault_dto.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/work_queue_client.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/delegates/work_queue_dto.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/domain.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/observability/__init__.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/observability/auth.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/observability/bootstrap.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/__init__.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/conftest.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_annotations_client.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_annotations_dto.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_base_client.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_client_facade.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_credentials.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_files_client.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_files_dto.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_gateway_dto.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_gateway_facade.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_gateway_openai.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_observability_auth.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_observability_facade_threadsafety.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_observability_langfuse.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_observability_packaging.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_observability_pipeline.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_ontology_client.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_ontology_dto.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_session_client.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_session_dto.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_session_templates_client.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_session_templates_dto.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_session_vault_client.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_session_vault_dto.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_stub_requests.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_work_queue_client.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/agency_sdk/test/test_work_queue_dto.py +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/gts_agency_python_sdk.egg-info/dependency_links.txt +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/gts_agency_python_sdk.egg-info/requires.txt +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/gts_agency_python_sdk.egg-info/top_level.txt +0 -0
- {gts_agency_python_sdk-0.0.2rc1 → gts_agency_python_sdk-0.0.2rc2}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gts-agency_python-sdk
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.2rc2
|
|
4
4
|
Author-email: Ground Control Systems <engineering@groundcontrolsystems.com>
|
|
5
5
|
Requires-Python: >=3.12
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -72,7 +72,7 @@ client = AgencyClient(token_supplier=credentials, base_url="https://api.example.
|
|
|
72
72
|
|
|
73
73
|
Access each API domain through the facade:
|
|
74
74
|
|
|
75
|
-
- `client.dataset()` — datasets CRUD, filesystem traversal, clone
|
|
75
|
+
- `client.dataset()` — datasets CRUD, filesystem traversal, clone, and the write lifecycle `draft` → `push` → `publish` (`.agency/` workspace shared with `gts-cli`; [docs/datasets.md](docs/datasets.md))
|
|
76
76
|
- `client.datasource()` — datasource and table introspection
|
|
77
77
|
- `client.files()` — tenant file storage: list, upload, folder management, delete, signed URLs, `gtsf://` URI resolution, streamed download
|
|
78
78
|
- `client.ontology()` — list ontologies (name → id, optional `kind` filter), export (JSON snapshot, Turtle/OWL, SHACL, package, package-zip), and entity-datasource mappings
|
|
@@ -85,6 +85,28 @@ Access each API domain through the facade:
|
|
|
85
85
|
- `client.annotations()` — publish a knowledge graph as annotator work: `push_graph()` does resolve-workflow → create-batch → bind-workflow → multipart upload → read-back in one call (one job per `class: "rule"` vertex), plus the individual legs, `bind_workflow`/`list_workflows` (a batch cannot hold jobs until a workflow is bound to it), batch reads, and the job specifications that seed each job's checklist. Reading the work back is read-only and four calls: `list_jobs` (paged summaries — the payloads live on the single read), `get_job`, `list_job_transitions` (approval evidence; its `note` is PHI-capable and must never be logged), `get_graph`, and `list_batch_members` (who held which role on a batch and when — role-per-row, and the display fields are PII) ([docs/annotations.md](docs/annotations.md))
|
|
86
86
|
- `client.gateway(org_id=..., gateway_base_url=...)` — OpenAI-compatible LLM calls routed through the org's agentgateway (shared credentials + `x-org` routing header)
|
|
87
87
|
|
|
88
|
+
### Dataset Write Example
|
|
89
|
+
|
|
90
|
+
`clone_dataset` writes an `.agency/` workspace. `push` uploads the tree's diff against it into the
|
|
91
|
+
current draft. A new dataset starts as a draft, so you only need `draft` after a publish.
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
datasets = client.dataset()
|
|
95
|
+
|
|
96
|
+
datasets.clone_dataset(dataset_id="ds-id", organisation_id=2, target_path="./samples")
|
|
97
|
+
datasets.draft(dataset_id="ds-id", organisation_id=2, root="./samples") # next mutable version
|
|
98
|
+
|
|
99
|
+
# ... edit files under ./samples (never inside .agency/) ...
|
|
100
|
+
|
|
101
|
+
result = datasets.push(organisation_id=2, root="./samples") # upload changed, delete removed
|
|
102
|
+
print(f"uploaded={result.uploaded} deleted={result.deleted}")
|
|
103
|
+
|
|
104
|
+
datasets.publish(dataset_id="ds-id", organisation_id=2, comment="Added Q1", root="./samples")
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
A self-checking end-to-end run is in [`examples/quick_dataset_draft_push_publish.py`](examples/quick_dataset_draft_push_publish.py)
|
|
108
|
+
(set `AGENCY_DATASET_ID`; each run publishes one new version).
|
|
109
|
+
|
|
88
110
|
### Rules Example
|
|
89
111
|
|
|
90
112
|
```python
|
|
@@ -47,7 +47,7 @@ client = AgencyClient(token_supplier=credentials, base_url="https://api.example.
|
|
|
47
47
|
|
|
48
48
|
Access each API domain through the facade:
|
|
49
49
|
|
|
50
|
-
- `client.dataset()` — datasets CRUD, filesystem traversal, clone
|
|
50
|
+
- `client.dataset()` — datasets CRUD, filesystem traversal, clone, and the write lifecycle `draft` → `push` → `publish` (`.agency/` workspace shared with `gts-cli`; [docs/datasets.md](docs/datasets.md))
|
|
51
51
|
- `client.datasource()` — datasource and table introspection
|
|
52
52
|
- `client.files()` — tenant file storage: list, upload, folder management, delete, signed URLs, `gtsf://` URI resolution, streamed download
|
|
53
53
|
- `client.ontology()` — list ontologies (name → id, optional `kind` filter), export (JSON snapshot, Turtle/OWL, SHACL, package, package-zip), and entity-datasource mappings
|
|
@@ -60,6 +60,28 @@ Access each API domain through the facade:
|
|
|
60
60
|
- `client.annotations()` — publish a knowledge graph as annotator work: `push_graph()` does resolve-workflow → create-batch → bind-workflow → multipart upload → read-back in one call (one job per `class: "rule"` vertex), plus the individual legs, `bind_workflow`/`list_workflows` (a batch cannot hold jobs until a workflow is bound to it), batch reads, and the job specifications that seed each job's checklist. Reading the work back is read-only and four calls: `list_jobs` (paged summaries — the payloads live on the single read), `get_job`, `list_job_transitions` (approval evidence; its `note` is PHI-capable and must never be logged), `get_graph`, and `list_batch_members` (who held which role on a batch and when — role-per-row, and the display fields are PII) ([docs/annotations.md](docs/annotations.md))
|
|
61
61
|
- `client.gateway(org_id=..., gateway_base_url=...)` — OpenAI-compatible LLM calls routed through the org's agentgateway (shared credentials + `x-org` routing header)
|
|
62
62
|
|
|
63
|
+
### Dataset Write Example
|
|
64
|
+
|
|
65
|
+
`clone_dataset` writes an `.agency/` workspace. `push` uploads the tree's diff against it into the
|
|
66
|
+
current draft. A new dataset starts as a draft, so you only need `draft` after a publish.
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
datasets = client.dataset()
|
|
70
|
+
|
|
71
|
+
datasets.clone_dataset(dataset_id="ds-id", organisation_id=2, target_path="./samples")
|
|
72
|
+
datasets.draft(dataset_id="ds-id", organisation_id=2, root="./samples") # next mutable version
|
|
73
|
+
|
|
74
|
+
# ... edit files under ./samples (never inside .agency/) ...
|
|
75
|
+
|
|
76
|
+
result = datasets.push(organisation_id=2, root="./samples") # upload changed, delete removed
|
|
77
|
+
print(f"uploaded={result.uploaded} deleted={result.deleted}")
|
|
78
|
+
|
|
79
|
+
datasets.publish(dataset_id="ds-id", organisation_id=2, comment="Added Q1", root="./samples")
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
A self-checking end-to-end run is in [`examples/quick_dataset_draft_push_publish.py`](examples/quick_dataset_draft_push_publish.py)
|
|
83
|
+
(set `AGENCY_DATASET_ID`; each run publishes one new version).
|
|
84
|
+
|
|
63
85
|
### Rules Example
|
|
64
86
|
|
|
65
87
|
```python
|
|
@@ -0,0 +1,395 @@
|
|
|
1
|
+
"""Datasets delegate: reads, clone, and the draft -> push -> publish write lifecycle.
|
|
2
|
+
|
|
3
|
+
The write lifecycle shares the ``.agency/`` workspace layout with ``gts-cli`` so a tree
|
|
4
|
+
cloned by either tool can be pushed by the other:
|
|
5
|
+
|
|
6
|
+
- ``.agency/dataset.json`` — the version GET body minus ``audit``; ``status`` 0 = draft, 1 = published.
|
|
7
|
+
- ``.agency/files.json`` — ``{relative/path: {hash: <sha256 hex>, last_modified: <unix secs>}}``.
|
|
8
|
+
|
|
9
|
+
The three write endpoints name the organisation differently and that is mirrored, not
|
|
10
|
+
normalised: dataset commands read body ``organisation_id``, upload reads query ``o``, and the
|
|
11
|
+
filesystem ``del`` command reads body ``organisation``. See ``docs/datasets.md``.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import hashlib
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import re
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
import requests
|
|
24
|
+
|
|
25
|
+
from agency_sdk.delegates.base_client import BaseDelegateClient
|
|
26
|
+
from agency_sdk.delegates.datasets_dto import (
|
|
27
|
+
DatasetDraftResult,
|
|
28
|
+
DatasetFilesResponse,
|
|
29
|
+
DatasetManifestEntry,
|
|
30
|
+
DatasetPublishResult,
|
|
31
|
+
DatasetPushResult,
|
|
32
|
+
DatasetResponse,
|
|
33
|
+
DatasetsPagedResult,
|
|
34
|
+
SignedUrlResponse,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
_AGENCY_DIR = ".agency"
|
|
38
|
+
_DATASET_JSON = "dataset.json"
|
|
39
|
+
_FILES_JSON = "files.json"
|
|
40
|
+
|
|
41
|
+
#: The server refuses to delete these (HTTP 200, ``success: false``); refuse locally first.
|
|
42
|
+
_PROTECTED_FILES = frozenset({"metadata.json", "README.md"})
|
|
43
|
+
|
|
44
|
+
#: The upload route rejects bodies over 100 MiB; keep headroom for the multipart framing.
|
|
45
|
+
_MAX_UPLOAD_BYTES = 100 * 1024 * 1024 - 64 * 1024
|
|
46
|
+
|
|
47
|
+
#: The one ``del`` refusal that means "already gone" — a retried push must not wedge on it.
|
|
48
|
+
_ALREADY_DELETED = re.compile(r"^Path '.*' not found$")
|
|
49
|
+
|
|
50
|
+
_DRAFT_STATUS = 0
|
|
51
|
+
_PUBLISHED_STATUS = 1
|
|
52
|
+
|
|
53
|
+
#: Spelled outside the class: its `list` method shadows the builtin inside the class body.
|
|
54
|
+
_RelPaths = list[str]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class DatasetCommandError(RuntimeError):
|
|
58
|
+
"""A dataset call returned HTTP 2xx with ``success: false``. HTTP errors still raise ``HTTPError``."""
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _check_success(body: dict[str, Any]) -> dict[str, Any]:
|
|
62
|
+
if body.get("success") is False:
|
|
63
|
+
raise DatasetCommandError(body.get("message") or str(body))
|
|
64
|
+
return body
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _sha256_file(path: Path) -> str:
|
|
68
|
+
digest = hashlib.sha256()
|
|
69
|
+
with path.open("rb") as handle:
|
|
70
|
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
71
|
+
digest.update(chunk)
|
|
72
|
+
return digest.hexdigest()
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _manifest_entry(path: Path) -> DatasetManifestEntry:
|
|
76
|
+
return DatasetManifestEntry(hash=_sha256_file(path), last_modified=str(int(path.stat().st_mtime)))
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _write_json(path: Path, obj: Any) -> None:
|
|
80
|
+
path.write_text(json.dumps(obj, indent=2))
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _read_json(path: Path) -> Any:
|
|
84
|
+
return json.loads(path.read_text())
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _validate_remote_name(name: str) -> None:
|
|
88
|
+
"""Mirror the server's upload filename rule so a bad name fails before any call."""
|
|
89
|
+
if not name or not name[0].isalnum() or ".." in name or "//" in name or "\\" in name:
|
|
90
|
+
raise ValueError(f"Invalid dataset filename {name!r}: must start alphanumeric, no '..', '//' or '\\'")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _scan_tree(root: Path) -> dict[str, Path]:
|
|
94
|
+
"""Map tree-relative POSIX path -> file, skipping the root ``.agency`` dir and symlinks."""
|
|
95
|
+
files: dict[str, Path] = {}
|
|
96
|
+
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
|
|
97
|
+
current = Path(dirpath)
|
|
98
|
+
if current == root and _AGENCY_DIR in dirnames:
|
|
99
|
+
dirnames.remove(_AGENCY_DIR)
|
|
100
|
+
for filename in filenames:
|
|
101
|
+
path = current / filename
|
|
102
|
+
if path.is_symlink() or not path.is_file():
|
|
103
|
+
continue
|
|
104
|
+
files[path.relative_to(root).as_posix()] = path
|
|
105
|
+
return files
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _update_local_dataset(root: str | Path | None, dataset_id: str, **fields: Any) -> None:
|
|
109
|
+
"""Patch ``.agency/dataset.json`` in place when it belongs to ``dataset_id``; otherwise do nothing."""
|
|
110
|
+
if root is None:
|
|
111
|
+
return
|
|
112
|
+
path = Path(root) / _AGENCY_DIR / _DATASET_JSON
|
|
113
|
+
if not path.is_file():
|
|
114
|
+
return
|
|
115
|
+
local = _read_json(path)
|
|
116
|
+
if local.get("id") != dataset_id:
|
|
117
|
+
return
|
|
118
|
+
local.update(fields)
|
|
119
|
+
_write_json(path, local)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class AgencyDatasetsClient(BaseDelegateClient):
|
|
123
|
+
api_path = "/api/datasets"
|
|
124
|
+
|
|
125
|
+
def list(self, organisation_id: int, page: int = 0, size: int = 10) -> DatasetsPagedResult:
|
|
126
|
+
"""List all datasets for an organisation."""
|
|
127
|
+
params = {"o": str(organisation_id), "s": str(size), "p": str(page)}
|
|
128
|
+
return DatasetsPagedResult(**self._make_request("GET", "", params=params))
|
|
129
|
+
|
|
130
|
+
def get(self, dataset_id: str, organisation_id: int, version: str = "latest") -> DatasetResponse:
|
|
131
|
+
"""Get a specific dataset by ID.
|
|
132
|
+
|
|
133
|
+
``version`` is ``latest`` (newest *published*), ``draft`` (the status-0 row), ``last``
|
|
134
|
+
(newest of any status) or a version number.
|
|
135
|
+
"""
|
|
136
|
+
params = {"o": str(organisation_id), "v": version}
|
|
137
|
+
return DatasetResponse(**self._make_request("GET", f"/{dataset_id}", params=params))
|
|
138
|
+
|
|
139
|
+
def filesystem_list(
|
|
140
|
+
self,
|
|
141
|
+
dataset_id: str,
|
|
142
|
+
organisation_id: int,
|
|
143
|
+
version: str,
|
|
144
|
+
path: str | None = None,
|
|
145
|
+
) -> DatasetFilesResponse:
|
|
146
|
+
"""List files and directories in a dataset filesystem."""
|
|
147
|
+
params: dict[str, str] = {"o": str(organisation_id), "v": version}
|
|
148
|
+
if path is not None:
|
|
149
|
+
params["path"] = path
|
|
150
|
+
return DatasetFilesResponse(**self._make_request("GET", f"/{dataset_id}/filesystem", params=params))
|
|
151
|
+
|
|
152
|
+
def filesystem_signed_url(
|
|
153
|
+
self,
|
|
154
|
+
dataset_id: str,
|
|
155
|
+
organisation_id: int,
|
|
156
|
+
version: str,
|
|
157
|
+
file_path: str,
|
|
158
|
+
) -> SignedUrlResponse:
|
|
159
|
+
"""Get a signed URL for downloading a dataset file."""
|
|
160
|
+
params = {"o": str(organisation_id), "v": version}
|
|
161
|
+
data = {
|
|
162
|
+
"file_path": f"/{file_path}",
|
|
163
|
+
"version": f"{version}",
|
|
164
|
+
}
|
|
165
|
+
return SignedUrlResponse(
|
|
166
|
+
**self._make_request("POST", f"/{dataset_id}/filesystem/_signed-url", data=data, params=params)
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
def clone_dataset(
|
|
170
|
+
self,
|
|
171
|
+
dataset_id: str,
|
|
172
|
+
organisation_id: int,
|
|
173
|
+
target_path: str,
|
|
174
|
+
version: str = "latest",
|
|
175
|
+
) -> None:
|
|
176
|
+
"""Clone a dataset to a given directory.
|
|
177
|
+
|
|
178
|
+
The process is:
|
|
179
|
+
1. Get a dataset.
|
|
180
|
+
2. Using the version from the get response call filesystem_list.
|
|
181
|
+
3. Using that response get signed urls for all files and recursively
|
|
182
|
+
call filesystem_list for directories.
|
|
183
|
+
4. With all the signed urls download the files to the target directory
|
|
184
|
+
and preserve the directory structure.
|
|
185
|
+
5. Write the ``.agency/`` workspace (``dataset.json`` + ``files.json``) so the
|
|
186
|
+
clone can be pushed by :meth:`push` or ``gts-cli``.
|
|
187
|
+
"""
|
|
188
|
+
dataset = self.get(dataset_id, organisation_id, version)
|
|
189
|
+
actual_version = str(dataset.version)
|
|
190
|
+
|
|
191
|
+
target_dir = Path(target_path)
|
|
192
|
+
target_dir.mkdir(parents=True, exist_ok=True)
|
|
193
|
+
|
|
194
|
+
downloaded: _RelPaths = []
|
|
195
|
+
self._clone_directory(dataset_id, organisation_id, actual_version, "", target_dir, downloaded)
|
|
196
|
+
self._write_workspace(target_dir, dataset, downloaded)
|
|
197
|
+
|
|
198
|
+
def _write_workspace(self, target_dir: Path, dataset: DatasetResponse, downloaded: _RelPaths) -> None:
|
|
199
|
+
"""Write ``.agency/`` for a fresh clone.
|
|
200
|
+
|
|
201
|
+
The manifest covers only what was downloaded: files already in the target are left
|
|
202
|
+
out, so a later push sees them as new rather than silently treating them as synced.
|
|
203
|
+
"""
|
|
204
|
+
agency_dir = target_dir / _AGENCY_DIR
|
|
205
|
+
agency_dir.mkdir(exist_ok=True)
|
|
206
|
+
_write_json(agency_dir / _DATASET_JSON, dataset.model_dump(mode="json", exclude={"audit"}))
|
|
207
|
+
manifest = {rel: _manifest_entry(target_dir / rel).model_dump() for rel in sorted(downloaded)}
|
|
208
|
+
_write_json(agency_dir / _FILES_JSON, manifest)
|
|
209
|
+
|
|
210
|
+
def _clone_directory(
|
|
211
|
+
self,
|
|
212
|
+
dataset_id: str,
|
|
213
|
+
organisation_id: int,
|
|
214
|
+
version: str,
|
|
215
|
+
path: str,
|
|
216
|
+
target_dir: Path,
|
|
217
|
+
downloaded: _RelPaths,
|
|
218
|
+
) -> None:
|
|
219
|
+
"""Recursively clone a directory and its contents, recording each downloaded relative path."""
|
|
220
|
+
files_response = self.filesystem_list(dataset_id, organisation_id, version, path if path else None)
|
|
221
|
+
|
|
222
|
+
for item in files_response.items:
|
|
223
|
+
if not path and item.name == _AGENCY_DIR:
|
|
224
|
+
continue # never let remote content overwrite the local workspace
|
|
225
|
+
item_target_path = target_dir / item.name
|
|
226
|
+
sub_path = f"{path}/{item.name}" if path else item.name
|
|
227
|
+
|
|
228
|
+
if item.item_type == "directory":
|
|
229
|
+
item_target_path.mkdir(exist_ok=True)
|
|
230
|
+
self._clone_directory(dataset_id, organisation_id, version, sub_path, item_target_path, downloaded)
|
|
231
|
+
|
|
232
|
+
elif item.item_type == "file":
|
|
233
|
+
signed_url_response = self.filesystem_signed_url(dataset_id, organisation_id, version, item.path)
|
|
234
|
+
self._download_file(signed_url_response.signed_url, item_target_path)
|
|
235
|
+
downloaded.append(sub_path)
|
|
236
|
+
|
|
237
|
+
def _download_file(self, signed_url: str, target_path: Path) -> None:
|
|
238
|
+
"""Download a file from a signed URL to the target path."""
|
|
239
|
+
response = requests.get(signed_url, timeout=60)
|
|
240
|
+
response.raise_for_status()
|
|
241
|
+
target_path.write_bytes(response.content)
|
|
242
|
+
|
|
243
|
+
# ------------------------------------------------------------------ write lifecycle
|
|
244
|
+
|
|
245
|
+
def draft(self, dataset_id: str, organisation_id: int, root: str | Path | None = None) -> DatasetDraftResult:
|
|
246
|
+
"""Open the next mutable version.
|
|
247
|
+
|
|
248
|
+
The server inserts ``MAX(version)+1`` as a draft and copies every object from the
|
|
249
|
+
latest version (of any status) into it. Local files are not uploaded.
|
|
250
|
+
|
|
251
|
+
Args:
|
|
252
|
+
root: Optional clone root. When its ``.agency/dataset.json`` is for this dataset, its
|
|
253
|
+
``version``/``status``/``comment`` are updated so a following :meth:`push` is allowed.
|
|
254
|
+
|
|
255
|
+
Raises:
|
|
256
|
+
requests.HTTPError: 409 when a draft already exists (one draft at a time).
|
|
257
|
+
DatasetCommandError: HTTP 2xx with ``success: false``.
|
|
258
|
+
"""
|
|
259
|
+
body = {"organisation_id": organisation_id, "command": "draft", "payload": {}}
|
|
260
|
+
response = _check_success(
|
|
261
|
+
self._make_request("POST", f"/{dataset_id}/_command", data=body, params={"o": str(organisation_id)})
|
|
262
|
+
)
|
|
263
|
+
result = DatasetDraftResult(**response)
|
|
264
|
+
_update_local_dataset(
|
|
265
|
+
root, dataset_id, version=result.data.version, status=_DRAFT_STATUS, comment="Draft version"
|
|
266
|
+
)
|
|
267
|
+
return result
|
|
268
|
+
|
|
269
|
+
def publish(
|
|
270
|
+
self, dataset_id: str, organisation_id: int, comment: str, root: str | Path | None = None
|
|
271
|
+
) -> DatasetPublishResult:
|
|
272
|
+
"""Freeze the current draft in place (status 0 -> 1). Objects are not copied or hashed.
|
|
273
|
+
|
|
274
|
+
``comment`` is sent for parity with ``gts-cli`` but **the server does not store it** — the
|
|
275
|
+
version keeps the comment written when the draft was created. Read the version back to
|
|
276
|
+
show a comment; never echo this argument as if it had been saved.
|
|
277
|
+
|
|
278
|
+
Args:
|
|
279
|
+
root: Optional clone root. When it is for this dataset, ``status``/``version`` are updated
|
|
280
|
+
so a later :meth:`push` refuses until the next :meth:`draft`.
|
|
281
|
+
|
|
282
|
+
Raises:
|
|
283
|
+
requests.HTTPError: 400 when the latest version is not a draft (a second publish is an
|
|
284
|
+
error, not a no-op); 404 when the dataset is not visible.
|
|
285
|
+
DatasetCommandError: HTTP 2xx with ``success: false``.
|
|
286
|
+
"""
|
|
287
|
+
body = {"organisation_id": organisation_id, "command": "publish", "payload": {"comment": comment}}
|
|
288
|
+
response = _check_success(
|
|
289
|
+
self._make_request("POST", f"/{dataset_id}/_command", data=body, params={"o": str(organisation_id)})
|
|
290
|
+
)
|
|
291
|
+
result = DatasetPublishResult(**response)
|
|
292
|
+
_update_local_dataset(root, dataset_id, version=result.data.version, status=_PUBLISHED_STATUS)
|
|
293
|
+
return result
|
|
294
|
+
|
|
295
|
+
def push(self, organisation_id: int, root: str | Path, dataset_id: str | None = None) -> DatasetPushResult:
|
|
296
|
+
"""Upload a clone's changes into the dataset's current draft.
|
|
297
|
+
|
|
298
|
+
Diffs the tree against ``.agency/files.json`` by SHA-256: new and modified files are
|
|
299
|
+
uploaded (overwriting), then files missing from disk are deleted, then the manifest is
|
|
300
|
+
rewritten. Every local check (draft status, filenames, size cap, protected deletes) runs
|
|
301
|
+
before the first call so a batch never stops halfway on a predictable refusal. The
|
|
302
|
+
manifest is only rewritten once every upload and delete succeeded, so a failed push is
|
|
303
|
+
safe to re-run. No changes means no calls and no manifest rewrite.
|
|
304
|
+
|
|
305
|
+
Args:
|
|
306
|
+
root: Clone root containing ``.agency/``.
|
|
307
|
+
dataset_id: Optional guard; must equal ``.agency/dataset.json``'s ``id``.
|
|
308
|
+
|
|
309
|
+
Raises:
|
|
310
|
+
ValueError: no workspace, id mismatch, local status not draft, a bad filename, a file
|
|
311
|
+
over the upload cap, a delete of ``metadata.json``/``README.md``, or no draft on the server.
|
|
312
|
+
requests.HTTPError: an upload/delete failed at the HTTP level.
|
|
313
|
+
DatasetCommandError: an upload/delete returned ``success: false``.
|
|
314
|
+
"""
|
|
315
|
+
root_dir = Path(root)
|
|
316
|
+
agency_dir = root_dir / _AGENCY_DIR
|
|
317
|
+
dataset_json = agency_dir / _DATASET_JSON
|
|
318
|
+
if not dataset_json.is_file():
|
|
319
|
+
raise ValueError(f"{dataset_json} not found; push needs a cloned dataset tree")
|
|
320
|
+
local = _read_json(dataset_json)
|
|
321
|
+
local_id = local.get("id")
|
|
322
|
+
if not isinstance(local_id, str) or not local_id:
|
|
323
|
+
raise ValueError(f"{dataset_json} has no dataset id")
|
|
324
|
+
if dataset_id is not None and dataset_id != local_id:
|
|
325
|
+
raise ValueError(f"dataset_id {dataset_id!r} does not match this tree's dataset {local_id!r}")
|
|
326
|
+
if local.get("status") != _DRAFT_STATUS:
|
|
327
|
+
raise ValueError(f"Dataset {local_id} is not a draft locally (status={local.get('status')!r}); run draft")
|
|
328
|
+
|
|
329
|
+
files_json = agency_dir / _FILES_JSON
|
|
330
|
+
previous: dict[str, Any] = _read_json(files_json) if files_json.is_file() else {}
|
|
331
|
+
current = _scan_tree(root_dir)
|
|
332
|
+
entries = {rel: _manifest_entry(path) for rel, path in current.items()}
|
|
333
|
+
|
|
334
|
+
uploads = sorted(rel for rel, entry in entries.items() if (previous.get(rel) or {}).get("hash") != entry.hash)
|
|
335
|
+
deletes = sorted(rel for rel in previous if rel not in current)
|
|
336
|
+
unchanged = sorted(set(entries) - set(uploads))
|
|
337
|
+
if not uploads and not deletes:
|
|
338
|
+
return DatasetPushResult(dataset_id=local_id, version=int(local.get("version", 0)), unchanged=unchanged)
|
|
339
|
+
|
|
340
|
+
for rel in uploads:
|
|
341
|
+
_validate_remote_name(rel)
|
|
342
|
+
if current[rel].stat().st_size >= _MAX_UPLOAD_BYTES:
|
|
343
|
+
raise ValueError(f"{rel} is over the {_MAX_UPLOAD_BYTES} byte upload limit")
|
|
344
|
+
protected = [rel for rel in deletes if rel in _PROTECTED_FILES]
|
|
345
|
+
if protected:
|
|
346
|
+
raise ValueError(f"Cannot delete protected files: {', '.join(protected)}")
|
|
347
|
+
|
|
348
|
+
draft_version = self._require_server_draft(local_id, organisation_id)
|
|
349
|
+
|
|
350
|
+
for rel in uploads:
|
|
351
|
+
self._upload_file(local_id, organisation_id, rel, current[rel], entries[rel].hash)
|
|
352
|
+
for rel in deletes:
|
|
353
|
+
self._delete_path(local_id, organisation_id, rel)
|
|
354
|
+
|
|
355
|
+
_write_json(files_json, {rel: entry.model_dump() for rel, entry in sorted(entries.items())})
|
|
356
|
+
return DatasetPushResult(
|
|
357
|
+
dataset_id=local_id, version=draft_version, uploaded=uploads, deleted=deletes, unchanged=unchanged
|
|
358
|
+
)
|
|
359
|
+
|
|
360
|
+
def _require_server_draft(self, dataset_id: str, organisation_id: int) -> int:
|
|
361
|
+
"""Confirm a draft exists server-side; ``del`` with no draft is a server panic (500), not an envelope."""
|
|
362
|
+
try:
|
|
363
|
+
return self.get(dataset_id, organisation_id, "draft").version
|
|
364
|
+
except requests.HTTPError as error:
|
|
365
|
+
if error.response is not None and error.response.status_code == 404:
|
|
366
|
+
raise ValueError(f"Dataset {dataset_id} has no draft version on the server; run draft") from error
|
|
367
|
+
raise
|
|
368
|
+
|
|
369
|
+
def _upload_file(self, dataset_id: str, organisation_id: int, rel: str, path: Path, checksum: str) -> None:
|
|
370
|
+
"""PUT one file into the current draft.
|
|
371
|
+
|
|
372
|
+
``requests`` encodes ``data`` fields before ``files``, so ``filename`` and ``checksum`` reach the
|
|
373
|
+
server before the ``file`` part — required, since it only honours parts parsed before ``file``.
|
|
374
|
+
No ``state`` part: push must overwrite.
|
|
375
|
+
"""
|
|
376
|
+
with path.open("rb") as handle:
|
|
377
|
+
response = requests.request(
|
|
378
|
+
"PUT",
|
|
379
|
+
f"{self.base_url}{self.api_path}/{dataset_id}/filesystem/_upload",
|
|
380
|
+
headers={"Authorization": f"Bearer {self.token_supplier.bearer_token()}"},
|
|
381
|
+
params={"o": str(organisation_id)},
|
|
382
|
+
data={"filename": rel, "checksum": checksum},
|
|
383
|
+
files={"file": (rel, handle)},
|
|
384
|
+
timeout=300,
|
|
385
|
+
)
|
|
386
|
+
response.raise_for_status()
|
|
387
|
+
_check_success(response.json() if response.content else {})
|
|
388
|
+
|
|
389
|
+
def _delete_path(self, dataset_id: str, organisation_id: int, rel: str) -> None:
|
|
390
|
+
"""Delete one path from the current draft; ``Path '…' not found`` counts as already deleted."""
|
|
391
|
+
body = {"organisation": organisation_id, "command": "del", "payload": {"path": rel, "recurse": False}}
|
|
392
|
+
response = self._make_request("POST", f"/{dataset_id}/filesystem/_command", data=body)
|
|
393
|
+
if response.get("success") is False and _ALREADY_DELETED.match(str(response.get("message", ""))):
|
|
394
|
+
return
|
|
395
|
+
_check_success(response)
|
|
@@ -79,3 +79,53 @@ class SignedUrlResponse(BaseModel):
|
|
|
79
79
|
class DatasetsPagedResult(BaseModel):
|
|
80
80
|
page: Page
|
|
81
81
|
items: list[DatasetSummary]
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class DatasetDraftData(BaseModel):
|
|
85
|
+
dataset_id: str
|
|
86
|
+
version: int
|
|
87
|
+
status: str
|
|
88
|
+
copied_from_version: int | None = None
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class DatasetDraftResult(BaseModel):
|
|
92
|
+
"""``POST /{id}/_command`` ``draft`` envelope. ``data.status`` is the string ``"draft"`` (stored as ``0``)."""
|
|
93
|
+
|
|
94
|
+
success: bool
|
|
95
|
+
message: str
|
|
96
|
+
data: DatasetDraftData
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class DatasetPublishData(BaseModel):
|
|
100
|
+
dataset_id: str
|
|
101
|
+
version: int
|
|
102
|
+
status: str
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class DatasetPublishResult(BaseModel):
|
|
106
|
+
"""``POST /{id}/_command`` ``publish`` envelope. ``data.version`` is the draft just frozen, not a new number."""
|
|
107
|
+
|
|
108
|
+
success: bool
|
|
109
|
+
message: str
|
|
110
|
+
data: DatasetPublishData
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class DatasetManifestEntry(BaseModel):
|
|
114
|
+
"""One ``.agency/files.json`` entry. Only ``hash`` drives change detection; ``last_modified`` is recorded."""
|
|
115
|
+
|
|
116
|
+
hash: str
|
|
117
|
+
last_modified: str
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class DatasetPushResult(BaseModel):
|
|
121
|
+
"""What a push did. Paths are tree-relative, forward-slashed."""
|
|
122
|
+
|
|
123
|
+
dataset_id: str
|
|
124
|
+
version: int
|
|
125
|
+
uploaded: list[str] = Field(default_factory=list)
|
|
126
|
+
deleted: list[str] = Field(default_factory=list)
|
|
127
|
+
unchanged: list[str] = Field(default_factory=list)
|
|
128
|
+
|
|
129
|
+
@property
|
|
130
|
+
def is_empty(self) -> bool:
|
|
131
|
+
return not self.uploaded and not self.deleted
|