gts-agency-python-sdk 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agency_sdk/__init__.py +0 -0
- agency_sdk/auth_hooks.py +55 -0
- agency_sdk/client.py +212 -0
- agency_sdk/credentials.py +54 -0
- agency_sdk/delegates/__init__.py +0 -0
- agency_sdk/delegates/annotations_client.py +495 -0
- agency_sdk/delegates/annotations_dto.py +200 -0
- agency_sdk/delegates/base_client.py +126 -0
- agency_sdk/delegates/datasets_client.py +109 -0
- agency_sdk/delegates/datasets_dto.py +81 -0
- agency_sdk/delegates/datasource_client.py +44 -0
- agency_sdk/delegates/datasource_dto.py +107 -0
- agency_sdk/delegates/files_client.py +184 -0
- agency_sdk/delegates/files_dto.py +36 -0
- agency_sdk/delegates/gateway_client.py +81 -0
- agency_sdk/delegates/gateway_dto.py +37 -0
- agency_sdk/delegates/ontology_client.py +203 -0
- agency_sdk/delegates/ontology_dto.py +221 -0
- agency_sdk/delegates/prompts_client.py +56 -0
- agency_sdk/delegates/rules_client.py +33 -0
- agency_sdk/delegates/rules_dto.py +108 -0
- agency_sdk/delegates/session_client.py +95 -0
- agency_sdk/delegates/session_dto.py +94 -0
- agency_sdk/delegates/session_templates_client.py +23 -0
- agency_sdk/delegates/session_templates_dto.py +27 -0
- agency_sdk/delegates/session_vault_client.py +135 -0
- agency_sdk/delegates/session_vault_dto.py +75 -0
- agency_sdk/delegates/work_queue_client.py +230 -0
- agency_sdk/delegates/work_queue_dto.py +127 -0
- agency_sdk/domain.py +140 -0
- agency_sdk/observability/__init__.py +60 -0
- agency_sdk/observability/auth.py +15 -0
- agency_sdk/observability/bootstrap.py +344 -0
- agency_sdk/test/__init__.py +0 -0
- agency_sdk/test/conftest.py +106 -0
- agency_sdk/test/test_annotations_client.py +599 -0
- agency_sdk/test/test_annotations_dto.py +299 -0
- agency_sdk/test/test_base_client.py +153 -0
- agency_sdk/test/test_client_facade.py +105 -0
- agency_sdk/test/test_credentials.py +67 -0
- agency_sdk/test/test_files_client.py +264 -0
- agency_sdk/test/test_files_dto.py +93 -0
- agency_sdk/test/test_gateway_dto.py +70 -0
- agency_sdk/test/test_gateway_facade.py +229 -0
- agency_sdk/test/test_gateway_openai.py +76 -0
- agency_sdk/test/test_observability_auth.py +125 -0
- agency_sdk/test/test_observability_facade_threadsafety.py +50 -0
- agency_sdk/test/test_observability_langfuse.py +76 -0
- agency_sdk/test/test_observability_packaging.py +78 -0
- agency_sdk/test/test_observability_pipeline.py +246 -0
- agency_sdk/test/test_ontology_client.py +152 -0
- agency_sdk/test/test_ontology_dto.py +158 -0
- agency_sdk/test/test_session_client.py +128 -0
- agency_sdk/test/test_session_dto.py +75 -0
- agency_sdk/test/test_session_templates_client.py +36 -0
- agency_sdk/test/test_session_templates_dto.py +45 -0
- agency_sdk/test/test_session_vault_client.py +151 -0
- agency_sdk/test/test_session_vault_dto.py +27 -0
- agency_sdk/test/test_stub_requests.py +39 -0
- agency_sdk/test/test_work_queue_client.py +388 -0
- agency_sdk/test/test_work_queue_dto.py +233 -0
- gts_agency_python_sdk-0.0.1.dist-info/METADATA +221 -0
- gts_agency_python_sdk-0.0.1.dist-info/RECORD +66 -0
- gts_agency_python_sdk-0.0.1.dist-info/WHEEL +5 -0
- gts_agency_python_sdk-0.0.1.dist-info/licenses/LICENSE +201 -0
- gts_agency_python_sdk-0.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""Shared base for the per-domain delegate clients.
|
|
2
|
+
|
|
3
|
+
Centralizes credential / base-URL storage and the common authenticated-request
|
|
4
|
+
plumbing (bearer header, optional JSON content type, 30s timeout,
|
|
5
|
+
``raise_for_status``) so each delegate only sets its ``api_path`` and declares its
|
|
6
|
+
typed methods.
|
|
7
|
+
|
|
8
|
+
- Most delegates use :meth:`_make_request`, which parses and returns the JSON body.
|
|
9
|
+
- Delegates that need the raw response (e.g. ontology export, which returns
|
|
10
|
+
non-JSON text) call :meth:`_request` with ``json_content_type=False``.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import time
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
import requests
|
|
19
|
+
|
|
20
|
+
from agency_sdk.credentials import CredentialsSupplier
|
|
21
|
+
|
|
22
|
+
#: Backoff waits between transport retries; ``len`` + 1 = total attempts (here: 3).
|
|
23
|
+
_TRANSPORT_RETRY_WAITS: tuple[float, ...] = (1.0, 2.0)
|
|
24
|
+
|
|
25
|
+
#: Methods whose connection failures are safe to retry automatically. Only pure
|
|
26
|
+
#: reads qualify: a ``ConnectionError`` can be a reset AFTER the server committed
|
|
27
|
+
#: (``RemoteDisconnected``), so retrying a write would double-apply a POST or hit a
|
|
28
|
+
#: now-404 on a DELETE. Writers that are genuinely idempotent opt in per call
|
|
29
|
+
#: (``retry=True``) — never by default.
|
|
30
|
+
_AUTO_RETRY_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class BaseDelegateClient:
|
|
34
|
+
#: API path prefix appended to ``base_url`` (e.g. ``"/api/datasets"``). Set by subclasses.
|
|
35
|
+
api_path: str = ""
|
|
36
|
+
|
|
37
|
+
def __init__(self, token_supplier: CredentialsSupplier, base_url: str = "http://localhost:9003"):
|
|
38
|
+
self.base_url = base_url.rstrip("/")
|
|
39
|
+
self.token_supplier = token_supplier
|
|
40
|
+
|
|
41
|
+
def _request(
|
|
42
|
+
self,
|
|
43
|
+
method: str,
|
|
44
|
+
endpoint: str,
|
|
45
|
+
*,
|
|
46
|
+
data: Any = None,
|
|
47
|
+
params: dict[str, Any] | None = None,
|
|
48
|
+
json_content_type: bool = True,
|
|
49
|
+
retry: bool | None = None,
|
|
50
|
+
) -> requests.Response:
|
|
51
|
+
"""Issue an authenticated request, raise on HTTP error, and return the raw response.
|
|
52
|
+
|
|
53
|
+
Args:
|
|
54
|
+
retry: Override connection-retry behaviour. ``None`` (default) auto-retries
|
|
55
|
+
only pure reads (:data:`_AUTO_RETRY_METHODS`). Pass ``True`` to retry a
|
|
56
|
+
write that is genuinely safe to repeat (e.g. an idempotent update),
|
|
57
|
+
``False`` to force a single attempt.
|
|
58
|
+
"""
|
|
59
|
+
headers = {"Authorization": f"Bearer {self.token_supplier.bearer_token()}"}
|
|
60
|
+
if json_content_type:
|
|
61
|
+
headers["Content-Type"] = "application/json"
|
|
62
|
+
should_retry = method.upper() in _AUTO_RETRY_METHODS if retry is None else retry
|
|
63
|
+
response = self._send_with_retry(
|
|
64
|
+
method=method,
|
|
65
|
+
url=f"{self.base_url}{self.api_path}{endpoint}",
|
|
66
|
+
headers=headers,
|
|
67
|
+
data=data,
|
|
68
|
+
params=params,
|
|
69
|
+
retry=should_retry,
|
|
70
|
+
)
|
|
71
|
+
response.raise_for_status()
|
|
72
|
+
return response
|
|
73
|
+
|
|
74
|
+
def _send_with_retry(
|
|
75
|
+
self,
|
|
76
|
+
*,
|
|
77
|
+
method: str,
|
|
78
|
+
url: str,
|
|
79
|
+
headers: dict[str, str],
|
|
80
|
+
data: Any,
|
|
81
|
+
params: dict[str, Any] | None,
|
|
82
|
+
retry: bool,
|
|
83
|
+
) -> requests.Response:
|
|
84
|
+
"""Issue the transport call, optionally retrying CONNECTION failures with backoff.
|
|
85
|
+
|
|
86
|
+
When ``retry`` is true, retries ``requests.ConnectionError`` (connect blips, DNS,
|
|
87
|
+
connect-timeout, or a mid-flight reset) with backoff before a final attempt whose
|
|
88
|
+
error propagates. Two things are NEVER retried, regardless of ``retry``:
|
|
89
|
+
|
|
90
|
+
- **Read timeouts:** a slow server means retrying would only multiply the hang
|
|
91
|
+
(cf. the files-inbox 3A publish block); ``ReadTimeout`` is not a
|
|
92
|
+
``ConnectionError``, so it falls straight through.
|
|
93
|
+
- **HTTP status errors:** ``raise_for_status`` runs in ``_request``, outside this
|
|
94
|
+
loop, so control-flow codes like 409 are never re-sent.
|
|
95
|
+
|
|
96
|
+
``retry`` is gated by the caller because a ``ConnectionError`` can be a reset that
|
|
97
|
+
arrives AFTER the server committed the request, so re-sending a non-idempotent
|
|
98
|
+
write (POST create, etc.) would double-apply it — hence only reads auto-retry and
|
|
99
|
+
writers must opt in explicitly.
|
|
100
|
+
"""
|
|
101
|
+
if retry:
|
|
102
|
+
for wait in _TRANSPORT_RETRY_WAITS:
|
|
103
|
+
try:
|
|
104
|
+
return requests.request(
|
|
105
|
+
method=method, url=url, headers=headers, json=data, params=params, timeout=30
|
|
106
|
+
)
|
|
107
|
+
except requests.ConnectionError:
|
|
108
|
+
time.sleep(wait)
|
|
109
|
+
return requests.request(method=method, url=url, headers=headers, json=data, params=params, timeout=30)
|
|
110
|
+
|
|
111
|
+
def _make_request(
|
|
112
|
+
self,
|
|
113
|
+
method: str,
|
|
114
|
+
endpoint: str,
|
|
115
|
+
data: Any = None,
|
|
116
|
+
params: dict[str, Any] | None = None,
|
|
117
|
+
retry: bool | None = None,
|
|
118
|
+
) -> dict[str, Any]:
|
|
119
|
+
"""Issue a JSON request and return the parsed dict body (``{}`` when empty).
|
|
120
|
+
|
|
121
|
+
``retry`` follows :meth:`_request`: ``None`` auto-retries reads only; ``True``
|
|
122
|
+
opts a safe-to-repeat write into connection-retry.
|
|
123
|
+
"""
|
|
124
|
+
response = self._request(method, endpoint, data=data, params=params, retry=retry)
|
|
125
|
+
result: dict[str, Any] = response.json() if response.content else {}
|
|
126
|
+
return result
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
import requests
|
|
4
|
+
|
|
5
|
+
from agency_sdk.delegates.base_client import BaseDelegateClient
|
|
6
|
+
from agency_sdk.delegates.datasets_dto import (
|
|
7
|
+
DatasetFilesResponse,
|
|
8
|
+
DatasetResponse,
|
|
9
|
+
DatasetsPagedResult,
|
|
10
|
+
SignedUrlResponse,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class AgencyDatasetsClient(BaseDelegateClient):
|
|
15
|
+
api_path = "/api/datasets"
|
|
16
|
+
|
|
17
|
+
def list(self, organisation_id: int, page: int = 0, size: int = 10) -> DatasetsPagedResult:
|
|
18
|
+
"""List all datasets for an organisation."""
|
|
19
|
+
params = {"o": str(organisation_id), "s": str(size), "p": str(page)}
|
|
20
|
+
return DatasetsPagedResult(**self._make_request("GET", "", params=params))
|
|
21
|
+
|
|
22
|
+
def get(self, dataset_id: str, organisation_id: int, version: str = "latest") -> DatasetResponse:
|
|
23
|
+
"""Get a specific dataset by ID."""
|
|
24
|
+
params = {"o": str(organisation_id), "v": version}
|
|
25
|
+
return DatasetResponse(**self._make_request("GET", f"/{dataset_id}", params=params))
|
|
26
|
+
|
|
27
|
+
def filesystem_list(
|
|
28
|
+
self,
|
|
29
|
+
dataset_id: str,
|
|
30
|
+
organisation_id: int,
|
|
31
|
+
version: str,
|
|
32
|
+
path: str | None = None,
|
|
33
|
+
) -> DatasetFilesResponse:
|
|
34
|
+
"""List files and directories in a dataset filesystem."""
|
|
35
|
+
params: dict[str, str] = {"o": str(organisation_id), "v": version}
|
|
36
|
+
if path is not None:
|
|
37
|
+
params["path"] = path
|
|
38
|
+
return DatasetFilesResponse(**self._make_request("GET", f"/{dataset_id}/filesystem", params=params))
|
|
39
|
+
|
|
40
|
+
def filesystem_signed_url(
|
|
41
|
+
self,
|
|
42
|
+
dataset_id: str,
|
|
43
|
+
organisation_id: int,
|
|
44
|
+
version: str,
|
|
45
|
+
file_path: str,
|
|
46
|
+
) -> SignedUrlResponse:
|
|
47
|
+
"""Get a signed URL for downloading a dataset file."""
|
|
48
|
+
params = {"o": str(organisation_id), "v": version}
|
|
49
|
+
data = {
|
|
50
|
+
"file_path": f"/{file_path}",
|
|
51
|
+
"version": f"{version}",
|
|
52
|
+
}
|
|
53
|
+
return SignedUrlResponse(
|
|
54
|
+
**self._make_request("POST", f"/{dataset_id}/filesystem/_signed-url", data=data, params=params)
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
def clone_dataset(
|
|
58
|
+
self,
|
|
59
|
+
dataset_id: str,
|
|
60
|
+
organisation_id: int,
|
|
61
|
+
target_path: str,
|
|
62
|
+
version: str = "latest",
|
|
63
|
+
) -> None:
|
|
64
|
+
"""Clone a dataset to a given directory.
|
|
65
|
+
|
|
66
|
+
The process is:
|
|
67
|
+
1. Get a dataset.
|
|
68
|
+
2. Using the version from the get response call filesystem_list.
|
|
69
|
+
3. Using that response get signed urls for all files and recursively
|
|
70
|
+
call filesystem_list for directories.
|
|
71
|
+
4. With all the signed urls download the files to the target directory
|
|
72
|
+
and preserve the directory structure.
|
|
73
|
+
"""
|
|
74
|
+
dataset = self.get(dataset_id, organisation_id, version)
|
|
75
|
+
actual_version = str(dataset.version)
|
|
76
|
+
|
|
77
|
+
target_dir = Path(target_path)
|
|
78
|
+
target_dir.mkdir(parents=True, exist_ok=True)
|
|
79
|
+
|
|
80
|
+
self._clone_directory(dataset_id, organisation_id, actual_version, "", target_dir)
|
|
81
|
+
|
|
82
|
+
def _clone_directory(
|
|
83
|
+
self,
|
|
84
|
+
dataset_id: str,
|
|
85
|
+
organisation_id: int,
|
|
86
|
+
version: str,
|
|
87
|
+
path: str,
|
|
88
|
+
target_dir: Path,
|
|
89
|
+
) -> None:
|
|
90
|
+
"""Recursively clone a directory and its contents."""
|
|
91
|
+
files_response = self.filesystem_list(dataset_id, organisation_id, version, path if path else None)
|
|
92
|
+
|
|
93
|
+
for item in files_response.items:
|
|
94
|
+
item_target_path = target_dir / item.name
|
|
95
|
+
|
|
96
|
+
if item.item_type == "directory":
|
|
97
|
+
item_target_path.mkdir(exist_ok=True)
|
|
98
|
+
sub_path = f"{path}/{item.name}" if path else item.name
|
|
99
|
+
self._clone_directory(dataset_id, organisation_id, version, sub_path, item_target_path)
|
|
100
|
+
|
|
101
|
+
elif item.item_type == "file":
|
|
102
|
+
signed_url_response = self.filesystem_signed_url(dataset_id, organisation_id, version, item.path)
|
|
103
|
+
self._download_file(signed_url_response.signed_url, item_target_path)
|
|
104
|
+
|
|
105
|
+
def _download_file(self, signed_url: str, target_path: Path) -> None:
|
|
106
|
+
"""Download a file from a signed URL to the target path."""
|
|
107
|
+
response = requests.get(signed_url, timeout=60)
|
|
108
|
+
response.raise_for_status()
|
|
109
|
+
target_path.write_bytes(response.content)
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
from pydantic import BaseModel, Field
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Page(BaseModel):
|
|
8
|
+
total: int
|
|
9
|
+
page: int
|
|
10
|
+
size: int
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class DatasetVersion(BaseModel):
|
|
14
|
+
version: str
|
|
15
|
+
published: bool
|
|
16
|
+
comment: str | None = None
|
|
17
|
+
created_at: str
|
|
18
|
+
created_by: str
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class DatasetSummary(BaseModel):
|
|
22
|
+
id: str
|
|
23
|
+
name: str
|
|
24
|
+
description: str
|
|
25
|
+
tags: Any
|
|
26
|
+
published_version: int | None = None
|
|
27
|
+
unpublished_version: int | None = None
|
|
28
|
+
modified_on: datetime.datetime
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class AuditData(BaseModel):
|
|
32
|
+
pass
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class DatasetResponse(BaseModel):
|
|
36
|
+
id: str
|
|
37
|
+
name: str
|
|
38
|
+
description: str
|
|
39
|
+
tags: Any
|
|
40
|
+
comment: str
|
|
41
|
+
version: int
|
|
42
|
+
status: int
|
|
43
|
+
metadata: Any
|
|
44
|
+
readme: str
|
|
45
|
+
audit: AuditData
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class DatasetFileItem(BaseModel):
|
|
49
|
+
name: str
|
|
50
|
+
item_type: str = Field(alias="type")
|
|
51
|
+
path: str
|
|
52
|
+
size: int | None = None
|
|
53
|
+
content_type: str | None = None
|
|
54
|
+
last_modified: str
|
|
55
|
+
file_count: int | None = None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class DatasetFilesResponse(BaseModel):
|
|
59
|
+
dataset_id: str
|
|
60
|
+
version: str
|
|
61
|
+
path: str
|
|
62
|
+
items: list[DatasetFileItem]
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class FileInfo(BaseModel):
|
|
66
|
+
name: str
|
|
67
|
+
path: str
|
|
68
|
+
size: int
|
|
69
|
+
content_type: str
|
|
70
|
+
last_modified: str
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class SignedUrlResponse(BaseModel):
|
|
74
|
+
signed_url: str
|
|
75
|
+
expires_at: str
|
|
76
|
+
file_info: FileInfo
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class DatasetsPagedResult(BaseModel):
|
|
80
|
+
page: Page
|
|
81
|
+
items: list[DatasetSummary]
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from agency_sdk.delegates.base_client import BaseDelegateClient
|
|
2
|
+
from agency_sdk.delegates.datasource_dto import (
|
|
3
|
+
DatasourceDetail,
|
|
4
|
+
DatasourceTableDetail,
|
|
5
|
+
DatasourceTablesPagedResult,
|
|
6
|
+
DatasourcesPagedResult,
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class AgencyDatasourceClient(BaseDelegateClient):
|
|
11
|
+
api_path = "/api/datasources"
|
|
12
|
+
|
|
13
|
+
def list(self, organisation_id: int, page: int = 0, size: int = 10) -> DatasourcesPagedResult:
|
|
14
|
+
"""List all datasources for an organisation."""
|
|
15
|
+
params = {"o": str(organisation_id), "s": str(size), "p": str(page)}
|
|
16
|
+
return DatasourcesPagedResult(**self._make_request("GET", "", params=params))
|
|
17
|
+
|
|
18
|
+
def get(self, datasource_id: str, organisation_id: int) -> DatasourceDetail:
|
|
19
|
+
"""Get a specific datasource by ID."""
|
|
20
|
+
params = {"o": str(organisation_id)}
|
|
21
|
+
return DatasourceDetail(**self._make_request("GET", f"/{datasource_id}", params=params))
|
|
22
|
+
|
|
23
|
+
def list_tables(
|
|
24
|
+
self,
|
|
25
|
+
datasource_id: str,
|
|
26
|
+
organisation_id: int,
|
|
27
|
+
page: int = 0,
|
|
28
|
+
size: int = 10,
|
|
29
|
+
) -> DatasourceTablesPagedResult:
|
|
30
|
+
"""List tables for a datasource."""
|
|
31
|
+
params = {"o": str(organisation_id), "s": str(size), "p": str(page)}
|
|
32
|
+
return DatasourceTablesPagedResult(**self._make_request("GET", f"/{datasource_id}/tables", params=params))
|
|
33
|
+
|
|
34
|
+
def get_table(
|
|
35
|
+
self,
|
|
36
|
+
datasource_id: str,
|
|
37
|
+
table_name: str,
|
|
38
|
+
organisation_id: int,
|
|
39
|
+
) -> DatasourceTableDetail:
|
|
40
|
+
"""Get detailed information about a specific table."""
|
|
41
|
+
params = {"o": str(organisation_id)}
|
|
42
|
+
return DatasourceTableDetail(
|
|
43
|
+
**self._make_request("GET", f"/{datasource_id}/tables/{table_name}", params=params)
|
|
44
|
+
)
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
from pydantic import BaseModel, ConfigDict
|
|
2
|
+
|
|
3
|
+
from agency_sdk.delegates.datasets_dto import Page
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def _to_camel(name: str) -> str:
|
|
7
|
+
parts = name.split("_")
|
|
8
|
+
return parts[0] + "".join(p.capitalize() for p in parts[1:])
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class DatasourceSummary(BaseModel):
|
|
12
|
+
model_config = ConfigDict(alias_generator=_to_camel, populate_by_name=True)
|
|
13
|
+
|
|
14
|
+
id: str
|
|
15
|
+
organization_id: int
|
|
16
|
+
name: str
|
|
17
|
+
display_name: str | None = None
|
|
18
|
+
description: str | None = None
|
|
19
|
+
datasource_type: str
|
|
20
|
+
status: str
|
|
21
|
+
table_count: int | None = None
|
|
22
|
+
last_sync_at: str | None = None
|
|
23
|
+
last_sync_status: str | None = None
|
|
24
|
+
created_at: str
|
|
25
|
+
updated_at: str
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class DatasourceDetail(BaseModel):
|
|
29
|
+
model_config = ConfigDict(alias_generator=_to_camel, populate_by_name=True)
|
|
30
|
+
|
|
31
|
+
id: str
|
|
32
|
+
organization_id: int
|
|
33
|
+
name: str
|
|
34
|
+
display_name: str | None = None
|
|
35
|
+
description: str | None = None
|
|
36
|
+
datasource_type: str
|
|
37
|
+
status: str
|
|
38
|
+
table_count: int | None = None
|
|
39
|
+
last_sync_at: str | None = None
|
|
40
|
+
last_sync_status: str | None = None
|
|
41
|
+
last_sync_error: str | None = None
|
|
42
|
+
host: str | None = None
|
|
43
|
+
port: int | None = None
|
|
44
|
+
database_name: str | None = None
|
|
45
|
+
username: str | None = None
|
|
46
|
+
connection_options: str | None = None
|
|
47
|
+
last_connection_test: str | None = None
|
|
48
|
+
last_connection_error: str | None = None
|
|
49
|
+
created_at: str
|
|
50
|
+
updated_at: str
|
|
51
|
+
created_by: str | None = None
|
|
52
|
+
updated_by: str | None = None
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class DatasourcesPagedResult(BaseModel):
|
|
56
|
+
page: Page
|
|
57
|
+
items: list[DatasourceSummary]
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class DatasourceTable(BaseModel):
|
|
61
|
+
model_config = ConfigDict(alias_generator=_to_camel, populate_by_name=True)
|
|
62
|
+
|
|
63
|
+
table_name: str
|
|
64
|
+
table_schema: str | None = None
|
|
65
|
+
table_type: str | None = None
|
|
66
|
+
table_comment: str | None = None
|
|
67
|
+
row_count: int | None = None
|
|
68
|
+
synced_at: str | None = None
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class DatasourceTableColumn(BaseModel):
|
|
72
|
+
model_config = ConfigDict(alias_generator=_to_camel, populate_by_name=True)
|
|
73
|
+
|
|
74
|
+
column_name: str
|
|
75
|
+
ordinal_position: int
|
|
76
|
+
data_type: str
|
|
77
|
+
is_nullable: bool
|
|
78
|
+
column_default: str | None = None
|
|
79
|
+
column_comment: str | None = None
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class DatasourceTableConstraint(BaseModel):
|
|
83
|
+
model_config = ConfigDict(alias_generator=_to_camel, populate_by_name=True)
|
|
84
|
+
|
|
85
|
+
constraint_name: str
|
|
86
|
+
constraint_type: str
|
|
87
|
+
column_names: list[str]
|
|
88
|
+
referenced_table: str | None = None
|
|
89
|
+
referenced_column: str | None = None
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class DatasourceTableDetail(BaseModel):
|
|
93
|
+
model_config = ConfigDict(alias_generator=_to_camel, populate_by_name=True)
|
|
94
|
+
|
|
95
|
+
table_name: str
|
|
96
|
+
table_schema: str | None = None
|
|
97
|
+
table_type: str | None = None
|
|
98
|
+
table_comment: str | None = None
|
|
99
|
+
row_count: int | None = None
|
|
100
|
+
synced_at: str | None = None
|
|
101
|
+
columns: list[DatasourceTableColumn]
|
|
102
|
+
constraints: list[DatasourceTableConstraint]
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class DatasourceTablesPagedResult(BaseModel):
|
|
106
|
+
page: Page
|
|
107
|
+
items: list[DatasourceTable]
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"""Client for the tenant file storage API (/api/files).
|
|
2
|
+
|
|
3
|
+
Includes resolution of ``gtsf://<file_id>`` URIs, the convention used in GTS
|
|
4
|
+
configurations and rule annotations to reference stored files.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import contextlib
|
|
8
|
+
import mimetypes
|
|
9
|
+
from collections.abc import Sequence
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any, BinaryIO
|
|
12
|
+
|
|
13
|
+
import requests
|
|
14
|
+
|
|
15
|
+
from agency_sdk.delegates.base_client import BaseDelegateClient
|
|
16
|
+
from agency_sdk.delegates.files_dto import FileEntry, FilesPagedResult, SignedUrlResponse, UploadResult
|
|
17
|
+
|
|
18
|
+
GTSF_SCHEME = "gtsf://"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class AgencyFilesClient(BaseDelegateClient):
|
|
22
|
+
api_path = "/api/files"
|
|
23
|
+
|
|
24
|
+
def list(self, organisation_id: int, path: str = "", page: int = 0, size: int = 50) -> FilesPagedResult:
|
|
25
|
+
"""List files and folders at a logical path (folders first, paginated).
|
|
26
|
+
|
|
27
|
+
Args:
|
|
28
|
+
organisation_id: The organisation ID.
|
|
29
|
+
path: Directory path to list (default: root, "").
|
|
30
|
+
page: Zero-indexed page number.
|
|
31
|
+
size: Page size (server default 50).
|
|
32
|
+
"""
|
|
33
|
+
params = {"o": str(organisation_id), "path": path, "p": str(page), "s": str(size)}
|
|
34
|
+
return FilesPagedResult(**self._make_request("GET", "", params=params))
|
|
35
|
+
|
|
36
|
+
def signed_url(self, file_id: str, organisation_id: int, expires: int | None = None) -> SignedUrlResponse:
|
|
37
|
+
"""Get a temporary signed download URL for a file.
|
|
38
|
+
|
|
39
|
+
Args:
|
|
40
|
+
file_id: The file identifier.
|
|
41
|
+
organisation_id: The organisation ID.
|
|
42
|
+
expires: URL lifetime in seconds. Server default is 900 (15 minutes),
|
|
43
|
+
clamped server-side to [1, 604800] (7 days).
|
|
44
|
+
|
|
45
|
+
Raises:
|
|
46
|
+
requests.HTTPError: 404 if the file does not exist, 400 if the id
|
|
47
|
+
refers to a folder.
|
|
48
|
+
"""
|
|
49
|
+
params = {"o": str(organisation_id)}
|
|
50
|
+
if expires is not None:
|
|
51
|
+
params["expires"] = str(expires)
|
|
52
|
+
return SignedUrlResponse(**self._make_request("GET", f"/{file_id}/_signed-url", params=params))
|
|
53
|
+
|
|
54
|
+
def upload(self, organisation_id: int, file_paths: Sequence[str | Path], path: str = "") -> UploadResult:
|
|
55
|
+
"""Upload one or more local files to a logical folder.
|
|
56
|
+
|
|
57
|
+
Each file is sent as a multipart "file" field with its content type
|
|
58
|
+
guessed from the filename. Server limits: 100 MiB per file AND per
|
|
59
|
+
request body (multiple files share the body cap). Uploading a name
|
|
60
|
+
that already exists in the folder overwrites it (the previous entry
|
|
61
|
+
is soft-deleted server-side).
|
|
62
|
+
|
|
63
|
+
Memory: ``requests`` does not stream file handles; it reads each file
|
|
64
|
+
fully and assembles the whole multipart body in memory, so peak usage
|
|
65
|
+
is roughly twice the total bytes uploaded during encoding (the server
|
|
66
|
+
100 MiB cap bounds this). Mind this under concurrency.
|
|
67
|
+
|
|
68
|
+
Args:
|
|
69
|
+
organisation_id: The organisation ID.
|
|
70
|
+
file_paths: Local paths of the files to upload.
|
|
71
|
+
path: Destination folder path ("" = root).
|
|
72
|
+
|
|
73
|
+
Raises:
|
|
74
|
+
ValueError: If file_paths is empty (before any network call).
|
|
75
|
+
"""
|
|
76
|
+
if not file_paths:
|
|
77
|
+
raise ValueError("file_paths must not be empty")
|
|
78
|
+
url = f"{self.base_url}/api/files/_upload"
|
|
79
|
+
params = {"o": str(organisation_id), "path": path}
|
|
80
|
+
with contextlib.ExitStack() as stack:
|
|
81
|
+
files: list[tuple[str, tuple[str, BinaryIO, str | None]]] = []
|
|
82
|
+
for raw_path in file_paths:
|
|
83
|
+
file_path = Path(raw_path)
|
|
84
|
+
handle = stack.enter_context(file_path.open("rb"))
|
|
85
|
+
content_type, _ = mimetypes.guess_type(file_path.name)
|
|
86
|
+
files.append(("file", (file_path.name, handle, content_type)))
|
|
87
|
+
response = requests.post(
|
|
88
|
+
url,
|
|
89
|
+
headers={"Authorization": f"Bearer {self.token_supplier.bearer_token()}"},
|
|
90
|
+
params=params,
|
|
91
|
+
files=files,
|
|
92
|
+
timeout=300,
|
|
93
|
+
)
|
|
94
|
+
response.raise_for_status()
|
|
95
|
+
return UploadResult(**response.json())
|
|
96
|
+
|
|
97
|
+
def create_folder(self, organisation_id: int, name: str, folder_path: str = "") -> FileEntry:
|
|
98
|
+
"""Create a virtual folder.
|
|
99
|
+
|
|
100
|
+
Args:
|
|
101
|
+
organisation_id: The organisation ID.
|
|
102
|
+
name: Name of the new folder. Must not be empty or contain
|
|
103
|
+
'/', '\\' or '..' (server-validated, 400).
|
|
104
|
+
folder_path: Parent folder path ("" = root).
|
|
105
|
+
|
|
106
|
+
Raises:
|
|
107
|
+
requests.HTTPError: 409 if a file or folder with that name already
|
|
108
|
+
exists in the parent folder.
|
|
109
|
+
"""
|
|
110
|
+
params = {"o": str(organisation_id)}
|
|
111
|
+
data = {"folder_path": folder_path, "name": name}
|
|
112
|
+
return FileEntry(**self._make_request("POST", "/_folder", data=data, params=params))
|
|
113
|
+
|
|
114
|
+
def resolve_gtsf_uri(self, uri: str, organisation_id: int, expires: int | None = None) -> SignedUrlResponse:
|
|
115
|
+
"""Resolve a ``gtsf://<file_id>`` URI to a signed download URL plus metadata.
|
|
116
|
+
|
|
117
|
+
Args:
|
|
118
|
+
uri: A strict ``gtsf://<file_id>`` URI (lowercase scheme, single
|
|
119
|
+
path-less file id).
|
|
120
|
+
organisation_id: The organisation ID.
|
|
121
|
+
expires: Optional URL lifetime in seconds (see signed_url).
|
|
122
|
+
|
|
123
|
+
Raises:
|
|
124
|
+
ValueError: If the URI is malformed; raised before any network call.
|
|
125
|
+
"""
|
|
126
|
+
if not uri.startswith(GTSF_SCHEME):
|
|
127
|
+
raise ValueError(f"Not a gtsf:// URI: {uri!r}")
|
|
128
|
+
file_id = uri[len(GTSF_SCHEME) :]
|
|
129
|
+
if not file_id or "/" in file_id:
|
|
130
|
+
raise ValueError(f"Invalid gtsf:// file id: {uri!r}")
|
|
131
|
+
return self.signed_url(file_id=file_id, organisation_id=organisation_id, expires=expires)
|
|
132
|
+
|
|
133
|
+
def download(self, file_id: str, organisation_id: int, target_path: str | Path) -> FileEntry:
|
|
134
|
+
"""Download a file to a local path via its signed URL (streamed).
|
|
135
|
+
|
|
136
|
+
Fetches a signed URL, then streams the blob from object storage in
|
|
137
|
+
chunks (files can be up to 100 MiB). Parent directories are created
|
|
138
|
+
as needed. After streaming, the number of bytes written is verified
|
|
139
|
+
against the file's known size to catch a silently truncated download
|
|
140
|
+
(e.g. a mid-stream connection drop).
|
|
141
|
+
|
|
142
|
+
Returns:
|
|
143
|
+
The downloaded file's metadata.
|
|
144
|
+
|
|
145
|
+
Raises:
|
|
146
|
+
IOError: If the number of bytes written does not match the
|
|
147
|
+
expected file size.
|
|
148
|
+
"""
|
|
149
|
+
resolved = self.signed_url(file_id=file_id, organisation_id=organisation_id)
|
|
150
|
+
target = Path(target_path)
|
|
151
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
152
|
+
response = requests.get(resolved.signed_url, stream=True, timeout=300)
|
|
153
|
+
response.raise_for_status()
|
|
154
|
+
bytes_written = 0
|
|
155
|
+
with target.open("wb") as handle:
|
|
156
|
+
for chunk in response.iter_content(chunk_size=1024 * 1024):
|
|
157
|
+
handle.write(chunk)
|
|
158
|
+
bytes_written += len(chunk)
|
|
159
|
+
if bytes_written != resolved.file.size_bytes:
|
|
160
|
+
raise IOError(
|
|
161
|
+
f"Truncated download for file {file_id}: wrote {bytes_written} bytes, "
|
|
162
|
+
f"expected {resolved.file.size_bytes}"
|
|
163
|
+
)
|
|
164
|
+
return resolved.file
|
|
165
|
+
|
|
166
|
+
def delete_file(self, file_id: str, organisation_id: int) -> None:
|
|
167
|
+
"""Soft-delete a single file.
|
|
168
|
+
|
|
169
|
+
Raises:
|
|
170
|
+
requests.HTTPError: 404 if the file does not exist, 400 if the id
|
|
171
|
+
refers to a folder (use delete_folder instead).
|
|
172
|
+
"""
|
|
173
|
+
params = {"o": str(organisation_id)}
|
|
174
|
+
self._make_request("DELETE", f"/{file_id}", params=params)
|
|
175
|
+
|
|
176
|
+
def delete_folder(self, organisation_id: int, path: str) -> None:
|
|
177
|
+
"""Recursively soft-delete a virtual folder and all its contents.
|
|
178
|
+
|
|
179
|
+
Args:
|
|
180
|
+
organisation_id: The organisation ID.
|
|
181
|
+
path: Full path of the folder to delete.
|
|
182
|
+
"""
|
|
183
|
+
params = {"o": str(organisation_id), "path": path}
|
|
184
|
+
self._make_request("DELETE", "/_folder", params=params)
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""DTOs for the tenant file storage API (snake_case, matching the API)."""
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel
|
|
4
|
+
|
|
5
|
+
from agency_sdk.delegates.datasets_dto import Page
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class FileEntry(BaseModel):
|
|
9
|
+
"""A file or folder as returned by the files API."""
|
|
10
|
+
|
|
11
|
+
id: str
|
|
12
|
+
name: str
|
|
13
|
+
folder_path: str
|
|
14
|
+
path: str
|
|
15
|
+
is_folder: bool
|
|
16
|
+
content_type: str | None = None
|
|
17
|
+
size_bytes: int
|
|
18
|
+
uploaded_by: int
|
|
19
|
+
created_on: str
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class FilesPagedResult(BaseModel):
|
|
23
|
+
page: Page
|
|
24
|
+
items: list[FileEntry]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class UploadResult(BaseModel):
|
|
28
|
+
uploaded: list[FileEntry]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class SignedUrlResponse(BaseModel):
|
|
32
|
+
"""A time-limited signed download URL with the file's metadata."""
|
|
33
|
+
|
|
34
|
+
signed_url: str
|
|
35
|
+
expires_at: str
|
|
36
|
+
file: FileEntry
|