pal-found-cli 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pal_found_cli/__init__.py +1 -0
- pal_found_cli/_version.py +24 -0
- pal_found_cli/admin/__init__.py +2 -0
- pal_found_cli/admin/scripts/__init__.py +2 -0
- pal_found_cli/admin/scripts/pal_found_admin_cli.py +600 -0
- pal_found_cli/aip_agents/__init__.py +1 -0
- pal_found_cli/aip_agents/metadata-allow-list.md +20 -0
- pal_found_cli/aip_agents/scripts/__init__.py +1 -0
- pal_found_cli/aip_agents/scripts/pal_found_aip_agents_cli.py +704 -0
- pal_found_cli/audit/__init__.py +1 -0
- pal_found_cli/audit/metadata-allow-list.md +4 -0
- pal_found_cli/audit/scripts/__init__.py +1 -0
- pal_found_cli/audit/scripts/pal_found_audit_cli.py +387 -0
- pal_found_cli/checkpoints/__init__.py +9 -0
- pal_found_cli/checkpoints/metadata-allow-list.md +7 -0
- pal_found_cli/checkpoints/scripts/__init__.py +1 -0
- pal_found_cli/checkpoints/scripts/pal_found_checkpoints_cli.py +527 -0
- pal_found_cli/common/__init__.py +56 -0
- pal_found_cli/common/access_control_guard.py +591 -0
- pal_found_cli/common/async_client_factory.py +161 -0
- pal_found_cli/common/auth_provider.py +82 -0
- pal_found_cli/common/binary_download_handler.py +288 -0
- pal_found_cli/common/config_loader.py +388 -0
- pal_found_cli/common/error_serializer.py +412 -0
- pal_found_cli/common/log_setup.py +241 -0
- pal_found_cli/common/metadata-allow-list.md +169 -0
- pal_found_cli/common/output_formatter.py +267 -0
- pal_found_cli/common/pagination_helper.py +212 -0
- pal_found_cli/common/retry.py +617 -0
- pal_found_cli/common/sdk_error_utils.py +112 -0
- pal_found_cli/common/session_manager.py +461 -0
- pal_found_cli/common/tracing_provider.py +107 -0
- pal_found_cli/connectivity/__init__.py +9 -0
- pal_found_cli/connectivity/metadata-allow-list.md +24 -0
- pal_found_cli/connectivity/scripts/__init__.py +1 -0
- pal_found_cli/connectivity/scripts/pal_found_connectivity_cli.py +720 -0
- pal_found_cli/data_health/__init__.py +9 -0
- pal_found_cli/data_health/metadata-allow-list.md +10 -0
- pal_found_cli/data_health/scripts/__init__.py +1 -0
- pal_found_cli/data_health/scripts/pal_found_data_health_cli.py +458 -0
- pal_found_cli/datasets/__init__.py +1 -0
- pal_found_cli/datasets/scripts/__init__.py +1 -0
- pal_found_cli/datasets/scripts/pal_found_datasets_cli.py +732 -0
- pal_found_cli/filesystem/__init__.py +2 -0
- pal_found_cli/filesystem/scripts/__init__.py +2 -0
- pal_found_cli/filesystem/scripts/pal_found_filesystem_cli.py +659 -0
- pal_found_cli/functions/__init__.py +2 -0
- pal_found_cli/functions/scripts/__init__.py +2 -0
- pal_found_cli/functions/scripts/pal_found_functions_cli.py +389 -0
- pal_found_cli/language_models/__init__.py +1 -0
- pal_found_cli/language_models/metadata-allow-list.md +6 -0
- pal_found_cli/language_models/scripts/__init__.py +1 -0
- pal_found_cli/language_models/scripts/pal_found_language_models_cli.py +342 -0
- pal_found_cli/media_sets/__init__.py +9 -0
- pal_found_cli/media_sets/metadata-allow-list.md +23 -0
- pal_found_cli/media_sets/scripts/__init__.py +1 -0
- pal_found_cli/media_sets/scripts/pal_found_media_sets_cli.py +642 -0
- pal_found_cli/models/__init__.py +5 -0
- pal_found_cli/models/metadata-allow-list.md +27 -0
- pal_found_cli/models/scripts/__init__.py +1 -0
- pal_found_cli/models/scripts/pal_found_models_cli.py +714 -0
- pal_found_cli/ontologies/__init__.py +1 -0
- pal_found_cli/ontologies/scripts/__init__.py +1 -0
- pal_found_cli/ontologies/scripts/pal_found_ontologies_cli.py +1225 -0
- pal_found_cli/orchestration/__init__.py +9 -0
- pal_found_cli/orchestration/metadata-allow-list.md +24 -0
- pal_found_cli/orchestration/scripts/__init__.py +1 -0
- pal_found_cli/orchestration/scripts/pal_found_orchestration_cli.py +640 -0
- pal_found_cli/sql_queries/__init__.py +9 -0
- pal_found_cli/sql_queries/metadata-allow-list.md +9 -0
- pal_found_cli/sql_queries/scripts/__init__.py +1 -0
- pal_found_cli/sql_queries/scripts/pal_found_sql_queries_cli.py +514 -0
- pal_found_cli/streams/__init__.py +9 -0
- pal_found_cli/streams/metadata-allow-list.md +19 -0
- pal_found_cli/streams/scripts/__init__.py +1 -0
- pal_found_cli/streams/scripts/pal_found_streams_cli.py +615 -0
- pal_found_cli/third_party_applications/__init__.py +9 -0
- pal_found_cli/third_party_applications/metadata-allow-list.md +13 -0
- pal_found_cli/third_party_applications/scripts/__init__.py +1 -0
- pal_found_cli/third_party_applications/scripts/pal_found_third_party_applications_cli.py +547 -0
- pal_found_cli/widgets/__init__.py +9 -0
- pal_found_cli/widgets/metadata-allow-list.md +12 -0
- pal_found_cli/widgets/scripts/__init__.py +1 -0
- pal_found_cli/widgets/scripts/pal_found_widgets_cli.py +571 -0
- pal_found_cli-0.1.2.dist-info/METADATA +74 -0
- pal_found_cli-0.1.2.dist-info/RECORD +89 -0
- pal_found_cli-0.1.2.dist-info/WHEEL +5 -0
- pal_found_cli-0.1.2.dist-info/entry_points.txt +19 -0
- pal_found_cli-0.1.2.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,732 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Foundry Datasets CLI - 33 operations across 5 resource clients.
|
|
3
|
+
|
|
4
|
+
Exposes all Foundry Datasets API v2 operations as CLI subcommands:
|
|
5
|
+
- Dataset (11): create, get, get-health-check-reports, get-health-checks,
|
|
6
|
+
get-schedules, get-schema, get-schema-batch, jobs, put-schema, read-table, transactions
|
|
7
|
+
- Branch (5): create, delete, get, list, transactions
|
|
8
|
+
- File (5): content, delete, get, list, upload
|
|
9
|
+
- Transaction (6): abort, build, commit, create, get, job
|
|
10
|
+
- View (6): add-backing-datasets, add-primary-key, create, get,
|
|
11
|
+
remove-backing-datasets, replace-backing-datasets
|
|
12
|
+
|
|
13
|
+
Usage: python pal_found_datasets_cli.py <resource> <operation> [options]
|
|
14
|
+
Output: JSON/TOON on stdout, metadata on stderr (ADR-004, ADR-005).
|
|
15
|
+
Exit codes per ADR-001 taxonomy.
|
|
16
|
+
Access Control: Auth -> Access Control -> API call order (SRS 4.2, ADR-007).
|
|
17
|
+
Retry: Exponential backoff with jitter per ADR-002.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
import argparse
|
|
21
|
+
import asyncio
|
|
22
|
+
import json
|
|
23
|
+
import logging
|
|
24
|
+
import sys
|
|
25
|
+
from typing import Any, cast
|
|
26
|
+
|
|
27
|
+
from pal_found_cli.common.access_control_guard import (
|
|
28
|
+
AccessControlError,
|
|
29
|
+
AccessControlGuard,
|
|
30
|
+
)
|
|
31
|
+
from pal_found_cli.common.async_client_factory import AsyncClientFactory
|
|
32
|
+
from pal_found_cli.common.config_loader import ConfigLoader
|
|
33
|
+
from pal_found_cli.common.error_serializer import (
|
|
34
|
+
EXIT_ACCESS_CONTROL,
|
|
35
|
+
EXIT_CONFIGURATION,
|
|
36
|
+
EXIT_NOT_FOUND,
|
|
37
|
+
EXIT_PERMISSION_DENIED,
|
|
38
|
+
EXIT_RATE_LIMIT,
|
|
39
|
+
EXIT_SERVER_ERROR,
|
|
40
|
+
EXIT_SUCCESS,
|
|
41
|
+
EXIT_TIMEOUT,
|
|
42
|
+
EXIT_USER_INPUT,
|
|
43
|
+
ErrorSerializer,
|
|
44
|
+
)
|
|
45
|
+
from pal_found_cli.common.log_setup import LogSetup
|
|
46
|
+
from pal_found_cli.common.output_formatter import OutputFormatter
|
|
47
|
+
from pal_found_cli.common.pagination_helper import PaginationHelper
|
|
48
|
+
from pal_found_cli.common.retry import RetryHandler
|
|
49
|
+
|
|
50
|
+
logger = logging.getLogger(__name__)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _model_to_dict(obj: Any) -> Any:
|
|
54
|
+
"""Convert Pydantic model to serializable dict.
|
|
55
|
+
|
|
56
|
+
Handles Pydantic v2 (model_dump), Pydantic v1 (dict), nested collections.
|
|
57
|
+
Checks list/dict before model methods to avoid treating collections as models.
|
|
58
|
+
Uses try/except so MagicMock or objects with dynamic attributes don't corrupt output.
|
|
59
|
+
"""
|
|
60
|
+
if obj is None:
|
|
61
|
+
return None
|
|
62
|
+
if isinstance(obj, list):
|
|
63
|
+
return [_model_to_dict(i) for i in obj]
|
|
64
|
+
if isinstance(obj, dict):
|
|
65
|
+
return {k: _model_to_dict(v) for k, v in obj.items()}
|
|
66
|
+
# Try Pydantic v2 first
|
|
67
|
+
try:
|
|
68
|
+
result = obj.model_dump()
|
|
69
|
+
if isinstance(result, dict):
|
|
70
|
+
return result
|
|
71
|
+
except (AttributeError, TypeError):
|
|
72
|
+
pass
|
|
73
|
+
# Fallback to Pydantic v1
|
|
74
|
+
try:
|
|
75
|
+
result = obj.dict()
|
|
76
|
+
if isinstance(result, dict):
|
|
77
|
+
return result
|
|
78
|
+
except (AttributeError, TypeError):
|
|
79
|
+
pass
|
|
80
|
+
return obj
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _get_client(cfg: ConfigLoader, resource: str) -> Any:
|
|
84
|
+
"""Get the SDK client for a resource.
|
|
85
|
+
|
|
86
|
+
Parameters
|
|
87
|
+
----------
|
|
88
|
+
cfg : ConfigLoader
|
|
89
|
+
Configuration instance.
|
|
90
|
+
resource : str
|
|
91
|
+
Resource name: dataset, branch, file, transaction, view.
|
|
92
|
+
|
|
93
|
+
Returns
|
|
94
|
+
-------
|
|
95
|
+
Any
|
|
96
|
+
SDK client for the requested resource.
|
|
97
|
+
"""
|
|
98
|
+
client = AsyncClientFactory().create(cfg).datasets.Dataset
|
|
99
|
+
resource_map = {
|
|
100
|
+
"branch": "Branch",
|
|
101
|
+
"file": "File",
|
|
102
|
+
"transaction": "Transaction",
|
|
103
|
+
"view": "View",
|
|
104
|
+
}
|
|
105
|
+
attr = resource_map.get(resource)
|
|
106
|
+
return getattr(client, attr) if attr else client
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
async def _invoke(
|
|
110
|
+
resource: str,
|
|
111
|
+
operation: str,
|
|
112
|
+
client: Any,
|
|
113
|
+
args: argparse.Namespace,
|
|
114
|
+
timeout: int | None,
|
|
115
|
+
) -> Any:
|
|
116
|
+
"""Invoke SDK operation (async).
|
|
117
|
+
|
|
118
|
+
Parameters
|
|
119
|
+
----------
|
|
120
|
+
resource : str
|
|
121
|
+
Resource name.
|
|
122
|
+
operation : str
|
|
123
|
+
snake_case operation name.
|
|
124
|
+
client : Any
|
|
125
|
+
SDK client instance.
|
|
126
|
+
args : argparse.Namespace
|
|
127
|
+
Parsed CLI arguments.
|
|
128
|
+
timeout : Optional[int]
|
|
129
|
+
Request timeout in seconds. May be ``None`` to let the SDK decide.
|
|
130
|
+
|
|
131
|
+
Returns
|
|
132
|
+
-------
|
|
133
|
+
Any
|
|
134
|
+
SDK response object.
|
|
135
|
+
|
|
136
|
+
Raises
|
|
137
|
+
------
|
|
138
|
+
ValueError
|
|
139
|
+
If the operation is not recognized.
|
|
140
|
+
"""
|
|
141
|
+
dr = getattr(args, "dataset_rid", None)
|
|
142
|
+
bn = getattr(args, "branch_name", None)
|
|
143
|
+
tr = getattr(args, "transaction_rid", None)
|
|
144
|
+
ps = getattr(args, "page_size", None)
|
|
145
|
+
pt = getattr(args, "page_token", None)
|
|
146
|
+
fp = getattr(args, "file_path", None)
|
|
147
|
+
vr = getattr(args, "view_dataset_rid", None)
|
|
148
|
+
br = getattr(args, "branch", None)
|
|
149
|
+
nm = getattr(args, "name", None)
|
|
150
|
+
pfr = getattr(args, "parent_folder_rid", None)
|
|
151
|
+
|
|
152
|
+
if resource == "dataset":
|
|
153
|
+
if operation == "create":
|
|
154
|
+
return await client.create(
|
|
155
|
+
name=nm, parent_folder_rid=pfr, request_timeout=timeout
|
|
156
|
+
)
|
|
157
|
+
if operation == "get":
|
|
158
|
+
return await client.get(dataset_rid=dr, request_timeout=timeout)
|
|
159
|
+
if operation == "get_health_check_reports":
|
|
160
|
+
return await client.get_health_check_reports(
|
|
161
|
+
dataset_rid=dr, branch_name=bn, request_timeout=timeout
|
|
162
|
+
)
|
|
163
|
+
if operation == "get_health_checks":
|
|
164
|
+
return await client.get_health_checks(
|
|
165
|
+
dataset_rid=dr, branch_name=bn, request_timeout=timeout
|
|
166
|
+
)
|
|
167
|
+
if operation == "get_schedules":
|
|
168
|
+
return await client.get_schedules(
|
|
169
|
+
dataset_rid=dr,
|
|
170
|
+
branch_name=bn,
|
|
171
|
+
page_size=ps,
|
|
172
|
+
page_token=pt,
|
|
173
|
+
request_timeout=timeout,
|
|
174
|
+
)
|
|
175
|
+
if operation == "get_schema":
|
|
176
|
+
return await client.get_schema(
|
|
177
|
+
dataset_rid=dr, branch_name=bn, request_timeout=timeout
|
|
178
|
+
)
|
|
179
|
+
if operation == "get_schema_batch":
|
|
180
|
+
rids_arg = getattr(args, "dataset_rids", None)
|
|
181
|
+
rids = json.loads(rids_arg) if isinstance(rids_arg, str) else rids_arg
|
|
182
|
+
return await client.get_schema_batch(
|
|
183
|
+
dataset_rids=rids, request_timeout=timeout
|
|
184
|
+
)
|
|
185
|
+
if operation == "jobs":
|
|
186
|
+
return await client.jobs(
|
|
187
|
+
dataset_rid=dr, page_size=ps, page_token=pt, request_timeout=timeout
|
|
188
|
+
)
|
|
189
|
+
if operation == "put_schema":
|
|
190
|
+
schema_arg = getattr(args, "schema", None)
|
|
191
|
+
schema = (
|
|
192
|
+
json.loads(schema_arg) if isinstance(schema_arg, str) else schema_arg
|
|
193
|
+
)
|
|
194
|
+
return await client.put_schema(
|
|
195
|
+
dataset_rid=dr, schema=schema, branch_name=bn, request_timeout=timeout
|
|
196
|
+
)
|
|
197
|
+
if operation == "read_table":
|
|
198
|
+
return await client.read_table(
|
|
199
|
+
dataset_rid=dr,
|
|
200
|
+
branch_name=bn,
|
|
201
|
+
page_size=ps,
|
|
202
|
+
page_token=pt,
|
|
203
|
+
request_timeout=timeout,
|
|
204
|
+
)
|
|
205
|
+
if operation == "transactions":
|
|
206
|
+
return await client.transactions(
|
|
207
|
+
dataset_rid=dr, page_size=ps, page_token=pt, request_timeout=timeout
|
|
208
|
+
)
|
|
209
|
+
elif resource == "branch":
|
|
210
|
+
if operation == "create":
|
|
211
|
+
return await client.create(
|
|
212
|
+
dataset_rid=dr, name=nm, transaction_rid=tr, request_timeout=timeout
|
|
213
|
+
)
|
|
214
|
+
if operation == "delete":
|
|
215
|
+
return await client.delete(
|
|
216
|
+
dataset_rid=dr, branch_name=bn, request_timeout=timeout
|
|
217
|
+
)
|
|
218
|
+
if operation == "get":
|
|
219
|
+
return await client.get(
|
|
220
|
+
dataset_rid=dr, branch_name=bn, request_timeout=timeout
|
|
221
|
+
)
|
|
222
|
+
if operation == "list":
|
|
223
|
+
return await client.list(
|
|
224
|
+
dataset_rid=dr, page_size=ps, page_token=pt, request_timeout=timeout
|
|
225
|
+
)
|
|
226
|
+
if operation == "transactions":
|
|
227
|
+
return await client.transactions(
|
|
228
|
+
dataset_rid=dr,
|
|
229
|
+
branch_name=bn,
|
|
230
|
+
page_size=ps,
|
|
231
|
+
page_token=pt,
|
|
232
|
+
request_timeout=timeout,
|
|
233
|
+
)
|
|
234
|
+
elif resource == "file":
|
|
235
|
+
if operation == "content":
|
|
236
|
+
return await client.content(
|
|
237
|
+
dataset_rid=dr,
|
|
238
|
+
file_path=fp,
|
|
239
|
+
branch_name=bn,
|
|
240
|
+
end_transaction_rid=getattr(args, "end_transaction_rid", None),
|
|
241
|
+
start_transaction_rid=getattr(args, "start_transaction_rid", None),
|
|
242
|
+
request_timeout=timeout,
|
|
243
|
+
)
|
|
244
|
+
if operation == "delete":
|
|
245
|
+
return await client.delete(
|
|
246
|
+
dataset_rid=dr,
|
|
247
|
+
file_path=fp,
|
|
248
|
+
transaction_rid=tr,
|
|
249
|
+
request_timeout=timeout,
|
|
250
|
+
)
|
|
251
|
+
if operation == "get":
|
|
252
|
+
return await client.get(
|
|
253
|
+
dataset_rid=dr,
|
|
254
|
+
file_path=fp,
|
|
255
|
+
transaction_rid=tr,
|
|
256
|
+
request_timeout=timeout,
|
|
257
|
+
)
|
|
258
|
+
if operation == "list":
|
|
259
|
+
return await client.list(
|
|
260
|
+
dataset_rid=dr,
|
|
261
|
+
transaction_rid=tr,
|
|
262
|
+
page_size=ps,
|
|
263
|
+
page_token=pt,
|
|
264
|
+
request_timeout=timeout,
|
|
265
|
+
)
|
|
266
|
+
if operation == "upload":
|
|
267
|
+
if fp is None:
|
|
268
|
+
raise ValueError("file_path is required for upload operation")
|
|
269
|
+
with open(fp, "rb") as fobj:
|
|
270
|
+
data = fobj.read()
|
|
271
|
+
return await client.upload(
|
|
272
|
+
dataset_rid=dr,
|
|
273
|
+
file_path=fp,
|
|
274
|
+
content=data,
|
|
275
|
+
transaction_rid=tr,
|
|
276
|
+
request_timeout=timeout,
|
|
277
|
+
)
|
|
278
|
+
elif resource == "transaction":
|
|
279
|
+
if operation == "abort":
|
|
280
|
+
return await client.abort(
|
|
281
|
+
dataset_rid=dr, transaction_rid=tr, request_timeout=timeout
|
|
282
|
+
)
|
|
283
|
+
if operation == "build":
|
|
284
|
+
return await client.build(
|
|
285
|
+
dataset_rid=dr, transaction_rid=tr, request_timeout=timeout
|
|
286
|
+
)
|
|
287
|
+
if operation == "commit":
|
|
288
|
+
return await client.commit(
|
|
289
|
+
dataset_rid=dr, transaction_rid=tr, request_timeout=timeout
|
|
290
|
+
)
|
|
291
|
+
if operation == "create":
|
|
292
|
+
return await client.create(
|
|
293
|
+
dataset_rid=dr, branch_name=bn, request_timeout=timeout
|
|
294
|
+
)
|
|
295
|
+
if operation == "get":
|
|
296
|
+
return await client.get(
|
|
297
|
+
dataset_rid=dr, transaction_rid=tr, request_timeout=timeout
|
|
298
|
+
)
|
|
299
|
+
if operation == "job":
|
|
300
|
+
return await client.job(
|
|
301
|
+
dataset_rid=dr, transaction_rid=tr, request_timeout=timeout
|
|
302
|
+
)
|
|
303
|
+
elif resource == "view":
|
|
304
|
+
bd_arg = getattr(args, "backing_datasets", None)
|
|
305
|
+
bd = json.loads(bd_arg) if isinstance(bd_arg, str) else bd_arg
|
|
306
|
+
pk_arg = getattr(args, "primary_key", None)
|
|
307
|
+
pk = json.loads(pk_arg) if isinstance(pk_arg, str) else pk_arg
|
|
308
|
+
if operation == "add_backing_datasets":
|
|
309
|
+
return await client.add_backing_datasets(
|
|
310
|
+
view_dataset_rid=vr,
|
|
311
|
+
backing_datasets=bd,
|
|
312
|
+
branch=br,
|
|
313
|
+
request_timeout=timeout,
|
|
314
|
+
)
|
|
315
|
+
if operation == "add_primary_key":
|
|
316
|
+
return await client.add_primary_key(
|
|
317
|
+
view_dataset_rid=vr, primary_key=pk, branch=br, request_timeout=timeout
|
|
318
|
+
)
|
|
319
|
+
if operation == "create":
|
|
320
|
+
return await client.create(
|
|
321
|
+
name=nm,
|
|
322
|
+
parent_folder_rid=pfr,
|
|
323
|
+
backing_datasets=bd,
|
|
324
|
+
request_timeout=timeout,
|
|
325
|
+
)
|
|
326
|
+
if operation == "get":
|
|
327
|
+
return await client.get(
|
|
328
|
+
view_dataset_rid=vr, branch=br, request_timeout=timeout
|
|
329
|
+
)
|
|
330
|
+
if operation == "remove_backing_datasets":
|
|
331
|
+
return await client.remove_backing_datasets(
|
|
332
|
+
view_dataset_rid=vr,
|
|
333
|
+
backing_datasets=bd,
|
|
334
|
+
branch=br,
|
|
335
|
+
request_timeout=timeout,
|
|
336
|
+
)
|
|
337
|
+
if operation == "replace_backing_datasets":
|
|
338
|
+
return await client.replace_backing_datasets(
|
|
339
|
+
view_dataset_rid=vr,
|
|
340
|
+
backing_datasets=bd,
|
|
341
|
+
branch=br,
|
|
342
|
+
request_timeout=timeout,
|
|
343
|
+
)
|
|
344
|
+
raise ValueError(f"Unknown operation: {resource}.{operation}")
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
# Operations that return a paginated response and accept page_size/page_token
|
|
348
|
+
# (FR-PAG-3). For these, the CLI wraps the SDK call with PaginationHelper to
|
|
349
|
+
# honour --page-size / --page-token / --batch-pages and emit metadata to stderr.
|
|
350
|
+
PAGINATED_OPS = frozenset(
|
|
351
|
+
{
|
|
352
|
+
("dataset", "get_schedules"),
|
|
353
|
+
("dataset", "jobs"),
|
|
354
|
+
("dataset", "read_table"),
|
|
355
|
+
("dataset", "transactions"),
|
|
356
|
+
("branch", "list"),
|
|
357
|
+
("branch", "transactions"),
|
|
358
|
+
("file", "list"),
|
|
359
|
+
("transaction", "transactions"),
|
|
360
|
+
}
|
|
361
|
+
)
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def _is_paginated(resource: str, operation: str) -> bool:
|
|
365
|
+
"""Return True if the operation supports pagination (FR-PAG-3)."""
|
|
366
|
+
return (resource, operation) in PAGINATED_OPS
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
async def _invoke_paginated(
|
|
370
|
+
resource: str,
|
|
371
|
+
operation: str,
|
|
372
|
+
client: Any,
|
|
373
|
+
args: argparse.Namespace,
|
|
374
|
+
timeout: int | None,
|
|
375
|
+
helper: PaginationHelper,
|
|
376
|
+
) -> Any:
|
|
377
|
+
"""Invoke a paginated SDK operation through PaginationHelper.
|
|
378
|
+
|
|
379
|
+
Builds a single-page callable that delegates to ``_invoke`` (which already
|
|
380
|
+
forwards page_size/page_token) and feeds it to ``helper.paginate`` so the
|
|
381
|
+
batch aggregation, max-cap enforcement, and token propagation behave per
|
|
382
|
+
FR-PAG-1/2/4/5. The helper's metadata is emitted to stderr by the caller.
|
|
383
|
+
|
|
384
|
+
The base pagination kwargs are extracted from the args namespace so they
|
|
385
|
+
are NOT doubly passed (the SDK call receives them inside ``_invoke``).
|
|
386
|
+
"""
|
|
387
|
+
|
|
388
|
+
async def _single_page(**page_kwargs: Any) -> Any:
|
|
389
|
+
# Materialize a fresh namespace so page tokens from the helper override
|
|
390
|
+
# the CLI-supplied values without mutating the original args.
|
|
391
|
+
paged_args = argparse.Namespace(**vars(args))
|
|
392
|
+
paged_args.page_size = page_kwargs.get(
|
|
393
|
+
"page_size", getattr(args, "page_size", None)
|
|
394
|
+
)
|
|
395
|
+
paged_args.page_token = page_kwargs.get("page_token", None)
|
|
396
|
+
return await _invoke(resource, operation, client, paged_args, timeout)
|
|
397
|
+
|
|
398
|
+
return await helper.paginate(_single_page)
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
OP_MAP = {
|
|
402
|
+
"dataset": {
|
|
403
|
+
"get-health-check-reports": "get_health_check_reports",
|
|
404
|
+
"get-health-checks": "get_health_checks",
|
|
405
|
+
"get-schedules": "get_schedules",
|
|
406
|
+
"get-schema": "get_schema",
|
|
407
|
+
"get-schema-batch": "get_schema_batch",
|
|
408
|
+
"put-schema": "put_schema",
|
|
409
|
+
"read-table": "read_table",
|
|
410
|
+
},
|
|
411
|
+
"view": {
|
|
412
|
+
"add-backing-datasets": "add_backing_datasets",
|
|
413
|
+
"add-primary-key": "add_primary_key",
|
|
414
|
+
"remove-backing-datasets": "remove_backing_datasets",
|
|
415
|
+
"replace-backing-datasets": "replace_backing_datasets",
|
|
416
|
+
},
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def _resolve(resource: str, op: str) -> str:
|
|
421
|
+
"""Resolve kebab-case operation name to snake_case."""
|
|
422
|
+
mapping = OP_MAP.get(resource, {})
|
|
423
|
+
return mapping.get(op, op.replace("-", "_"))
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _common_parser() -> argparse.ArgumentParser:
|
|
427
|
+
"""Build the shared argparse parent with global options.
|
|
428
|
+
|
|
429
|
+
Every leaf operation subparser inherits these arguments via ``parents``,
|
|
430
|
+
so options like ``--timeout`` may appear after the operation positional.
|
|
431
|
+
"""
|
|
432
|
+
common = argparse.ArgumentParser(add_help=False)
|
|
433
|
+
common.add_argument("--timeout", type=int, default=None)
|
|
434
|
+
common.add_argument("--format", choices=["json", "toon", "auto"], default="auto")
|
|
435
|
+
common.add_argument("--pretty", action="store_true")
|
|
436
|
+
common.add_argument("--page-size", type=int, default=None, dest="page_size")
|
|
437
|
+
common.add_argument("--page-token", type=str, default=None, dest="page_token")
|
|
438
|
+
common.add_argument("--batch-pages", type=int, default=None, dest="batch_pages")
|
|
439
|
+
return common
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def _add_operation(
|
|
443
|
+
subparsers: Any,
|
|
444
|
+
name: str,
|
|
445
|
+
*arguments: tuple[tuple[str, ...], dict[str, Any]],
|
|
446
|
+
) -> argparse.ArgumentParser:
|
|
447
|
+
operation_parser = subparsers.add_parser(name, parents=[_common_parser()])
|
|
448
|
+
for flags, kwargs in arguments:
|
|
449
|
+
operation_parser.add_argument(*flags, **kwargs)
|
|
450
|
+
return cast(argparse.ArgumentParser, operation_parser)
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
454
|
+
"""Build argparse parser with all 33 operations."""
|
|
455
|
+
parser = argparse.ArgumentParser(
|
|
456
|
+
prog="pal_found_datasets_cli",
|
|
457
|
+
description="Foundry Datasets CLI - 33 operations across 5 resource clients",
|
|
458
|
+
)
|
|
459
|
+
subparsers = parser.add_subparsers(dest="resource", help="Resource type")
|
|
460
|
+
|
|
461
|
+
ds = subparsers.add_parser("dataset")
|
|
462
|
+
ds_sub = ds.add_subparsers(dest="operation")
|
|
463
|
+
_add_operation(
|
|
464
|
+
ds_sub,
|
|
465
|
+
"create",
|
|
466
|
+
(("--name",), {"required": True}),
|
|
467
|
+
(("--parent-folder-rid",), {"required": True, "dest": "parent_folder_rid"}),
|
|
468
|
+
)
|
|
469
|
+
_add_operation(ds_sub, "get", (("dataset_rid",), {}))
|
|
470
|
+
for operation_name in (
|
|
471
|
+
"get-health-check-reports",
|
|
472
|
+
"get-health-checks",
|
|
473
|
+
"get-schedules",
|
|
474
|
+
"get-schema",
|
|
475
|
+
"read-table",
|
|
476
|
+
):
|
|
477
|
+
_add_operation(
|
|
478
|
+
ds_sub,
|
|
479
|
+
operation_name,
|
|
480
|
+
(("dataset_rid",), {}),
|
|
481
|
+
(("--branch-name",), {"default": None, "dest": "branch_name"}),
|
|
482
|
+
)
|
|
483
|
+
_add_operation(
|
|
484
|
+
ds_sub,
|
|
485
|
+
"get-schema-batch",
|
|
486
|
+
(("--dataset-r",), {"required": True, "dest": "dataset_rids"}),
|
|
487
|
+
)
|
|
488
|
+
_add_operation(ds_sub, "jobs", (("dataset_rid",), {}))
|
|
489
|
+
_add_operation(
|
|
490
|
+
ds_sub,
|
|
491
|
+
"put-schema",
|
|
492
|
+
(("dataset_rid",), {}),
|
|
493
|
+
(("--schema",), {"required": True}),
|
|
494
|
+
(("--branch-name",), {"default": None, "dest": "branch_name"}),
|
|
495
|
+
)
|
|
496
|
+
_add_operation(ds_sub, "transactions", (("dataset_rid",), {}))
|
|
497
|
+
|
|
498
|
+
br = subparsers.add_parser("branch")
|
|
499
|
+
br_sub = br.add_subparsers(dest="operation")
|
|
500
|
+
_add_operation(
|
|
501
|
+
br_sub,
|
|
502
|
+
"create",
|
|
503
|
+
(("dataset_rid",), {}),
|
|
504
|
+
(("--name",), {"required": True}),
|
|
505
|
+
(("--transaction-rid",), {"default": None, "dest": "transaction_rid"}),
|
|
506
|
+
)
|
|
507
|
+
_add_operation(
|
|
508
|
+
br_sub,
|
|
509
|
+
"delete",
|
|
510
|
+
(("dataset_rid",), {}),
|
|
511
|
+
(("--branch-name",), {"required": True, "dest": "branch_name"}),
|
|
512
|
+
)
|
|
513
|
+
for operation_name in ("get", "transactions"):
|
|
514
|
+
_add_operation(
|
|
515
|
+
br_sub,
|
|
516
|
+
operation_name,
|
|
517
|
+
(("dataset_rid",), {}),
|
|
518
|
+
(("--branch-name",), {"default": None, "dest": "branch_name"}),
|
|
519
|
+
)
|
|
520
|
+
_add_operation(br_sub, "list", (("dataset_rid",), {}))
|
|
521
|
+
|
|
522
|
+
fl = subparsers.add_parser("file")
|
|
523
|
+
fl_sub = fl.add_subparsers(dest="operation")
|
|
524
|
+
_add_operation(
|
|
525
|
+
fl_sub,
|
|
526
|
+
"content",
|
|
527
|
+
(("dataset_rid",), {}),
|
|
528
|
+
(("--file-path",), {"required": True, "dest": "file_path"}),
|
|
529
|
+
(("--branch-name",), {"default": None, "dest": "branch_name"}),
|
|
530
|
+
(("--end-transaction-rid",), {"default": None, "dest": "end_transaction_rid"}),
|
|
531
|
+
(
|
|
532
|
+
("--start-transaction-rid",),
|
|
533
|
+
{"default": None, "dest": "start_transaction_rid"},
|
|
534
|
+
),
|
|
535
|
+
)
|
|
536
|
+
for operation_name in ("delete", "get", "upload"):
|
|
537
|
+
_add_operation(
|
|
538
|
+
fl_sub,
|
|
539
|
+
operation_name,
|
|
540
|
+
(("dataset_rid",), {}),
|
|
541
|
+
(("--file-path",), {"required": True, "dest": "file_path"}),
|
|
542
|
+
(("--transaction-rid",), {"default": None, "dest": "transaction_rid"}),
|
|
543
|
+
)
|
|
544
|
+
_add_operation(
|
|
545
|
+
fl_sub,
|
|
546
|
+
"list",
|
|
547
|
+
(("dataset_rid",), {}),
|
|
548
|
+
(("--transaction-rid",), {"default": None, "dest": "transaction_rid"}),
|
|
549
|
+
)
|
|
550
|
+
|
|
551
|
+
tx = subparsers.add_parser("transaction")
|
|
552
|
+
tx_sub = tx.add_subparsers(dest="operation")
|
|
553
|
+
for operation_name in ("abort", "build", "commit", "get", "job"):
|
|
554
|
+
_add_operation(
|
|
555
|
+
tx_sub,
|
|
556
|
+
operation_name,
|
|
557
|
+
(("dataset_rid",), {}),
|
|
558
|
+
(("--transaction-rid",), {"required": True, "dest": "transaction_rid"}),
|
|
559
|
+
)
|
|
560
|
+
_add_operation(
|
|
561
|
+
tx_sub,
|
|
562
|
+
"create",
|
|
563
|
+
(("dataset_rid",), {}),
|
|
564
|
+
(("--branch-name",), {"default": None, "dest": "branch_name"}),
|
|
565
|
+
)
|
|
566
|
+
|
|
567
|
+
vw = subparsers.add_parser("view")
|
|
568
|
+
vw_sub = vw.add_subparsers(dest="operation")
|
|
569
|
+
for operation_name in (
|
|
570
|
+
"add-backing-datasets",
|
|
571
|
+
"remove-backing-datasets",
|
|
572
|
+
"replace-backing-datasets",
|
|
573
|
+
):
|
|
574
|
+
_add_operation(
|
|
575
|
+
vw_sub,
|
|
576
|
+
operation_name,
|
|
577
|
+
(("--view-dataset-rid",), {"required": True, "dest": "view_dataset_rid"}),
|
|
578
|
+
(("--backing-datasets",), {"required": True, "dest": "backing_datasets"}),
|
|
579
|
+
(("--branch",), {"default": None}),
|
|
580
|
+
)
|
|
581
|
+
_add_operation(
|
|
582
|
+
vw_sub,
|
|
583
|
+
"add-primary-key",
|
|
584
|
+
(("--view-dataset-rid",), {"required": True, "dest": "view_dataset_rid"}),
|
|
585
|
+
(("--primary-key",), {"required": True, "dest": "primary_key"}),
|
|
586
|
+
(("--branch",), {"default": None}),
|
|
587
|
+
)
|
|
588
|
+
_add_operation(
|
|
589
|
+
vw_sub,
|
|
590
|
+
"create",
|
|
591
|
+
(("--name",), {"required": True}),
|
|
592
|
+
(("--parent-folder-rid",), {"required": True, "dest": "parent_folder_rid"}),
|
|
593
|
+
(("--backing-datasets",), {"default": None, "dest": "backing_datasets"}),
|
|
594
|
+
)
|
|
595
|
+
_add_operation(
|
|
596
|
+
vw_sub,
|
|
597
|
+
"get",
|
|
598
|
+
(("--view-dataset-rid",), {"required": True, "dest": "view_dataset_rid"}),
|
|
599
|
+
(("--branch",), {"default": None}),
|
|
600
|
+
)
|
|
601
|
+
|
|
602
|
+
return parser
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
async def async_main() -> int:
|
|
606
|
+
"""Main entry point: async, returns exit code."""
|
|
607
|
+
parser = build_parser()
|
|
608
|
+
args = parser.parse_args()
|
|
609
|
+
|
|
610
|
+
if not args.resource:
|
|
611
|
+
parser.print_help()
|
|
612
|
+
return EXIT_USER_INPUT
|
|
613
|
+
|
|
614
|
+
# WARNING-3 fix: validate operation is provided
|
|
615
|
+
if not getattr(args, "operation", None):
|
|
616
|
+
parser.print_help()
|
|
617
|
+
return EXIT_USER_INPUT
|
|
618
|
+
|
|
619
|
+
# Load configuration (ADR-006)
|
|
620
|
+
cfg = ConfigLoader()
|
|
621
|
+
cfg.load()
|
|
622
|
+
|
|
623
|
+
# Initialize logging (ADR-005)
|
|
624
|
+
LogSetup.configure(log_level=cfg.log_level)
|
|
625
|
+
|
|
626
|
+
# Resolve operation name
|
|
627
|
+
resource = args.resource
|
|
628
|
+
operation = _resolve(resource, args.operation)
|
|
629
|
+
logger.info(
|
|
630
|
+
"Executing operation", extra={"resource": resource, "operation": operation}
|
|
631
|
+
)
|
|
632
|
+
|
|
633
|
+
# Access control check (ADR-007)
|
|
634
|
+
guard = AccessControlGuard(cfg, "DATASETS")
|
|
635
|
+
try:
|
|
636
|
+
guard.check(resource, operation)
|
|
637
|
+
except AccessControlError as exc:
|
|
638
|
+
logger.warning("Access control denied", extra={"error": str(exc)})
|
|
639
|
+
serializer = ErrorSerializer()
|
|
640
|
+
serializer.serialize(exc)
|
|
641
|
+
return EXIT_ACCESS_CONTROL
|
|
642
|
+
|
|
643
|
+
# Create SDK client
|
|
644
|
+
try:
|
|
645
|
+
client = _get_client(cfg, resource)
|
|
646
|
+
except Exception as exc:
|
|
647
|
+
logger.error("Failed to create client", extra={"error": str(exc)})
|
|
648
|
+
serializer = ErrorSerializer()
|
|
649
|
+
serializer.serialize(exc)
|
|
650
|
+
return EXIT_CONFIGURATION
|
|
651
|
+
|
|
652
|
+
# WARNING-2 fix: resolve timeout from CLI flag or config default.
|
|
653
|
+
# cfg.timeout_s is the project-wide default (ADR-002). CLI flag wins.
|
|
654
|
+
timeout = getattr(args, "timeout", None) or getattr(cfg, "timeout_s", None)
|
|
655
|
+
|
|
656
|
+
# WARNING-1 fix: apply RetryHandler for transient error recovery (ADR-002).
|
|
657
|
+
retry_handler = RetryHandler()
|
|
658
|
+
|
|
659
|
+
# Build PaginationHelper when the operation is paginated (FR-PAG).
|
|
660
|
+
# --page-size / --page-token / --batch-pages are parsed by _common_parser.
|
|
661
|
+
helper: PaginationHelper | None = None
|
|
662
|
+
if _is_paginated(resource, operation):
|
|
663
|
+
helper = PaginationHelper(
|
|
664
|
+
page_size=getattr(args, "page_size", None),
|
|
665
|
+
page_token=getattr(args, "page_token", None),
|
|
666
|
+
batch_pages=getattr(args, "batch_pages", None),
|
|
667
|
+
)
|
|
668
|
+
|
|
669
|
+
# Invoke operation
|
|
670
|
+
try:
|
|
671
|
+
if helper is not None:
|
|
672
|
+
result = await retry_handler.execute(
|
|
673
|
+
_invoke_paginated,
|
|
674
|
+
resource,
|
|
675
|
+
operation,
|
|
676
|
+
client,
|
|
677
|
+
args,
|
|
678
|
+
timeout,
|
|
679
|
+
helper,
|
|
680
|
+
)
|
|
681
|
+
else:
|
|
682
|
+
result = await retry_handler.execute(
|
|
683
|
+
_invoke, resource, operation, client, args, timeout
|
|
684
|
+
)
|
|
685
|
+
formatter = OutputFormatter(
|
|
686
|
+
format_setting=getattr(args, "format", "auto"),
|
|
687
|
+
pretty=getattr(args, "pretty", False),
|
|
688
|
+
)
|
|
689
|
+
output = formatter.format(_model_to_dict(result))
|
|
690
|
+
print(output)
|
|
691
|
+
# Emit pagination metadata to stderr after stdout (ADR-005, FR-PAG-2).
|
|
692
|
+
if helper is not None:
|
|
693
|
+
helper.emit_metadata()
|
|
694
|
+
return EXIT_SUCCESS
|
|
695
|
+
except AccessControlError as exc:
|
|
696
|
+
logger.warning("Access control denied", extra={"error": str(exc)})
|
|
697
|
+
serializer = ErrorSerializer()
|
|
698
|
+
serializer.serialize(exc)
|
|
699
|
+
return EXIT_ACCESS_CONTROL
|
|
700
|
+
except PermissionError as exc:
|
|
701
|
+
serializer = ErrorSerializer()
|
|
702
|
+
serializer.serialize(exc)
|
|
703
|
+
return EXIT_PERMISSION_DENIED
|
|
704
|
+
except FileNotFoundError as exc:
|
|
705
|
+
serializer = ErrorSerializer()
|
|
706
|
+
serializer.serialize(exc)
|
|
707
|
+
return EXIT_NOT_FOUND
|
|
708
|
+
except TimeoutError as exc:
|
|
709
|
+
serializer = ErrorSerializer()
|
|
710
|
+
serializer.serialize(exc)
|
|
711
|
+
return EXIT_TIMEOUT
|
|
712
|
+
except OSError as exc:
|
|
713
|
+
serializer = ErrorSerializer()
|
|
714
|
+
exit_code = serializer.serialize(exc)
|
|
715
|
+
# Check if it looks like a rate limit (errno 11 or EAGAIN)
|
|
716
|
+
if getattr(exc, "errno", None) in (11, 115):
|
|
717
|
+
return EXIT_RATE_LIMIT
|
|
718
|
+
return exit_code
|
|
719
|
+
except Exception as exc:
|
|
720
|
+
serializer = ErrorSerializer()
|
|
721
|
+
serializer.serialize(exc)
|
|
722
|
+
return EXIT_SERVER_ERROR
|
|
723
|
+
|
|
724
|
+
|
|
725
|
+
def main() -> int:
|
|
726
|
+
"""Run the async CLI from the console script entry point."""
|
|
727
|
+
return asyncio.run(async_main())
|
|
728
|
+
|
|
729
|
+
|
|
730
|
+
if __name__ == "__main__":
|
|
731
|
+
exit_code = main()
|
|
732
|
+
sys.exit(exit_code)
|