tinybird 0.0.1.dev234__py3-none-any.whl → 0.0.1.dev235__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/check_pypi.py +3 -8
- tinybird/tb/cli.py +0 -6
- tinybird/tb/client.py +314 -340
- tinybird/tb/config.py +4 -5
- tinybird/tb/modules/build.py +21 -24
- tinybird/tb/modules/cicd.py +2 -2
- tinybird/tb/modules/cli.py +18 -28
- tinybird/tb/modules/common.py +123 -138
- tinybird/tb/modules/config.py +2 -4
- tinybird/tb/modules/connection.py +21 -26
- tinybird/tb/modules/copy.py +7 -9
- tinybird/tb/modules/create.py +18 -21
- tinybird/tb/modules/datafile/build.py +39 -39
- tinybird/tb/modules/datafile/build_common.py +9 -9
- tinybird/tb/modules/datafile/build_datasource.py +24 -24
- tinybird/tb/modules/datafile/build_pipe.py +11 -13
- tinybird/tb/modules/datafile/diff.py +12 -12
- tinybird/tb/modules/datafile/format_datasource.py +5 -5
- tinybird/tb/modules/datafile/format_pipe.py +6 -6
- tinybird/tb/modules/datafile/playground.py +42 -42
- tinybird/tb/modules/datafile/pull.py +24 -26
- tinybird/tb/modules/datasource.py +42 -56
- tinybird/tb/modules/endpoint.py +14 -19
- tinybird/tb/modules/info.py +14 -15
- tinybird/tb/modules/infra.py +43 -48
- tinybird/tb/modules/job.py +7 -10
- tinybird/tb/modules/local.py +6 -12
- tinybird/tb/modules/local_common.py +4 -4
- tinybird/tb/modules/login.py +9 -10
- tinybird/tb/modules/materialization.py +7 -10
- tinybird/tb/modules/mock.py +8 -9
- tinybird/tb/modules/open.py +1 -3
- tinybird/tb/modules/pipe.py +2 -4
- tinybird/tb/modules/secret.py +12 -16
- tinybird/tb/modules/shell.py +7 -20
- tinybird/tb/modules/sink.py +6 -8
- tinybird/tb/modules/test.py +9 -14
- tinybird/tb/modules/tinyunit/tinyunit.py +3 -3
- tinybird/tb/modules/token.py +16 -24
- tinybird/tb/modules/watch.py +3 -7
- tinybird/tb/modules/workspace.py +26 -37
- tinybird/tb/modules/workspace_members.py +16 -23
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev235.dist-info}/METADATA +1 -1
- tinybird-0.0.1.dev235.dist-info/RECORD +89 -0
- tinybird-0.0.1.dev234.dist-info/RECORD +0 -89
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev235.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev235.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev235.dist-info}/top_level.txt +0 -0
tinybird/tb/client.py
CHANGED
|
@@ -1,20 +1,18 @@
|
|
|
1
|
-
import asyncio
|
|
2
1
|
import json
|
|
3
2
|
import logging
|
|
4
3
|
import os
|
|
5
4
|
import ssl
|
|
5
|
+
import time
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Tuple, Union
|
|
8
8
|
from urllib.parse import quote, urlencode
|
|
9
9
|
|
|
10
|
-
import aiofiles
|
|
11
10
|
import requests
|
|
12
11
|
import requests.adapters
|
|
13
12
|
from requests import Response
|
|
14
13
|
from urllib3 import Retry
|
|
15
14
|
|
|
16
15
|
from tinybird.ch_utils.constants import COPY_ENABLED_TABLE_FUNCTIONS
|
|
17
|
-
from tinybird.syncasync import sync_to_async
|
|
18
16
|
from tinybird.tb.modules.telemetry import add_telemetry_event
|
|
19
17
|
|
|
20
18
|
HOST = "https://api.tinybird.co"
|
|
@@ -108,7 +106,7 @@ class TinyB:
|
|
|
108
106
|
self.env = env
|
|
109
107
|
self.staging = staging
|
|
110
108
|
|
|
111
|
-
|
|
109
|
+
def _req_raw(
|
|
112
110
|
self,
|
|
113
111
|
endpoint: str,
|
|
114
112
|
data=None,
|
|
@@ -143,25 +141,15 @@ class TinyB:
|
|
|
143
141
|
session.mount("http://", requests.adapters.HTTPAdapter(max_retries=retry))
|
|
144
142
|
if method == "POST":
|
|
145
143
|
if files:
|
|
146
|
-
response =
|
|
147
|
-
url, files=files, verify=verify_ssl, **kwargs
|
|
148
|
-
)
|
|
144
|
+
response = session.post(url, files=files, verify=verify_ssl, **kwargs)
|
|
149
145
|
else:
|
|
150
|
-
response =
|
|
151
|
-
url, data=data, verify=verify_ssl, **kwargs
|
|
152
|
-
)
|
|
146
|
+
response = session.post(url, data=data, verify=verify_ssl, **kwargs)
|
|
153
147
|
elif method == "PUT":
|
|
154
|
-
response =
|
|
155
|
-
url, data=data, verify=verify_ssl, **kwargs
|
|
156
|
-
)
|
|
148
|
+
response = session.put(url, data=data, verify=verify_ssl, **kwargs)
|
|
157
149
|
elif method == "DELETE":
|
|
158
|
-
response =
|
|
159
|
-
url, data=data, verify=verify_ssl, **kwargs
|
|
160
|
-
)
|
|
150
|
+
response = session.delete(url, data=data, verify=verify_ssl, **kwargs)
|
|
161
151
|
else:
|
|
162
|
-
response =
|
|
163
|
-
url, verify=verify_ssl, **kwargs
|
|
164
|
-
)
|
|
152
|
+
response = session.get(url, verify=verify_ssl, **kwargs)
|
|
165
153
|
except Exception as e:
|
|
166
154
|
raise e
|
|
167
155
|
|
|
@@ -183,7 +171,7 @@ class TinyB:
|
|
|
183
171
|
|
|
184
172
|
return response
|
|
185
173
|
|
|
186
|
-
|
|
174
|
+
def _req(
|
|
187
175
|
self,
|
|
188
176
|
endpoint: str,
|
|
189
177
|
data=None,
|
|
@@ -194,7 +182,7 @@ class TinyB:
|
|
|
194
182
|
**kwargs,
|
|
195
183
|
):
|
|
196
184
|
token_to_use = use_token if use_token else self.token
|
|
197
|
-
response =
|
|
185
|
+
response = self._req_raw(endpoint, data, files, method, retries, use_token, **kwargs)
|
|
198
186
|
|
|
199
187
|
if response.status_code == 403:
|
|
200
188
|
error = parse_error_response(response)
|
|
@@ -231,18 +219,18 @@ class TinyB:
|
|
|
231
219
|
|
|
232
220
|
return response
|
|
233
221
|
|
|
234
|
-
|
|
235
|
-
response =
|
|
222
|
+
def tokens(self):
|
|
223
|
+
response = self._req("/v0/tokens")
|
|
236
224
|
return response["tokens"]
|
|
237
225
|
|
|
238
|
-
|
|
239
|
-
tokens =
|
|
226
|
+
def get_token_by_name(self, name: str):
|
|
227
|
+
tokens = self.tokens()
|
|
240
228
|
for tk in tokens:
|
|
241
229
|
if tk["name"] == name:
|
|
242
230
|
return tk
|
|
243
231
|
return None
|
|
244
232
|
|
|
245
|
-
|
|
233
|
+
def create_token(
|
|
246
234
|
self, name: str, scope: List[str], origin_code: Optional[str], origin_resource_name_or_id: Optional[str] = None
|
|
247
235
|
):
|
|
248
236
|
origin = origin_code or "C" # == Origins.CUSTOM if none specified
|
|
@@ -256,70 +244,68 @@ class TinyB:
|
|
|
256
244
|
# TODO: We should support sending multiple scopes in the body of the request
|
|
257
245
|
url = f"/v0/tokens?{urlencode(params)}"
|
|
258
246
|
url = url + "&" + "&".join([f"scope={scope}" for scope in scope])
|
|
259
|
-
return
|
|
247
|
+
return self._req(
|
|
260
248
|
url,
|
|
261
249
|
method="POST",
|
|
262
250
|
data="",
|
|
263
251
|
)
|
|
264
252
|
|
|
265
|
-
|
|
253
|
+
def alter_tokens(self, name: str, scopes: List[str]):
|
|
266
254
|
if not scopes:
|
|
267
255
|
return
|
|
268
256
|
scopes_url: str = "&".join([f"scope={scope}" for scope in scopes])
|
|
269
257
|
url = f"/v0/tokens/{name}"
|
|
270
258
|
if len(url + "?" + scopes_url) > TinyB.MAX_GET_LENGTH:
|
|
271
|
-
return
|
|
259
|
+
return self._req(url, method="PUT", data=scopes_url)
|
|
272
260
|
else:
|
|
273
261
|
url = url + "?" + scopes_url
|
|
274
|
-
return
|
|
262
|
+
return self._req(url, method="PUT", data="")
|
|
275
263
|
|
|
276
|
-
|
|
264
|
+
def datasources(self, branch: Optional[str] = None, used_by: bool = False) -> List[Dict[str, Any]]:
|
|
277
265
|
params = {}
|
|
278
266
|
if used_by:
|
|
279
267
|
params["attrs"] = "used_by"
|
|
280
|
-
response =
|
|
268
|
+
response = self._req(f"/v0/datasources?{urlencode(params)}")
|
|
281
269
|
ds = response["datasources"]
|
|
282
270
|
|
|
283
271
|
if branch:
|
|
284
272
|
ds = [x for x in ds if x["name"].startswith(branch)]
|
|
285
273
|
return ds
|
|
286
274
|
|
|
287
|
-
|
|
288
|
-
response =
|
|
275
|
+
def secrets(self) -> List[Dict[str, Any]]:
|
|
276
|
+
response = self._req("/v0/variables")
|
|
289
277
|
return response["variables"]
|
|
290
278
|
|
|
291
|
-
|
|
292
|
-
return
|
|
279
|
+
def get_secret(self, name: str) -> Optional[Dict[str, Any]]:
|
|
280
|
+
return self._req(f"/v0/variables/{name}")
|
|
293
281
|
|
|
294
|
-
|
|
295
|
-
response =
|
|
282
|
+
def create_secret(self, name: str, value: str):
|
|
283
|
+
response = self._req("/v0/variables", method="POST", data={"name": name, "value": value})
|
|
296
284
|
return response
|
|
297
285
|
|
|
298
|
-
|
|
299
|
-
response =
|
|
286
|
+
def update_secret(self, name: str, value: str):
|
|
287
|
+
response = self._req(f"/v0/variables/{name}", method="PUT", data={"value": value})
|
|
300
288
|
return response
|
|
301
289
|
|
|
302
|
-
|
|
303
|
-
response =
|
|
290
|
+
def delete_secret(self, name: str):
|
|
291
|
+
response = self._req(f"/v0/variables/{name}", method="DELETE")
|
|
304
292
|
return response
|
|
305
293
|
|
|
306
|
-
|
|
294
|
+
def get_connections(self, service: Optional[str] = None):
|
|
307
295
|
params = {}
|
|
308
296
|
|
|
309
297
|
if service:
|
|
310
298
|
params["service"] = service
|
|
311
299
|
|
|
312
|
-
response =
|
|
300
|
+
response = self._req(f"/v0/connectors?{urlencode(params)}")
|
|
313
301
|
return response["connectors"]
|
|
314
302
|
|
|
315
|
-
|
|
316
|
-
response =
|
|
303
|
+
def connections(self, connector: Optional[str] = None, skip_bigquery: Optional[bool] = False):
|
|
304
|
+
response = self._req("/v0/connectors")
|
|
317
305
|
connectors = response["connectors"]
|
|
318
306
|
bigquery_connection = None
|
|
319
307
|
if not skip_bigquery:
|
|
320
|
-
bigquery_connection = (
|
|
321
|
-
await self.bigquery_connection() if connector == "bigquery" or connector is None else None
|
|
322
|
-
)
|
|
308
|
+
bigquery_connection = self.bigquery_connection() if connector == "bigquery" or connector is None else None
|
|
323
309
|
connectors = [*connectors, bigquery_connection] if bigquery_connection else connectors
|
|
324
310
|
if connector:
|
|
325
311
|
return [
|
|
@@ -344,13 +330,13 @@ class TinyB:
|
|
|
344
330
|
for c in connectors
|
|
345
331
|
]
|
|
346
332
|
|
|
347
|
-
|
|
348
|
-
bigquery_resources =
|
|
333
|
+
def bigquery_connection(self):
|
|
334
|
+
bigquery_resources = self.list_gcp_resources()
|
|
349
335
|
if len(bigquery_resources) == 0:
|
|
350
336
|
return None
|
|
351
337
|
|
|
352
|
-
gcp_account_details: Dict[str, Any] =
|
|
353
|
-
datasources =
|
|
338
|
+
gcp_account_details: Dict[str, Any] = self.get_gcp_service_account_details()
|
|
339
|
+
datasources = self.datasources()
|
|
354
340
|
bigquery_datasources = [ds["name"] for ds in datasources if ds["type"] == "bigquery"]
|
|
355
341
|
return {
|
|
356
342
|
"id": gcp_account_details["account"].split("@")[0],
|
|
@@ -360,13 +346,13 @@ class TinyB:
|
|
|
360
346
|
"settings": gcp_account_details,
|
|
361
347
|
}
|
|
362
348
|
|
|
363
|
-
|
|
349
|
+
def get_datasource(self, ds_name: str, used_by: bool = False) -> Dict[str, Any]:
|
|
364
350
|
params = {
|
|
365
351
|
"attrs": "used_by" if used_by else "",
|
|
366
352
|
}
|
|
367
|
-
return
|
|
353
|
+
return self._req(f"/v0/datasources/{ds_name}?{urlencode(params)}")
|
|
368
354
|
|
|
369
|
-
|
|
355
|
+
def alter_datasource(
|
|
370
356
|
self,
|
|
371
357
|
ds_name: str,
|
|
372
358
|
new_schema: Optional[str] = None,
|
|
@@ -384,44 +370,44 @@ class TinyB:
|
|
|
384
370
|
params.update({"ttl": ttl})
|
|
385
371
|
if indexes:
|
|
386
372
|
params.update({"indexes": indexes})
|
|
387
|
-
res =
|
|
373
|
+
res = self._req(f"/v0/datasources/{ds_name}/alter", method="POST", data=params)
|
|
388
374
|
|
|
389
375
|
if "Error" in res:
|
|
390
376
|
raise Exception(res["error"])
|
|
391
377
|
|
|
392
378
|
return res
|
|
393
379
|
|
|
394
|
-
|
|
395
|
-
res =
|
|
380
|
+
def update_datasource(self, ds_name: str, data: Dict[str, Any]):
|
|
381
|
+
res = self._req(f"/v0/datasources/{ds_name}", method="PUT", data=data)
|
|
396
382
|
|
|
397
383
|
if "Error" in res:
|
|
398
384
|
raise Exception(res["error"])
|
|
399
385
|
|
|
400
386
|
return res
|
|
401
387
|
|
|
402
|
-
|
|
403
|
-
return
|
|
388
|
+
def pipe_file(self, pipe: str):
|
|
389
|
+
return self._req(f"/v1/pipes/{pipe}.pipe")
|
|
404
390
|
|
|
405
|
-
|
|
406
|
-
return
|
|
391
|
+
def connection_file(self, connection: str):
|
|
392
|
+
return self._req(f"/v0/connectors/{connection}.connection")
|
|
407
393
|
|
|
408
|
-
|
|
394
|
+
def datasource_file(self, datasource: str):
|
|
409
395
|
try:
|
|
410
|
-
return
|
|
396
|
+
return self._req(f"/v0/datasources/{datasource}.datasource")
|
|
411
397
|
except DoesNotExistException:
|
|
412
398
|
raise Exception(f"Data Source {datasource} not found.")
|
|
413
399
|
|
|
414
|
-
|
|
400
|
+
def datasource_analyze(self, url):
|
|
415
401
|
params = {"url": url}
|
|
416
|
-
return
|
|
402
|
+
return self._req(f"/v0/analyze?{urlencode(params)}", method="POST", data="")
|
|
417
403
|
|
|
418
|
-
|
|
419
|
-
return
|
|
404
|
+
def datasource_analyze_file(self, data):
|
|
405
|
+
return self._req("/v0/analyze", method="POST", data=data)
|
|
420
406
|
|
|
421
|
-
|
|
422
|
-
return
|
|
407
|
+
def datasource_create_from_definition(self, parameter_definition: Dict[str, str]):
|
|
408
|
+
return self._req("/v0/datasources", method="POST", data=parameter_definition)
|
|
423
409
|
|
|
424
|
-
|
|
410
|
+
def datasource_create_from_url(
|
|
425
411
|
self,
|
|
426
412
|
table_name: str,
|
|
427
413
|
url: str,
|
|
@@ -440,18 +426,18 @@ class TinyB:
|
|
|
440
426
|
params[option] = "true"
|
|
441
427
|
|
|
442
428
|
req_url = f"/v0/datasources?{urlencode(params, safe='')}"
|
|
443
|
-
res =
|
|
429
|
+
res = self._req(req_url, method="POST", data=b"")
|
|
444
430
|
|
|
445
431
|
if "error" in res:
|
|
446
432
|
raise Exception(res["error"])
|
|
447
433
|
|
|
448
|
-
return
|
|
434
|
+
return self.wait_for_job(res["id"], status_callback, backoff_multiplier=1.5, maximum_backoff_seconds=20)
|
|
449
435
|
|
|
450
|
-
|
|
436
|
+
def datasource_delete(self, datasource_name: str, force: bool = False, dry_run: bool = False):
|
|
451
437
|
params = {"force": "true" if force else "false", "dry_run": "true" if dry_run else "false"}
|
|
452
|
-
return
|
|
438
|
+
return self._req(f"/v0/datasources/{datasource_name}?{urlencode(params)}", method="DELETE")
|
|
453
439
|
|
|
454
|
-
|
|
440
|
+
def datasource_append_data(
|
|
455
441
|
self,
|
|
456
442
|
datasource_name: str,
|
|
457
443
|
file: Union[str, Path],
|
|
@@ -469,33 +455,27 @@ class TinyB:
|
|
|
469
455
|
for option in list(replace_options):
|
|
470
456
|
params[option] = "true"
|
|
471
457
|
|
|
472
|
-
|
|
473
|
-
file_content =
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
method="POST",
|
|
483
|
-
)
|
|
484
|
-
if status_callback:
|
|
485
|
-
status_callback(res)
|
|
486
|
-
|
|
487
|
-
return res
|
|
458
|
+
with open(file, "rb") as content:
|
|
459
|
+
file_content = content.read()
|
|
460
|
+
if format == "csv":
|
|
461
|
+
files = {"csv": ("csv", file_content)}
|
|
462
|
+
else:
|
|
463
|
+
files = {"ndjson": ("ndjson", file_content)}
|
|
464
|
+
res = self._req(f"v0/datasources?{urlencode(params, safe='')}", files=files, method="POST")
|
|
465
|
+
if status_callback:
|
|
466
|
+
status_callback(res)
|
|
467
|
+
return res
|
|
488
468
|
|
|
489
|
-
|
|
490
|
-
return
|
|
469
|
+
def datasource_truncate(self, datasource_name: str):
|
|
470
|
+
return self._req(f"/v0/datasources/{datasource_name}/truncate", method="POST", data="")
|
|
491
471
|
|
|
492
|
-
|
|
472
|
+
def datasource_delete_rows(self, datasource_name: str, delete_condition: str, dry_run: bool = False):
|
|
493
473
|
params = {"delete_condition": delete_condition}
|
|
494
474
|
if dry_run:
|
|
495
475
|
params.update({"dry_run": "true"})
|
|
496
|
-
return
|
|
476
|
+
return self._req(f"/v0/datasources/{datasource_name}/delete", method="POST", data=params)
|
|
497
477
|
|
|
498
|
-
|
|
478
|
+
def datasource_dependencies(
|
|
499
479
|
self, no_deps: bool, match: str, pipe: str, datasource: str, check_for_partial_replace: bool, recursive: bool
|
|
500
480
|
):
|
|
501
481
|
params = {
|
|
@@ -510,45 +490,45 @@ class TinyB:
|
|
|
510
490
|
if datasource:
|
|
511
491
|
params["datasource"] = datasource
|
|
512
492
|
|
|
513
|
-
return
|
|
493
|
+
return self._req(f"/v0/dependencies?{urlencode(params)}", timeout=60)
|
|
514
494
|
|
|
515
|
-
|
|
495
|
+
def datasource_share(self, datasource_id: str, current_workspace_id: str, destination_workspace_id: str):
|
|
516
496
|
params = {"origin_workspace_id": current_workspace_id, "destination_workspace_id": destination_workspace_id}
|
|
517
|
-
return
|
|
497
|
+
return self._req(f"/v0/datasources/{datasource_id}/share", method="POST", data=params)
|
|
518
498
|
|
|
519
|
-
|
|
499
|
+
def datasource_unshare(self, datasource_id: str, current_workspace_id: str, destination_workspace_id: str):
|
|
520
500
|
params = {"origin_workspace_id": current_workspace_id, "destination_workspace_id": destination_workspace_id}
|
|
521
|
-
return
|
|
501
|
+
return self._req(f"/v0/datasources/{datasource_id}/share", method="DELETE", data=params)
|
|
522
502
|
|
|
523
|
-
|
|
524
|
-
return
|
|
503
|
+
def datasource_sync(self, datasource_id: str):
|
|
504
|
+
return self._req(f"/v0/datasources/{datasource_id}/scheduling/runs", method="POST", data="")
|
|
525
505
|
|
|
526
|
-
|
|
527
|
-
response =
|
|
506
|
+
def datasource_scheduling_state(self, datasource_id: str):
|
|
507
|
+
response = self._req(f"/v0/datasources/{datasource_id}/scheduling/state", method="GET")
|
|
528
508
|
return response["state"]
|
|
529
509
|
|
|
530
|
-
|
|
531
|
-
return
|
|
510
|
+
def datasource_scheduling_pause(self, datasource_id: str):
|
|
511
|
+
return self._req(
|
|
532
512
|
f"/v0/datasources/{datasource_id}/scheduling/state",
|
|
533
513
|
method="PUT",
|
|
534
514
|
data='{"state": "paused"}',
|
|
535
515
|
)
|
|
536
516
|
|
|
537
|
-
|
|
538
|
-
return
|
|
517
|
+
def datasource_scheduling_resume(self, datasource_id: str):
|
|
518
|
+
return self._req(
|
|
539
519
|
f"/v0/datasources/{datasource_id}/scheduling/state",
|
|
540
520
|
method="PUT",
|
|
541
521
|
data='{"state": "running"}',
|
|
542
522
|
)
|
|
543
523
|
|
|
544
|
-
|
|
524
|
+
def datasource_exchange(self, datasource_a: str, datasource_b: str):
|
|
545
525
|
payload = {"datasource_a": datasource_a, "datasource_b": datasource_b}
|
|
546
|
-
return
|
|
526
|
+
return self._req("/v0/datasources/exchange", method="POST", data=payload)
|
|
547
527
|
|
|
548
|
-
|
|
549
|
-
return
|
|
528
|
+
def datasource_events(self, datasource_name: str, data: str):
|
|
529
|
+
return self._req(f"/v0/events?name={datasource_name}", method="POST", data=data)
|
|
550
530
|
|
|
551
|
-
|
|
531
|
+
def analyze_pipe_node(
|
|
552
532
|
self, pipe_name: str, node: Dict[str, Any], dry_run: str = "false", datasource_name: Optional[str] = None
|
|
553
533
|
):
|
|
554
534
|
params = {"include_datafile": "true", "dry_run": dry_run, **node.get("params", node)}
|
|
@@ -557,10 +537,10 @@ class TinyB:
|
|
|
557
537
|
node_name = node["params"]["name"] if node.get("params", None) else node["name"]
|
|
558
538
|
if datasource_name:
|
|
559
539
|
params["datasource"] = datasource_name
|
|
560
|
-
response =
|
|
540
|
+
response = self._req(f"/v0/pipes/{pipe_name}/nodes/{node_name}/analysis?{urlencode(params)}")
|
|
561
541
|
return response
|
|
562
542
|
|
|
563
|
-
|
|
543
|
+
def populate_node(
|
|
564
544
|
self,
|
|
565
545
|
pipe_name: str,
|
|
566
546
|
node_name: str,
|
|
@@ -573,27 +553,25 @@ class TinyB:
|
|
|
573
553
|
params.update({"populate_subset": populate_subset})
|
|
574
554
|
if populate_condition:
|
|
575
555
|
params.update({"populate_condition": populate_condition})
|
|
576
|
-
response =
|
|
577
|
-
f"/v0/pipes/{pipe_name}/nodes/{node_name}/population?{urlencode(params)}", method="POST"
|
|
578
|
-
)
|
|
556
|
+
response = self._req(f"/v0/pipes/{pipe_name}/nodes/{node_name}/population?{urlencode(params)}", method="POST")
|
|
579
557
|
return response
|
|
580
558
|
|
|
581
|
-
|
|
559
|
+
def pipes(self, branch=None, dependencies: bool = False, node_attrs=None, attrs=None) -> List[Dict[str, Any]]:
|
|
582
560
|
params = {
|
|
583
561
|
"dependencies": "true" if dependencies else "false",
|
|
584
562
|
"attrs": attrs if attrs else "",
|
|
585
563
|
"node_attrs": node_attrs if node_attrs else "",
|
|
586
564
|
}
|
|
587
|
-
response =
|
|
565
|
+
response = self._req(f"/v0/pipes?{urlencode(params)}")
|
|
588
566
|
pipes = response["pipes"]
|
|
589
567
|
if branch:
|
|
590
568
|
pipes = [x for x in pipes if x["name"].startswith(branch)]
|
|
591
569
|
return pipes
|
|
592
570
|
|
|
593
|
-
|
|
594
|
-
return
|
|
571
|
+
def pipe(self, pipe: str):
|
|
572
|
+
return self._req(f"/v0/pipes/{pipe}")
|
|
595
573
|
|
|
596
|
-
|
|
574
|
+
def pipe_data(
|
|
597
575
|
self,
|
|
598
576
|
pipe_name_or_uid: str,
|
|
599
577
|
sql: Optional[str] = None,
|
|
@@ -607,29 +585,27 @@ class TinyB:
|
|
|
607
585
|
url = f"/v0/pipes/{pipe_name_or_uid}.{format}"
|
|
608
586
|
query_string = urlencode(params)
|
|
609
587
|
if len(url + "?" + query_string) > TinyB.MAX_GET_LENGTH:
|
|
610
|
-
return
|
|
588
|
+
return self._req(f"/v0/pipes/{pipe_name_or_uid}.{format}", method="POST", data=params)
|
|
611
589
|
else:
|
|
612
590
|
url = url + "?" + query_string
|
|
613
|
-
return
|
|
591
|
+
return self._req(url)
|
|
614
592
|
|
|
615
|
-
|
|
616
|
-
return
|
|
617
|
-
f"/v0/pipes?name={pipe_name}&sql={quote(sql, safe='')}", method="POST", data=sql.encode()
|
|
618
|
-
)
|
|
593
|
+
def pipe_create(self, pipe_name: str, sql: str):
|
|
594
|
+
return self._req(f"/v0/pipes?name={pipe_name}&sql={quote(sql, safe='')}", method="POST", data=sql.encode())
|
|
619
595
|
|
|
620
|
-
|
|
621
|
-
return
|
|
596
|
+
def pipe_delete(self, pipe_name: str):
|
|
597
|
+
return self._req(f"/v0/pipes/{pipe_name}", method="DELETE")
|
|
622
598
|
|
|
623
|
-
|
|
624
|
-
return
|
|
599
|
+
def pipe_append_node(self, pipe_name_or_uid: str, sql: str):
|
|
600
|
+
return self._req(f"/v0/pipes/{pipe_name_or_uid}/nodes", method="POST", data=sql.encode())
|
|
625
601
|
|
|
626
|
-
|
|
627
|
-
return
|
|
602
|
+
def pipe_set_endpoint(self, pipe_name_or_uid: str, published_node_uid: str):
|
|
603
|
+
return self._req(f"/v0/pipes/{pipe_name_or_uid}/nodes/{published_node_uid}/endpoint", method="POST")
|
|
628
604
|
|
|
629
|
-
|
|
630
|
-
return
|
|
605
|
+
def pipe_remove_endpoint(self, pipe_name_or_uid: str, published_node_uid: str):
|
|
606
|
+
return self._req(f"/v0/pipes/{pipe_name_or_uid}/nodes/{published_node_uid}/endpoint", method="DELETE")
|
|
631
607
|
|
|
632
|
-
|
|
608
|
+
def pipe_update_copy(
|
|
633
609
|
self,
|
|
634
610
|
pipe_name_or_id: str,
|
|
635
611
|
node_id: str,
|
|
@@ -645,45 +621,45 @@ class TinyB:
|
|
|
645
621
|
if mode:
|
|
646
622
|
data["mode"] = mode
|
|
647
623
|
|
|
648
|
-
return
|
|
624
|
+
return self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/copy", method="PUT", data=data)
|
|
649
625
|
|
|
650
|
-
|
|
651
|
-
return
|
|
626
|
+
def pipe_remove_copy(self, pipe_name_or_id: str, node_id: str):
|
|
627
|
+
return self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/copy", method="DELETE")
|
|
652
628
|
|
|
653
|
-
|
|
629
|
+
def pipe_run(
|
|
654
630
|
self, pipe_name_or_id: str, pipe_type: str, params: Optional[Dict[str, str]] = None, mode: Optional[str] = None
|
|
655
631
|
):
|
|
656
632
|
params = {**params} if params else {}
|
|
657
633
|
if mode:
|
|
658
634
|
params["_mode"] = mode
|
|
659
|
-
return
|
|
635
|
+
return self._req(f"/v0/pipes/{pipe_name_or_id}/{pipe_type}?{urlencode(params)}", method="POST")
|
|
660
636
|
|
|
661
|
-
|
|
662
|
-
return
|
|
637
|
+
def pipe_resume_copy(self, pipe_name_or_id: str):
|
|
638
|
+
return self._req(f"/v0/pipes/{pipe_name_or_id}/copy/resume", method="POST")
|
|
663
639
|
|
|
664
|
-
|
|
665
|
-
return
|
|
640
|
+
def pipe_pause_copy(self, pipe_name_or_id: str):
|
|
641
|
+
return self._req(f"/v0/pipes/{pipe_name_or_id}/copy/pause", method="POST")
|
|
666
642
|
|
|
667
|
-
|
|
643
|
+
def pipe_create_sink(self, pipe_name_or_id: str, node_id: str, params: Optional[Dict[str, str]] = None):
|
|
668
644
|
params = {**params} if params else {}
|
|
669
|
-
return
|
|
645
|
+
return self._req(
|
|
670
646
|
f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/sink?{urlencode(params)}", method="POST", data=""
|
|
671
647
|
)
|
|
672
648
|
|
|
673
|
-
|
|
674
|
-
return
|
|
649
|
+
def pipe_remove_sink(self, pipe_name_or_id: str, node_id: str):
|
|
650
|
+
return self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/sink", method="DELETE")
|
|
675
651
|
|
|
676
|
-
|
|
677
|
-
return
|
|
652
|
+
def pipe_remove_stream(self, pipe_name_or_id: str, node_id: str):
|
|
653
|
+
return self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/stream", method="DELETE")
|
|
678
654
|
|
|
679
|
-
|
|
655
|
+
def pipe_run_sink(self, pipe_name_or_id: str, params: Optional[Dict[str, str]] = None):
|
|
680
656
|
params = {**params} if params else {}
|
|
681
|
-
return
|
|
657
|
+
return self._req(f"/v0/pipes/{pipe_name_or_id}/sink?{urlencode(params)}", method="POST")
|
|
682
658
|
|
|
683
|
-
|
|
684
|
-
return
|
|
659
|
+
def pipe_unlink_materialized(self, pipe_name: str, node_id: str):
|
|
660
|
+
return self._req(f"/v0/pipes/{pipe_name}/nodes/{node_id}/materialization", method="DELETE")
|
|
685
661
|
|
|
686
|
-
|
|
662
|
+
def query(self, sql: str, pipeline: Optional[str] = None, playground: Optional[str] = None):
|
|
687
663
|
params = {}
|
|
688
664
|
if pipeline:
|
|
689
665
|
params = {"pipeline": pipeline}
|
|
@@ -692,43 +668,43 @@ class TinyB:
|
|
|
692
668
|
params.update({"release_replacements": "true"})
|
|
693
669
|
|
|
694
670
|
if len(sql) > TinyB.MAX_GET_LENGTH:
|
|
695
|
-
return
|
|
671
|
+
return self._req(f"/v0/sql?{urlencode(params)}", data=sql, method="POST")
|
|
696
672
|
else:
|
|
697
|
-
return
|
|
673
|
+
return self._req(f"/v0/sql?q={quote(sql, safe='')}&{urlencode(params)}")
|
|
698
674
|
|
|
699
|
-
|
|
675
|
+
def jobs(
|
|
700
676
|
self, status: Optional[Tuple[str, ...]] = None, kind: Optional[Tuple[str, ...]] = None
|
|
701
677
|
) -> List[Dict[str, Any]]:
|
|
702
|
-
|
|
678
|
+
def fetch_jobs(params: Dict[str, str]) -> List[Dict[str, Any]]:
|
|
703
679
|
query_string = urlencode(params) if params else ""
|
|
704
680
|
endpoint = f"/v0/jobs?{query_string}" if query_string else "/v0/jobs"
|
|
705
|
-
response =
|
|
681
|
+
response = self._req(endpoint)
|
|
706
682
|
return response["jobs"]
|
|
707
683
|
|
|
708
684
|
if not status and not kind:
|
|
709
|
-
return
|
|
685
|
+
return fetch_jobs({})
|
|
710
686
|
result: List[Dict[str, Any]] = []
|
|
711
687
|
if status and kind:
|
|
712
688
|
for s in status:
|
|
713
689
|
for k in kind:
|
|
714
|
-
result.extend(
|
|
690
|
+
result.extend(fetch_jobs({"status": s, "kind": k}))
|
|
715
691
|
elif status:
|
|
716
692
|
for s in status:
|
|
717
|
-
result.extend(
|
|
693
|
+
result.extend(fetch_jobs({"status": s}))
|
|
718
694
|
elif kind:
|
|
719
695
|
for k in kind:
|
|
720
|
-
result.extend(
|
|
696
|
+
result.extend(fetch_jobs({"kind": k}))
|
|
721
697
|
|
|
722
698
|
return result
|
|
723
699
|
|
|
724
|
-
|
|
725
|
-
return
|
|
700
|
+
def job(self, job_id: str):
|
|
701
|
+
return self._req(f"/v0/jobs/{job_id}")
|
|
726
702
|
|
|
727
|
-
|
|
728
|
-
return
|
|
703
|
+
def job_cancel(self, job_id: str):
|
|
704
|
+
return self._req(f"/v0/jobs/{job_id}/cancel", method="POST", data=b"")
|
|
729
705
|
|
|
730
|
-
|
|
731
|
-
data =
|
|
706
|
+
def user_workspaces(self, version: str = "v0"):
|
|
707
|
+
data = self._req(f"/{version}/user/workspaces/?with_environments=false")
|
|
732
708
|
# TODO: this is repeated in local_common.py but I'm avoiding circular imports
|
|
733
709
|
local_port = int(os.getenv("TB_LOCAL_PORT", 80))
|
|
734
710
|
local_host = f"http://localhost:{local_port}"
|
|
@@ -738,24 +714,24 @@ class TinyB:
|
|
|
738
714
|
local_workspaces = [x for x in data["workspaces"] if not x["name"].startswith("Tinybird_Local_")]
|
|
739
715
|
return {**data, "workspaces": local_workspaces}
|
|
740
716
|
|
|
741
|
-
|
|
742
|
-
return
|
|
717
|
+
def user_workspaces_and_branches(self, version: str = "v0"):
|
|
718
|
+
return self._req(f"/{version}/user/workspaces/?with_environments=true")
|
|
743
719
|
|
|
744
|
-
|
|
745
|
-
return
|
|
720
|
+
def user_workspaces_with_organization(self, version: str = "v0"):
|
|
721
|
+
return self._req(
|
|
746
722
|
f"/{version}/user/workspaces/?with_environments=false&with_organization=true&with_members_and_owner=false"
|
|
747
723
|
)
|
|
748
724
|
|
|
749
|
-
|
|
750
|
-
return
|
|
725
|
+
def user_workspace_branches(self, version: str = "v0"):
|
|
726
|
+
return self._req(f"/{version}/user/workspaces/?with_environments=true&only_environments=true")
|
|
751
727
|
|
|
752
|
-
|
|
753
|
-
return
|
|
728
|
+
def branches(self):
|
|
729
|
+
return self._req("/v0/environments")
|
|
754
730
|
|
|
755
|
-
|
|
756
|
-
return
|
|
731
|
+
def releases(self, workspace_id):
|
|
732
|
+
return self._req(f"/v0/workspaces/{workspace_id}/releases")
|
|
757
733
|
|
|
758
|
-
|
|
734
|
+
def create_workspace(
|
|
759
735
|
self,
|
|
760
736
|
name: str,
|
|
761
737
|
assign_to_organization_id: Optional[str] = None,
|
|
@@ -764,9 +740,9 @@ class TinyB:
|
|
|
764
740
|
url = f"/{version}/workspaces?name={name}"
|
|
765
741
|
if assign_to_organization_id:
|
|
766
742
|
url += f"&assign_to_organization_id={assign_to_organization_id}"
|
|
767
|
-
return
|
|
743
|
+
return self._req(url, method="POST", data=b"")
|
|
768
744
|
|
|
769
|
-
|
|
745
|
+
def create_workspace_branch(
|
|
770
746
|
self,
|
|
771
747
|
branch_name: str,
|
|
772
748
|
last_partition: Optional[bool],
|
|
@@ -779,9 +755,9 @@ class TinyB:
|
|
|
779
755
|
}
|
|
780
756
|
if ignore_datasources:
|
|
781
757
|
params["ignore_datasources"] = ",".join(ignore_datasources)
|
|
782
|
-
return
|
|
758
|
+
return self._req(f"/v0/environments?{urlencode(params)}", method="POST", data=b"")
|
|
783
759
|
|
|
784
|
-
|
|
760
|
+
def branch_workspace_data(
|
|
785
761
|
self,
|
|
786
762
|
workspace_id: str,
|
|
787
763
|
last_partition: bool,
|
|
@@ -797,9 +773,9 @@ class TinyB:
|
|
|
797
773
|
if ignore_datasources:
|
|
798
774
|
params["ignore_datasources"] = ",".join(ignore_datasources)
|
|
799
775
|
url = f"/v0/environments/{workspace_id}/data?{urlencode(params)}"
|
|
800
|
-
return
|
|
776
|
+
return self._req(url, method="POST", data=b"")
|
|
801
777
|
|
|
802
|
-
|
|
778
|
+
def branch_regression_tests(
|
|
803
779
|
self,
|
|
804
780
|
branch_id: str,
|
|
805
781
|
pipe_name: Optional[str],
|
|
@@ -850,9 +826,9 @@ class TinyB:
|
|
|
850
826
|
url = f"/v0/environments/{branch_id}/regression/main"
|
|
851
827
|
else:
|
|
852
828
|
url = f"/v0/environments/{branch_id}/regression"
|
|
853
|
-
return
|
|
829
|
+
return self._req(url, method="POST", data=data, headers={"Content-Type": "application/json"})
|
|
854
830
|
|
|
855
|
-
|
|
831
|
+
def branch_regression_tests_file(
|
|
856
832
|
self, branch_id: str, regression_commands: List[Dict[str, Any]], run_in_main: Optional[bool] = False
|
|
857
833
|
):
|
|
858
834
|
data = json.dumps(regression_commands)
|
|
@@ -860,72 +836,72 @@ class TinyB:
|
|
|
860
836
|
url = f"/v0/environments/{branch_id}/regression/main"
|
|
861
837
|
else:
|
|
862
838
|
url = f"/v0/environments/{branch_id}/regression"
|
|
863
|
-
return
|
|
839
|
+
return self._req(url, method="POST", data=data, headers={"Content-Type": "application/json"})
|
|
864
840
|
|
|
865
|
-
|
|
841
|
+
def delete_workspace(self, id: str, hard_delete_confirmation: Optional[str], version: str = "v0"):
|
|
866
842
|
data = {"confirmation": hard_delete_confirmation}
|
|
867
|
-
return
|
|
843
|
+
return self._req(f"/{version}/workspaces/{id}", data, method="DELETE")
|
|
868
844
|
|
|
869
|
-
|
|
870
|
-
return
|
|
845
|
+
def delete_branch(self, id: str):
|
|
846
|
+
return self._req(f"/v0/environments/{id}", method="DELETE")
|
|
871
847
|
|
|
872
|
-
|
|
848
|
+
def add_users_to_workspace(self, workspace: Dict[str, Any], users_emails: List[str], role: Optional[str]):
|
|
873
849
|
users = ",".join(users_emails)
|
|
874
|
-
return
|
|
850
|
+
return self._req(
|
|
875
851
|
f"/v0/workspaces/{workspace['id']}/users/",
|
|
876
852
|
method="PUT",
|
|
877
853
|
data={"operation": "add", "users": users, "role": role},
|
|
878
854
|
)
|
|
879
855
|
|
|
880
|
-
|
|
856
|
+
def remove_users_from_workspace(self, workspace: Dict[str, Any], users_emails: List[str]):
|
|
881
857
|
users = ",".join(users_emails)
|
|
882
|
-
return
|
|
858
|
+
return self._req(
|
|
883
859
|
f"/v0/workspaces/{workspace['id']}/users/", method="PUT", data={"operation": "remove", "users": users}
|
|
884
860
|
)
|
|
885
861
|
|
|
886
|
-
|
|
862
|
+
def set_role_for_users_in_workspace(self, workspace: Dict[str, Any], users_emails: List[str], role: str):
|
|
887
863
|
users = ",".join(users_emails)
|
|
888
|
-
return
|
|
864
|
+
return self._req(
|
|
889
865
|
f"/v0/workspaces/{workspace['id']}/users/",
|
|
890
866
|
method="PUT",
|
|
891
867
|
data={"operation": "change_role", "users": users, "new_role": role},
|
|
892
868
|
)
|
|
893
869
|
|
|
894
|
-
|
|
870
|
+
def workspace(self, workspace_id: str, with_token: bool = False):
|
|
895
871
|
params = {"with_token": "true" if with_token else "false"}
|
|
896
|
-
return
|
|
872
|
+
return self._req(f"/v0/workspaces/{workspace_id}?{urlencode(params)}")
|
|
897
873
|
|
|
898
|
-
|
|
899
|
-
return
|
|
874
|
+
def workspace_info(self, version: str = "v0") -> Dict[str, Any]:
|
|
875
|
+
return self._req(f"/{version}/workspace")
|
|
900
876
|
|
|
901
|
-
|
|
902
|
-
return
|
|
877
|
+
def organization(self, organization_id: str):
|
|
878
|
+
return self._req(f"/v0/organizations/{organization_id}")
|
|
903
879
|
|
|
904
|
-
|
|
880
|
+
def create_organization(
|
|
905
881
|
self,
|
|
906
882
|
name: str,
|
|
907
883
|
):
|
|
908
884
|
url = f"/v0/organizations?name={name}"
|
|
909
|
-
return
|
|
885
|
+
return self._req(url, method="POST", data=b"")
|
|
910
886
|
|
|
911
|
-
|
|
887
|
+
def add_workspaces_to_organization(self, organization_id: str, workspace_ids: List[str]):
|
|
912
888
|
if not workspace_ids:
|
|
913
889
|
return
|
|
914
|
-
return
|
|
890
|
+
return self._req(
|
|
915
891
|
f"/v0/organizations/{organization_id}/workspaces",
|
|
916
892
|
method="PUT",
|
|
917
893
|
data=json.dumps({"workspace_ids": ",".join(workspace_ids)}),
|
|
918
894
|
)
|
|
919
895
|
|
|
920
|
-
|
|
896
|
+
def infra_create(self, organization_id: str, name: str, host: str) -> Dict[str, Any]:
|
|
921
897
|
params = {
|
|
922
898
|
"organization_id": organization_id,
|
|
923
899
|
"name": name,
|
|
924
900
|
"host": host,
|
|
925
901
|
}
|
|
926
|
-
return
|
|
902
|
+
return self._req(f"/v1/infra?{urlencode(params)}", method="POST")
|
|
927
903
|
|
|
928
|
-
|
|
904
|
+
def infra_update(self, infra_id: str, organization_id: str, name: str, host: str) -> Dict[str, Any]:
|
|
929
905
|
params = {
|
|
930
906
|
"organization_id": organization_id,
|
|
931
907
|
}
|
|
@@ -933,16 +909,16 @@ class TinyB:
|
|
|
933
909
|
params["name"] = name
|
|
934
910
|
if host:
|
|
935
911
|
params["host"] = host
|
|
936
|
-
return
|
|
912
|
+
return self._req(f"/v1/infra/{infra_id}?{urlencode(params)}", method="PUT")
|
|
937
913
|
|
|
938
|
-
|
|
939
|
-
data =
|
|
914
|
+
def infra_list(self, organization_id: str) -> List[Dict[str, Any]]:
|
|
915
|
+
data = self._req(f"/v1/infra?organization_id={organization_id}")
|
|
940
916
|
return data.get("infras", [])
|
|
941
917
|
|
|
942
|
-
|
|
943
|
-
return
|
|
918
|
+
def infra_delete(self, infra_id: str, organization_id: str) -> Dict[str, Any]:
|
|
919
|
+
return self._req(f"/v1/infra/{infra_id}?organization_id={organization_id}", method="DELETE")
|
|
944
920
|
|
|
945
|
-
|
|
921
|
+
def wait_for_job(
|
|
946
922
|
self,
|
|
947
923
|
job_id: str,
|
|
948
924
|
status_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
|
|
@@ -954,7 +930,7 @@ class TinyB:
|
|
|
954
930
|
done: bool = False
|
|
955
931
|
while not done:
|
|
956
932
|
params = {"debug": "blocks,block_log"}
|
|
957
|
-
res =
|
|
933
|
+
res = self._req(f"/v0/jobs/{job_id}?{urlencode(params)}")
|
|
958
934
|
|
|
959
935
|
if res["status"] == "error":
|
|
960
936
|
error_message = "There has been an error"
|
|
@@ -974,19 +950,19 @@ class TinyB:
|
|
|
974
950
|
|
|
975
951
|
if not done:
|
|
976
952
|
backoff_seconds = min(backoff_seconds * backoff_multiplier, maximum_backoff_seconds)
|
|
977
|
-
|
|
953
|
+
time.sleep(backoff_seconds)
|
|
978
954
|
|
|
979
955
|
return res
|
|
980
956
|
|
|
981
|
-
|
|
982
|
-
return
|
|
957
|
+
def datasource_kafka_connect(self, connection_id, datasource_name, topic, group, auto_offset_reset):
|
|
958
|
+
return self._req(
|
|
983
959
|
f"/v0/datasources?connector={connection_id}&name={datasource_name}&"
|
|
984
960
|
f"kafka_topic={topic}&kafka_group_id={group}&kafka_auto_offset_reset={auto_offset_reset}",
|
|
985
961
|
method="POST",
|
|
986
962
|
data=b"",
|
|
987
963
|
)
|
|
988
964
|
|
|
989
|
-
|
|
965
|
+
def connection_create_kafka(
|
|
990
966
|
self,
|
|
991
967
|
kafka_bootstrap_servers,
|
|
992
968
|
kafka_key,
|
|
@@ -1016,24 +992,24 @@ class TinyB:
|
|
|
1016
992
|
params["kafka_ssl_ca_pem"] = kafka_ssl_ca_pem
|
|
1017
993
|
connection_params = {key: value for key, value in params.items() if value is not None}
|
|
1018
994
|
|
|
1019
|
-
return
|
|
995
|
+
return self._req(
|
|
1020
996
|
"/v0/connectors",
|
|
1021
997
|
method="POST",
|
|
1022
998
|
headers={"Content-Type": "application/json"},
|
|
1023
999
|
data=json.dumps(connection_params),
|
|
1024
1000
|
)
|
|
1025
1001
|
|
|
1026
|
-
|
|
1027
|
-
resp =
|
|
1002
|
+
def kafka_list_topics(self, connection_id: str, timeout=5):
|
|
1003
|
+
resp = self._req(f"/v0/connectors/{connection_id}/preview?preview_activity=false", timeout=timeout)
|
|
1028
1004
|
return [x["topic"] for x in resp["preview"]]
|
|
1029
1005
|
|
|
1030
|
-
|
|
1031
|
-
return
|
|
1006
|
+
def get_gcp_service_account_details(self) -> Dict[str, Any]:
|
|
1007
|
+
return self._req("/v0/datasources-bigquery-credentials")
|
|
1032
1008
|
|
|
1033
|
-
|
|
1009
|
+
def list_connectors(self, service: Optional[str] = None) -> List[Dict[str, Any]]:
|
|
1034
1010
|
try:
|
|
1035
1011
|
params: str = f"?service={service}" if service else ""
|
|
1036
|
-
result =
|
|
1012
|
+
result = self._req(f"/v0/connections/{params}")
|
|
1037
1013
|
if not result:
|
|
1038
1014
|
return []
|
|
1039
1015
|
|
|
@@ -1041,7 +1017,7 @@ class TinyB:
|
|
|
1041
1017
|
except Exception:
|
|
1042
1018
|
return []
|
|
1043
1019
|
|
|
1044
|
-
|
|
1020
|
+
def get_connector(
|
|
1045
1021
|
self,
|
|
1046
1022
|
name_or_id: str,
|
|
1047
1023
|
service: Optional[str] = None,
|
|
@@ -1049,14 +1025,14 @@ class TinyB:
|
|
|
1049
1025
|
skip_bigquery: Optional[bool] = False,
|
|
1050
1026
|
) -> Optional[Dict[str, Any]]:
|
|
1051
1027
|
return next(
|
|
1052
|
-
(c for c in
|
|
1028
|
+
(c for c in self.connections(connector=service, skip_bigquery=skip_bigquery) if c[key] == name_or_id),
|
|
1053
1029
|
None,
|
|
1054
1030
|
)
|
|
1055
1031
|
|
|
1056
|
-
|
|
1057
|
-
return
|
|
1032
|
+
def get_connector_by_id(self, connector_id: Optional[str] = None):
|
|
1033
|
+
return self._req(f"/v0/connectors/{connector_id}")
|
|
1058
1034
|
|
|
1059
|
-
|
|
1035
|
+
def get_snowflake_integration_query(
|
|
1060
1036
|
self, role: str, stage: Optional[str], integration: Optional[str]
|
|
1061
1037
|
) -> Optional[Dict[str, Any]]:
|
|
1062
1038
|
try:
|
|
@@ -1068,13 +1044,13 @@ class TinyB:
|
|
|
1068
1044
|
if integration:
|
|
1069
1045
|
params["integration"] = integration
|
|
1070
1046
|
|
|
1071
|
-
return
|
|
1047
|
+
return self._req(f"/v0/connectors/snowflake/instructions?{urlencode(params)}")
|
|
1072
1048
|
except Exception:
|
|
1073
1049
|
return None
|
|
1074
1050
|
|
|
1075
|
-
|
|
1051
|
+
def list_gcp_resources(self) -> List[Dict[str, Any]]:
|
|
1076
1052
|
try:
|
|
1077
|
-
resources =
|
|
1053
|
+
resources = self._req("/v0/connections/bigquery")
|
|
1078
1054
|
if not resources:
|
|
1079
1055
|
return []
|
|
1080
1056
|
|
|
@@ -1082,7 +1058,7 @@ class TinyB:
|
|
|
1082
1058
|
except Exception:
|
|
1083
1059
|
return []
|
|
1084
1060
|
|
|
1085
|
-
|
|
1061
|
+
def check_gcp_read_permissions(self) -> bool:
|
|
1086
1062
|
"""Returns `True` if our service account (see `TinyB::get_gcp_service_account_details()`)
|
|
1087
1063
|
has the proper permissions in GCP.
|
|
1088
1064
|
|
|
@@ -1092,17 +1068,17 @@ class TinyB:
|
|
|
1092
1068
|
See https://gitlab.com/tinybird/analytics/-/issues/6485.
|
|
1093
1069
|
"""
|
|
1094
1070
|
try:
|
|
1095
|
-
items =
|
|
1071
|
+
items = self.list_gcp_resources()
|
|
1096
1072
|
if not items:
|
|
1097
1073
|
return False
|
|
1098
1074
|
return len(items) > 0
|
|
1099
1075
|
except Exception:
|
|
1100
1076
|
return False
|
|
1101
1077
|
|
|
1102
|
-
|
|
1103
|
-
return
|
|
1078
|
+
def connector_delete(self, connection_id):
|
|
1079
|
+
return self._req(f"/v0/connectors/{connection_id}", method="DELETE")
|
|
1104
1080
|
|
|
1105
|
-
|
|
1081
|
+
def connection_create_snowflake(
|
|
1106
1082
|
self,
|
|
1107
1083
|
account_identifier: str,
|
|
1108
1084
|
user: str,
|
|
@@ -1128,42 +1104,42 @@ class TinyB:
|
|
|
1128
1104
|
if stage:
|
|
1129
1105
|
params["stage"] = stage
|
|
1130
1106
|
|
|
1131
|
-
return
|
|
1107
|
+
return self._req(f"/v0/connectors?{urlencode(params)}", method="POST", data="")
|
|
1132
1108
|
|
|
1133
|
-
|
|
1109
|
+
def validate_snowflake_connection(self, account_identifier: str, user: str, password: str) -> bool:
|
|
1134
1110
|
try:
|
|
1135
|
-
roles =
|
|
1111
|
+
roles = self.get_snowflake_roles(account_identifier, user, password)
|
|
1136
1112
|
if not roles:
|
|
1137
1113
|
return False
|
|
1138
1114
|
return len(roles) > 0
|
|
1139
1115
|
except Exception:
|
|
1140
1116
|
return False
|
|
1141
1117
|
|
|
1142
|
-
|
|
1118
|
+
def validate_preview_connection(self, service: str, params: Dict[str, Any]) -> bool:
|
|
1143
1119
|
params = {"service": service, "dry_run": "true", **params}
|
|
1144
1120
|
bucket_list = None
|
|
1145
1121
|
try:
|
|
1146
|
-
bucket_list =
|
|
1122
|
+
bucket_list = self._req(f"/v0/connectors?{urlencode(params)}", method="POST", data="")
|
|
1147
1123
|
if not bucket_list:
|
|
1148
1124
|
return False
|
|
1149
1125
|
return len(bucket_list) > 0
|
|
1150
1126
|
except Exception:
|
|
1151
1127
|
return False
|
|
1152
1128
|
|
|
1153
|
-
|
|
1129
|
+
def preview_bucket(self, connector: str, bucket_uri: str):
|
|
1154
1130
|
params = {"bucket_uri": bucket_uri, "service": "s3", "summary": "true"}
|
|
1155
|
-
return
|
|
1131
|
+
return self._req(f"/v0/connectors/{connector}/preview?{urlencode(params)}", method="GET")
|
|
1156
1132
|
|
|
1157
|
-
|
|
1158
|
-
return
|
|
1133
|
+
def connection_create(self, params: Dict[str, Any]) -> Dict[str, Any]:
|
|
1134
|
+
return self._req(f"/v0/connectors?{urlencode(params)}", method="POST", data="")
|
|
1159
1135
|
|
|
1160
|
-
|
|
1136
|
+
def get_snowflake_roles(self, account_identifier: str, user: str, password: str) -> Optional[List[str]]:
|
|
1161
1137
|
params = {"account": account_identifier, "username": user, "password": password}
|
|
1162
1138
|
|
|
1163
|
-
response =
|
|
1139
|
+
response = self._req(f"/v0/connectors/snowflake/roles?{urlencode(params)}", method="POST", data="")
|
|
1164
1140
|
return response["roles"]
|
|
1165
1141
|
|
|
1166
|
-
|
|
1142
|
+
def get_snowflake_warehouses(
|
|
1167
1143
|
self, account_identifier: str, user: str, password: str, role: str
|
|
1168
1144
|
) -> Optional[List[Dict[str, Any]]]:
|
|
1169
1145
|
params = {
|
|
@@ -1173,48 +1149,48 @@ class TinyB:
|
|
|
1173
1149
|
"role": role,
|
|
1174
1150
|
}
|
|
1175
1151
|
|
|
1176
|
-
response =
|
|
1152
|
+
response = self._req(f"/v0/connectors/snowflake/warehouses?{urlencode(params)}", method="POST", data="")
|
|
1177
1153
|
return response["warehouses"]
|
|
1178
1154
|
|
|
1179
|
-
|
|
1155
|
+
def check_aws_credentials(self) -> bool:
|
|
1180
1156
|
try:
|
|
1181
|
-
|
|
1157
|
+
self._req("/v0/integrations/s3/settings")
|
|
1182
1158
|
return True
|
|
1183
1159
|
except Exception:
|
|
1184
1160
|
return False
|
|
1185
1161
|
|
|
1186
|
-
|
|
1162
|
+
def get_trust_policy(self, service: str, external_id_seed: Optional[str] = None) -> Dict[str, Any]:
|
|
1187
1163
|
params = {}
|
|
1188
1164
|
if external_id_seed:
|
|
1189
1165
|
params["external_id_seed"] = external_id_seed
|
|
1190
|
-
return
|
|
1166
|
+
return self._req(f"/v0/integrations/{service}/policies/trust-policy?{urlencode(params)}")
|
|
1191
1167
|
|
|
1192
|
-
|
|
1168
|
+
def get_access_write_policy(self, service: str, bucket: Optional[str] = None) -> Dict[str, Any]:
|
|
1193
1169
|
params = {}
|
|
1194
1170
|
if bucket:
|
|
1195
1171
|
params["bucket"] = bucket
|
|
1196
|
-
return
|
|
1172
|
+
return self._req(f"/v0/integrations/{service}/policies/write-access-policy?{urlencode(params)}")
|
|
1197
1173
|
|
|
1198
|
-
|
|
1174
|
+
def get_access_read_policy(self, service: str, bucket: Optional[str] = None) -> Dict[str, Any]:
|
|
1199
1175
|
params = {}
|
|
1200
1176
|
if bucket:
|
|
1201
1177
|
params["bucket"] = bucket
|
|
1202
|
-
return
|
|
1178
|
+
return self._req(f"/v0/integrations/{service}/policies/read-access-policy?{urlencode(params)}")
|
|
1203
1179
|
|
|
1204
|
-
|
|
1180
|
+
def sql_get_format(self, sql: str, with_clickhouse_format: bool = False) -> str:
|
|
1205
1181
|
try:
|
|
1206
1182
|
if with_clickhouse_format:
|
|
1207
1183
|
from tinybird.sql_toolset import format_sql
|
|
1208
1184
|
|
|
1209
1185
|
return format_sql(sql)
|
|
1210
1186
|
else:
|
|
1211
|
-
return
|
|
1187
|
+
return self._sql_get_format_remote(sql, with_clickhouse_format)
|
|
1212
1188
|
except ModuleNotFoundError:
|
|
1213
|
-
return
|
|
1189
|
+
return self._sql_get_format_remote(sql, with_clickhouse_format)
|
|
1214
1190
|
|
|
1215
|
-
|
|
1191
|
+
def _sql_get_format_remote(self, sql: str, with_clickhouse_format: bool = False) -> str:
|
|
1216
1192
|
params = {"with_clickhouse_format": "true" if with_clickhouse_format else "false"}
|
|
1217
|
-
result =
|
|
1193
|
+
result = self._req(f"/v0/sql_format?q={quote(sql, safe='')}&{urlencode(params)}")
|
|
1218
1194
|
return result["q"]
|
|
1219
1195
|
|
|
1220
1196
|
@staticmethod
|
|
@@ -1226,7 +1202,7 @@ class TinyB:
|
|
|
1226
1202
|
)
|
|
1227
1203
|
return [t[1] if t[0] == "" else f"{t[0]}.{t[1]}" for t in tables]
|
|
1228
1204
|
|
|
1229
|
-
|
|
1205
|
+
def _sql_get_used_tables_remote(
|
|
1230
1206
|
self, sql: str, raising: bool = False, is_copy: Optional[bool] = False
|
|
1231
1207
|
) -> List[str]:
|
|
1232
1208
|
params = {
|
|
@@ -1235,15 +1211,15 @@ class TinyB:
|
|
|
1235
1211
|
"table_functions": "false",
|
|
1236
1212
|
"is_copy": "true" if is_copy else "false",
|
|
1237
1213
|
}
|
|
1238
|
-
result =
|
|
1214
|
+
result = self._req("/v0/sql_tables", data=params, method="POST")
|
|
1239
1215
|
return [t[1] if t[0] == "" else f"{t[0]}.{t[1]}" for t in result["tables"]]
|
|
1240
1216
|
|
|
1241
1217
|
# Get used tables from a query. Does not include table functions
|
|
1242
|
-
|
|
1218
|
+
def sql_get_used_tables(self, sql: str, raising: bool = False, is_copy: Optional[bool] = False) -> List[str]:
|
|
1243
1219
|
try:
|
|
1244
1220
|
return self._sql_get_used_tables_local(sql, raising, is_copy)
|
|
1245
1221
|
except ModuleNotFoundError:
|
|
1246
|
-
return
|
|
1222
|
+
return self._sql_get_used_tables_remote(sql, raising, is_copy)
|
|
1247
1223
|
|
|
1248
1224
|
@staticmethod
|
|
1249
1225
|
def _replace_tables_local(q: str, replacements):
|
|
@@ -1251,60 +1227,58 @@ class TinyB:
|
|
|
1251
1227
|
|
|
1252
1228
|
return replace_tables(q, replacements_to_tuples(replacements))
|
|
1253
1229
|
|
|
1254
|
-
|
|
1230
|
+
def _replace_tables_remote(self, q: str, replacements):
|
|
1255
1231
|
params = {
|
|
1256
1232
|
"q": q,
|
|
1257
1233
|
"replacements": json.dumps({k[1] if isinstance(k, tuple) else k: v for k, v in replacements.items()}),
|
|
1258
1234
|
}
|
|
1259
|
-
result =
|
|
1235
|
+
result = self._req("/v0/sql_replace", data=params, method="POST")
|
|
1260
1236
|
return result["query"]
|
|
1261
1237
|
|
|
1262
|
-
|
|
1238
|
+
def replace_tables(self, q: str, replacements):
|
|
1263
1239
|
try:
|
|
1264
1240
|
return self._replace_tables_local(q, replacements)
|
|
1265
1241
|
except ModuleNotFoundError:
|
|
1266
|
-
return
|
|
1242
|
+
return self._replace_tables_remote(q, replacements)
|
|
1267
1243
|
|
|
1268
|
-
|
|
1269
|
-
result =
|
|
1244
|
+
def get_connection(self, **kwargs):
|
|
1245
|
+
result = self._req("/v0/connectors")
|
|
1270
1246
|
return next((connector for connector in result["connectors"] if connector_equals(connector, kwargs)), None)
|
|
1271
1247
|
|
|
1272
|
-
|
|
1273
|
-
regions =
|
|
1248
|
+
def regions(self):
|
|
1249
|
+
regions = self._req("/v0/regions")
|
|
1274
1250
|
return regions
|
|
1275
1251
|
|
|
1276
|
-
|
|
1252
|
+
def datasource_query_copy(self, datasource_name: str, sql_query: str):
|
|
1277
1253
|
params = {"copy_to": datasource_name}
|
|
1278
|
-
return
|
|
1254
|
+
return self._req(f"/v0/sql_copy?{urlencode(params)}", data=sql_query, method="POST")
|
|
1279
1255
|
|
|
1280
|
-
|
|
1281
|
-
return
|
|
1282
|
-
f"/v0/workspaces/{workspace_id}/releases/?commit={commit}&force=true", method="POST", data=""
|
|
1283
|
-
)
|
|
1256
|
+
def workspace_commit_update(self, workspace_id: str, commit: str):
|
|
1257
|
+
return self._req(f"/v0/workspaces/{workspace_id}/releases/?commit={commit}&force=true", method="POST", data="")
|
|
1284
1258
|
|
|
1285
|
-
|
|
1286
|
-
return
|
|
1259
|
+
def update_release_semver(self, workspace_id: str, semver: str, new_semver: str):
|
|
1260
|
+
return self._req(f"/v0/workspaces/{workspace_id}/releases/{semver}?new_semver={new_semver}", method="PUT")
|
|
1287
1261
|
|
|
1288
|
-
|
|
1262
|
+
def release_new(self, workspace_id: str, semver: str, commit: str):
|
|
1289
1263
|
params = {
|
|
1290
1264
|
"commit": commit,
|
|
1291
1265
|
"semver": semver,
|
|
1292
1266
|
}
|
|
1293
|
-
return
|
|
1267
|
+
return self._req(f"/v0/workspaces/{workspace_id}/releases/?{urlencode(params)}", method="POST", data="")
|
|
1294
1268
|
|
|
1295
|
-
|
|
1296
|
-
return
|
|
1269
|
+
def release_failed(self, workspace_id: str, semver: str):
|
|
1270
|
+
return self._req(f"/v0/workspaces/{workspace_id}/releases/{semver}?status=failed", method="PUT")
|
|
1297
1271
|
|
|
1298
|
-
|
|
1299
|
-
return
|
|
1272
|
+
def release_preview(self, workspace_id: str, semver: str):
|
|
1273
|
+
return self._req(f"/v0/workspaces/{workspace_id}/releases/{semver}?status=preview", method="PUT")
|
|
1300
1274
|
|
|
1301
|
-
|
|
1302
|
-
return
|
|
1275
|
+
def release_promote(self, workspace_id: str, semver: str):
|
|
1276
|
+
return self._req(f"/v0/workspaces/{workspace_id}/releases/{semver}?status=live", method="PUT")
|
|
1303
1277
|
|
|
1304
|
-
|
|
1305
|
-
return
|
|
1278
|
+
def release_rollback(self, workspace_id: str, semver: str):
|
|
1279
|
+
return self._req(f"/v0/workspaces/{workspace_id}/releases/{semver}?status=rollback", method="PUT")
|
|
1306
1280
|
|
|
1307
|
-
|
|
1281
|
+
def release_rm(
|
|
1308
1282
|
self,
|
|
1309
1283
|
workspace_id: str,
|
|
1310
1284
|
semver: str,
|
|
@@ -1315,29 +1289,29 @@ class TinyB:
|
|
|
1315
1289
|
params = {"force": "true" if force else "false", "dry_run": "true" if dry_run else "false"}
|
|
1316
1290
|
if confirmation:
|
|
1317
1291
|
params["confirmation"] = confirmation
|
|
1318
|
-
return
|
|
1292
|
+
return self._req(f"/v0/workspaces/{workspace_id}/releases/{semver}?{urlencode(params)}", method="DELETE")
|
|
1319
1293
|
|
|
1320
|
-
|
|
1294
|
+
def release_oldest_rollback(
|
|
1321
1295
|
self,
|
|
1322
1296
|
workspace_id: str,
|
|
1323
1297
|
):
|
|
1324
|
-
return
|
|
1298
|
+
return self._req(f"/v0/workspaces/{workspace_id}/releases/oldest-rollback", method="GET")
|
|
1325
1299
|
|
|
1326
|
-
|
|
1327
|
-
tokens =
|
|
1300
|
+
def token_list(self, match: Optional[str] = None):
|
|
1301
|
+
tokens = self.tokens()
|
|
1328
1302
|
return [token for token in tokens if (not match or token["name"].find(match) != -1) and "token" in token]
|
|
1329
1303
|
|
|
1330
|
-
|
|
1331
|
-
return
|
|
1304
|
+
def token_delete(self, token_id: str):
|
|
1305
|
+
return self._req(f"/v0/tokens/{token_id}", method="DELETE")
|
|
1332
1306
|
|
|
1333
|
-
|
|
1334
|
-
return
|
|
1307
|
+
def token_refresh(self, token_id: str):
|
|
1308
|
+
return self._req(f"/v0/tokens/{token_id}/refresh", method="POST", data="")
|
|
1335
1309
|
|
|
1336
|
-
|
|
1337
|
-
return
|
|
1310
|
+
def token_get(self, token_id: str):
|
|
1311
|
+
return self._req(f"/v0/tokens/{token_id}", method="GET")
|
|
1338
1312
|
|
|
1339
|
-
|
|
1340
|
-
token =
|
|
1313
|
+
def token_scopes(self, token_id: str):
|
|
1314
|
+
token = self.token_get(token_id)
|
|
1341
1315
|
return token["scopes"]
|
|
1342
1316
|
|
|
1343
1317
|
def _token_to_params(self, token: Dict[str, Any]) -> str:
|
|
@@ -1362,31 +1336,31 @@ class TinyB:
|
|
|
1362
1336
|
params += f"&scope={scope}"
|
|
1363
1337
|
return params
|
|
1364
1338
|
|
|
1365
|
-
|
|
1339
|
+
def token_create(self, token: Dict[str, Any]):
|
|
1366
1340
|
params = self._token_to_params(token)
|
|
1367
|
-
return
|
|
1341
|
+
return self._req(f"/v0/tokens?{params}", method="POST", data="")
|
|
1368
1342
|
|
|
1369
|
-
|
|
1343
|
+
def create_jwt_token(self, name: str, expiration_time: int, scopes: List[Dict[str, Any]]):
|
|
1370
1344
|
url_params = {"name": name, "expiration_time": expiration_time}
|
|
1371
1345
|
body = json.dumps({"scopes": scopes})
|
|
1372
|
-
return
|
|
1346
|
+
return self._req(f"/v0/tokens?{urlencode(url_params)}", method="POST", data=body)
|
|
1373
1347
|
|
|
1374
|
-
|
|
1348
|
+
def token_update(self, token: Dict[str, Any]):
|
|
1375
1349
|
name = token["name"]
|
|
1376
1350
|
params = self._token_to_params(token)
|
|
1377
|
-
return
|
|
1351
|
+
return self._req(f"/v0/tokens/{name}?{params}", method="PUT", data="")
|
|
1378
1352
|
|
|
1379
|
-
|
|
1380
|
-
return
|
|
1353
|
+
def token_file(self, token_id: str):
|
|
1354
|
+
return self._req(f"/v0/tokens/{token_id}.token")
|
|
1381
1355
|
|
|
1382
|
-
|
|
1383
|
-
return
|
|
1356
|
+
def check_auth_login(self) -> Dict[str, Any]:
|
|
1357
|
+
return self._req("/v0/auth")
|
|
1384
1358
|
|
|
1385
|
-
|
|
1386
|
-
return
|
|
1359
|
+
def get_all_tags(self) -> Dict[str, Any]:
|
|
1360
|
+
return self._req("/v0/tags")
|
|
1387
1361
|
|
|
1388
|
-
|
|
1389
|
-
return
|
|
1362
|
+
def create_tag_with_resource(self, name: str, resource_id: str, resource_name: str, resource_type: str):
|
|
1363
|
+
return self._req(
|
|
1390
1364
|
"/v0/tags",
|
|
1391
1365
|
method="POST",
|
|
1392
1366
|
headers={"Content-Type": "application/json"},
|
|
@@ -1398,16 +1372,16 @@ class TinyB:
|
|
|
1398
1372
|
),
|
|
1399
1373
|
)
|
|
1400
1374
|
|
|
1401
|
-
|
|
1402
|
-
return
|
|
1375
|
+
def create_tag(self, name: str):
|
|
1376
|
+
return self._req(
|
|
1403
1377
|
"/v0/tags",
|
|
1404
1378
|
method="POST",
|
|
1405
1379
|
headers={"Content-Type": "application/json"},
|
|
1406
1380
|
data=json.dumps({"name": name}),
|
|
1407
1381
|
)
|
|
1408
1382
|
|
|
1409
|
-
|
|
1410
|
-
|
|
1383
|
+
def update_tag(self, name: str, resources: List[Dict[str, Any]]):
|
|
1384
|
+
self._req(
|
|
1411
1385
|
f"/v0/tags/{name}",
|
|
1412
1386
|
method="PUT",
|
|
1413
1387
|
headers={"Content-Type": "application/json"},
|
|
@@ -1418,5 +1392,5 @@ class TinyB:
|
|
|
1418
1392
|
),
|
|
1419
1393
|
)
|
|
1420
1394
|
|
|
1421
|
-
|
|
1422
|
-
|
|
1395
|
+
def delete_tag(self, name: str):
|
|
1396
|
+
self._req(f"/v0/tags/{name}", method="DELETE")
|