tinybird 0.0.1.dev17__py3-none-any.whl → 0.0.1.dev19__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/client.py +22 -2
- tinybird/tb/__cli__.py +8 -0
- tinybird/tb/modules/build.py +32 -10
- tinybird/tb/modules/build_shell.py +253 -34
- tinybird/tb/modules/cicd.py +9 -89
- tinybird/tb/modules/cli.py +3 -1
- tinybird/tb/modules/common.py +1 -108
- tinybird/tb/modules/create.py +2 -6
- tinybird/tb/modules/datafile/common.py +224 -247
- tinybird/tb/modules/datafile/parse_datasource.py +8 -0
- tinybird/tb/modules/datafile/parse_pipe.py +10 -1
- tinybird/tb/modules/llm.py +4 -3
- tinybird/tb/modules/local_common.py +1 -1
- tinybird/tb/modules/login.py +17 -10
- tinybird/tb/modules/mock.py +14 -12
- tinybird/tb/modules/test.py +116 -24
- {tinybird-0.0.1.dev17.dist-info → tinybird-0.0.1.dev19.dist-info}/METADATA +2 -1
- {tinybird-0.0.1.dev17.dist-info → tinybird-0.0.1.dev19.dist-info}/RECORD +21 -20
- {tinybird-0.0.1.dev17.dist-info → tinybird-0.0.1.dev19.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev17.dist-info → tinybird-0.0.1.dev19.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev17.dist-info → tinybird-0.0.1.dev19.dist-info}/top_level.txt +0 -0
tinybird/tb/modules/common.py
CHANGED
|
@@ -17,7 +17,7 @@ from contextlib import closing
|
|
|
17
17
|
from copy import deepcopy
|
|
18
18
|
from enum import Enum
|
|
19
19
|
from functools import wraps
|
|
20
|
-
from os import
|
|
20
|
+
from os import environ, getcwd, getenv
|
|
21
21
|
from pathlib import Path
|
|
22
22
|
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Union
|
|
23
23
|
from urllib.parse import urlparse
|
|
@@ -61,9 +61,7 @@ if TYPE_CHECKING:
|
|
|
61
61
|
from tinybird.connectors import Connector
|
|
62
62
|
|
|
63
63
|
from tinybird.feedback_manager import FeedbackManager, warning_message
|
|
64
|
-
from tinybird.git_settings import DEFAULT_TINYENV_FILE
|
|
65
64
|
from tinybird.syncasync import async_to_sync, sync_to_async
|
|
66
|
-
from tinybird.tb.modules.cicd import APPEND_FIXTURES_SH, DEFAULT_REQUIREMENTS_FILE, EXEC_TEST_SH
|
|
67
65
|
from tinybird.tb.modules.config import CLIConfig
|
|
68
66
|
from tinybird.tb.modules.exceptions import (
|
|
69
67
|
CLIAuthException,
|
|
@@ -371,111 +369,6 @@ async def folder_init(
|
|
|
371
369
|
for path in Path(folder).glob(f"*.{format}"):
|
|
372
370
|
await _generate_datafile(str(path), client, format=format, force=force)
|
|
373
371
|
|
|
374
|
-
if generate_releases:
|
|
375
|
-
base = Path(".")
|
|
376
|
-
f = base / (".tinyenv")
|
|
377
|
-
if not f.exists() or force:
|
|
378
|
-
async with aiofiles.open(".tinyenv", "w") as file:
|
|
379
|
-
await file.write(DEFAULT_TINYENV_FILE)
|
|
380
|
-
click.echo(FeedbackManager.info_file_created(file=".tinyenv"))
|
|
381
|
-
else:
|
|
382
|
-
click.echo(FeedbackManager.info_dottinyenv_already_exists())
|
|
383
|
-
|
|
384
|
-
base = Path(".")
|
|
385
|
-
f = base / ("requirements.txt")
|
|
386
|
-
if not f.exists() or force:
|
|
387
|
-
async with aiofiles.open("requirements.txt", "w") as file:
|
|
388
|
-
await file.write(DEFAULT_REQUIREMENTS_FILE)
|
|
389
|
-
click.echo(FeedbackManager.info_file_created(file="requirements.txt"))
|
|
390
|
-
|
|
391
|
-
base = Path("scripts")
|
|
392
|
-
if not base.exists():
|
|
393
|
-
base = Path()
|
|
394
|
-
f = base / ("exec_test.sh")
|
|
395
|
-
if not f.exists() or force:
|
|
396
|
-
async with aiofiles.open(f"{f}", "w") as t_file:
|
|
397
|
-
await t_file.write(EXEC_TEST_SH)
|
|
398
|
-
click.echo(FeedbackManager.info_file_created(file="scripts/exec_test.sh"))
|
|
399
|
-
chmod(f, 0o755)
|
|
400
|
-
|
|
401
|
-
f = base / ("append_fixtures.sh")
|
|
402
|
-
if not f.exists() or force:
|
|
403
|
-
async with aiofiles.open(f"{f}", "w") as t_file:
|
|
404
|
-
await t_file.write(APPEND_FIXTURES_SH)
|
|
405
|
-
click.echo(FeedbackManager.info_file_created(file="scripts/append_fixtures.sh"))
|
|
406
|
-
chmod(f, 0o755)
|
|
407
|
-
|
|
408
|
-
base = Path("tests")
|
|
409
|
-
if not base.exists():
|
|
410
|
-
base = Path()
|
|
411
|
-
f = base / ("example.yml")
|
|
412
|
-
if not base.exists() or force:
|
|
413
|
-
async with aiofiles.open(f"{f}", "w") as t_file:
|
|
414
|
-
await t_file.write(
|
|
415
|
-
"""
|
|
416
|
-
##############################################################################################################################
|
|
417
|
-
### Visit https://www.tinybird.co/docs/production/implementing-test-strategies.html#data-quality-tests ###
|
|
418
|
-
### for more details on Data Quality tests ###
|
|
419
|
-
##############################################################################################################################
|
|
420
|
-
|
|
421
|
-
- example_no_negative_numbers:
|
|
422
|
-
max_bytes_read: null
|
|
423
|
-
max_time: null
|
|
424
|
-
sql: |
|
|
425
|
-
SELECT
|
|
426
|
-
number
|
|
427
|
-
FROM numbers(10)
|
|
428
|
-
WHERE
|
|
429
|
-
number < 0
|
|
430
|
-
|
|
431
|
-
# - example_top_products_params_no_empty_top_10_on_2023:
|
|
432
|
-
# max_bytes_read: null
|
|
433
|
-
# max_time: null
|
|
434
|
-
# sql: |
|
|
435
|
-
# SELECT *
|
|
436
|
-
# FROM top_products_params
|
|
437
|
-
# WHERE empty(top_10)
|
|
438
|
-
# pipe:
|
|
439
|
-
# name: top_products_params
|
|
440
|
-
# params:
|
|
441
|
-
# start: '2023-01-01'
|
|
442
|
-
# end: '2023-12-31'
|
|
443
|
-
|
|
444
|
-
"""
|
|
445
|
-
)
|
|
446
|
-
|
|
447
|
-
f = base / ("regression.yaml")
|
|
448
|
-
if not base.exists() or force:
|
|
449
|
-
async with aiofiles.open(f"{f}", "w") as t_file:
|
|
450
|
-
await t_file.write(
|
|
451
|
-
"""
|
|
452
|
-
############################################################################################################################
|
|
453
|
-
### Visit https://www.tinybird.co/docs/production/implementing-test-strategies.html#regression-tests ###
|
|
454
|
-
### for more details on Regression tests ###
|
|
455
|
-
############################################################################################################################
|
|
456
|
-
|
|
457
|
-
###
|
|
458
|
-
### New pipes are covered by this rule, rules below this one supersede this setting
|
|
459
|
-
###
|
|
460
|
-
- pipe: '.*'
|
|
461
|
-
tests:
|
|
462
|
-
- coverage:
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
###
|
|
467
|
-
### These are rules to customize regression testing by pipe using regular expressions
|
|
468
|
-
### For instance skip regression tests for the pipes matching `endpoint_name.*`
|
|
469
|
-
###
|
|
470
|
-
- pipe: 'endpoint_name.*'
|
|
471
|
-
tests:
|
|
472
|
-
- coverage:
|
|
473
|
-
config:
|
|
474
|
-
skip: True
|
|
475
|
-
|
|
476
|
-
"""
|
|
477
|
-
)
|
|
478
|
-
|
|
479
372
|
|
|
480
373
|
async def configure_connector(connector):
|
|
481
374
|
if connector not in SUPPORTED_CONNECTORS:
|
tinybird/tb/modules/create.py
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import os
|
|
2
|
-
import subprocess
|
|
3
2
|
from os import getcwd
|
|
4
3
|
from pathlib import Path
|
|
5
4
|
from typing import Optional
|
|
@@ -126,7 +125,7 @@ async def create(
|
|
|
126
125
|
test_name = "api_token_usage"
|
|
127
126
|
test_path = Path(folder) / "tests" / f"{test_name}.yaml"
|
|
128
127
|
test_content = fetch_gist_content(
|
|
129
|
-
"https://gist.githubusercontent.com/gnzjgo/e58620bbb977d6f42f1d0c2a7b46ac8f/raw/
|
|
128
|
+
"https://gist.githubusercontent.com/gnzjgo/e58620bbb977d6f42f1d0c2a7b46ac8f/raw/a3a1cd0ce3a90bcd2f6dfce00da51e6051443612/api_token_usage.yaml"
|
|
130
129
|
)
|
|
131
130
|
test_path.write_text(test_content)
|
|
132
131
|
click.echo(FeedbackManager.info(message=f"✓ /tests/{test_name}.yaml"))
|
|
@@ -147,7 +146,7 @@ async def create(
|
|
|
147
146
|
datasource_content = datasource_path.read_text()
|
|
148
147
|
has_json_path = "`json:" in datasource_content
|
|
149
148
|
if has_json_path:
|
|
150
|
-
sql = await llm.generate_sql_sample_data(schema=datasource_content, rows=rows,
|
|
149
|
+
sql = await llm.generate_sql_sample_data(schema=datasource_content, rows=rows, prompt=prompt)
|
|
151
150
|
result = await tb_client.query(f"{sql} FORMAT JSON")
|
|
152
151
|
data = result.get("data", [])
|
|
153
152
|
fixture_name = build_fixture_name(datasource_path.absolute(), datasource_name, datasource_content)
|
|
@@ -214,9 +213,6 @@ def init_git(folder: str):
|
|
|
214
213
|
try:
|
|
215
214
|
path = Path(folder)
|
|
216
215
|
gitignore_file = path / ".gitignore"
|
|
217
|
-
git_folder = path / ".git"
|
|
218
|
-
if not git_folder.exists():
|
|
219
|
-
subprocess.run(["git", "init"], cwd=path, check=True, capture_output=True)
|
|
220
216
|
|
|
221
217
|
if gitignore_file.exists():
|
|
222
218
|
content = gitignore_file.read_text()
|