tinybird 0.0.1.dev13__py3-none-any.whl → 0.0.1.dev15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/tb/cli.py +0 -1
- tinybird/tb/modules/build.py +67 -18
- tinybird/tb/modules/cli.py +8 -29
- tinybird/tb/modules/create.py +5 -9
- tinybird/tb/modules/datafile/build.py +29 -21
- tinybird/tb/modules/datafile/build_pipe.py +4 -0
- tinybird/tb/modules/datafile/common.py +989 -49
- tinybird/tb/modules/datafile/diff.py +2 -2
- tinybird/tb/modules/datafile/parse_datasource.py +1 -0
- tinybird/tb/modules/datafile/pull.py +17 -21
- tinybird/tb/modules/llm.py +27 -37
- tinybird/tb/modules/local.py +3 -4
- tinybird/tb/modules/login.py +5 -3
- tinybird/tb/modules/mock.py +5 -4
- tinybird/tb/modules/prompts.py +23 -1
- tinybird/tb/modules/test.py +94 -74
- {tinybird-0.0.1.dev13.dist-info → tinybird-0.0.1.dev15.dist-info}/METADATA +1 -1
- {tinybird-0.0.1.dev13.dist-info → tinybird-0.0.1.dev15.dist-info}/RECORD +21 -22
- tinybird/check_pypi.py +0 -25
- {tinybird-0.0.1.dev13.dist-info → tinybird-0.0.1.dev15.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev13.dist-info → tinybird-0.0.1.dev15.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev13.dist-info → tinybird-0.0.1.dev15.dist-info}/top_level.txt +0 -0
|
@@ -121,12 +121,12 @@ async def diff_command(
|
|
|
121
121
|
if filenames:
|
|
122
122
|
if len(filenames) == 1:
|
|
123
123
|
filenames = [filenames[0], *get_project_filenames(filenames[0])]
|
|
124
|
-
await folder_pull(client, target_dir,
|
|
124
|
+
await folder_pull(client, target_dir, True, verbose=False)
|
|
125
125
|
else:
|
|
126
126
|
filenames = get_project_filenames(".")
|
|
127
127
|
if verbose:
|
|
128
128
|
click.echo("Saving remote resources in .diff_tmp folder.\n")
|
|
129
|
-
await folder_pull(client, target_dir,
|
|
129
|
+
await folder_pull(client, target_dir, True, verbose=verbose, progress_bar=progress_bar)
|
|
130
130
|
|
|
131
131
|
remote_datasources: List[Dict[str, Any]] = await client.datasources()
|
|
132
132
|
remote_pipes: List[Dict[str, Any]] = await client.pipes()
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import re
|
|
2
2
|
from asyncio import Semaphore, gather
|
|
3
3
|
from pathlib import Path
|
|
4
|
-
from typing import Any, Dict, List, Optional
|
|
4
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
5
5
|
|
|
6
6
|
import aiofiles
|
|
7
7
|
import click
|
|
@@ -16,38 +16,39 @@ from tinybird.tb.modules.datafile.format_pipe import format_pipe
|
|
|
16
16
|
async def folder_pull(
|
|
17
17
|
client: TinyB,
|
|
18
18
|
folder: str,
|
|
19
|
-
auto: bool,
|
|
20
|
-
match: Optional[str],
|
|
21
19
|
force: bool,
|
|
22
20
|
verbose: bool = True,
|
|
23
21
|
progress_bar: bool = False,
|
|
24
22
|
fmt: bool = False,
|
|
25
23
|
):
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
def _get_latest_versions(resources: List[str]):
|
|
24
|
+
def _get_latest_versions(resources: List[Tuple[str, str]]):
|
|
29
25
|
versions: Dict[str, Any] = {}
|
|
30
26
|
|
|
31
|
-
for x in resources:
|
|
27
|
+
for x, resource_type in resources:
|
|
32
28
|
t = get_name_version(x)
|
|
33
29
|
t["original_name"] = x
|
|
34
30
|
if t["version"] is None:
|
|
35
31
|
t["version"] = -1
|
|
36
32
|
name = t["name"]
|
|
33
|
+
t["type"] = resource_type
|
|
37
34
|
|
|
38
35
|
if name not in versions or name == x or versions[name]["version"] < t["version"]:
|
|
39
36
|
versions[name] = t
|
|
40
37
|
return versions
|
|
41
38
|
|
|
42
|
-
def get_file_folder(extension: str):
|
|
43
|
-
if not auto:
|
|
44
|
-
return None
|
|
39
|
+
def get_file_folder(extension: str, resource_type: Optional[str]):
|
|
45
40
|
if extension == "datasource":
|
|
46
41
|
return "datasources"
|
|
42
|
+
if resource_type == "endpoint":
|
|
43
|
+
return "endpoints"
|
|
44
|
+
if resource_type == "sink":
|
|
45
|
+
return "sinks"
|
|
46
|
+
if resource_type == "copy":
|
|
47
|
+
return "copies"
|
|
48
|
+
if resource_type == "materialized":
|
|
49
|
+
return "materializations"
|
|
47
50
|
if extension == "pipe":
|
|
48
51
|
return "pipes"
|
|
49
|
-
if extension == "token":
|
|
50
|
-
return "tokens"
|
|
51
52
|
return None
|
|
52
53
|
|
|
53
54
|
async def write_files(
|
|
@@ -61,11 +62,6 @@ async def folder_pull(
|
|
|
61
62
|
async def write_resource(k: Dict[str, Any]):
|
|
62
63
|
name = f"{k['name']}.{extension}"
|
|
63
64
|
try:
|
|
64
|
-
if pattern and not pattern.search(name):
|
|
65
|
-
if verbose:
|
|
66
|
-
click.echo(FeedbackManager.info_skipping_resource(resource=name))
|
|
67
|
-
return
|
|
68
|
-
|
|
69
65
|
resource = await getattr(client, get_resource_function)(k["original_name"])
|
|
70
66
|
resource_to_write = resource
|
|
71
67
|
|
|
@@ -76,11 +72,11 @@ async def folder_pull(
|
|
|
76
72
|
resource_to_write = await format_pipe(name, content=resource)
|
|
77
73
|
|
|
78
74
|
dest_folder = folder
|
|
79
|
-
if "." in k["name"]
|
|
75
|
+
if "." in k["name"]:
|
|
80
76
|
dest_folder = Path(folder) / "vendor" / k["name"].split(".", 1)[0]
|
|
81
77
|
name = f"{k['name'].split('.', 1)[1]}.{extension}"
|
|
82
78
|
|
|
83
|
-
file_folder = get_file_folder(extension)
|
|
79
|
+
file_folder = get_file_folder(extension, k["type"])
|
|
84
80
|
f = Path(dest_folder) / file_folder if file_folder is not None else Path(dest_folder)
|
|
85
81
|
|
|
86
82
|
if not f.exists():
|
|
@@ -125,11 +121,11 @@ async def folder_pull(
|
|
|
125
121
|
|
|
126
122
|
try:
|
|
127
123
|
datasources = await client.datasources()
|
|
128
|
-
remote_datasources = sorted([x["name"] for x in datasources])
|
|
124
|
+
remote_datasources = sorted([(x["name"], x.get("type", "csv")) for x in datasources], key=lambda x: x[0])
|
|
129
125
|
datasources_versions = _get_latest_versions(remote_datasources)
|
|
130
126
|
|
|
131
127
|
pipes = await client.pipes()
|
|
132
|
-
remote_pipes = sorted([
|
|
128
|
+
remote_pipes = sorted([(pipe["name"], pipe.get("type", "default")) for pipe in pipes], key=lambda x: x[0])
|
|
133
129
|
pipes_versions = _get_latest_versions(remote_pipes)
|
|
134
130
|
|
|
135
131
|
resources = list(datasources_versions.keys()) + list(pipes_versions.keys())
|
tinybird/tb/modules/llm.py
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
-
|
|
2
|
+
import urllib.parse
|
|
3
|
+
from copy import deepcopy
|
|
3
4
|
from typing import Awaitable, Callable, List
|
|
4
5
|
|
|
5
|
-
from openai import OpenAI
|
|
6
6
|
from pydantic import BaseModel
|
|
7
7
|
|
|
8
8
|
from tinybird.client import TinyB
|
|
9
|
-
from tinybird.tb.modules.
|
|
9
|
+
from tinybird.tb.modules.config import CLIConfig
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class DataFile(BaseModel):
|
|
@@ -20,8 +20,12 @@ class DataProject(BaseModel):
|
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
class LLM:
|
|
23
|
-
def __init__(self,
|
|
24
|
-
self.client =
|
|
23
|
+
def __init__(self, client: TinyB):
|
|
24
|
+
self.client = client
|
|
25
|
+
user_token = CLIConfig.get_project_config().get_user_token()
|
|
26
|
+
user_client = deepcopy(client)
|
|
27
|
+
user_client.token = user_token
|
|
28
|
+
self.user_client = user_client
|
|
25
29
|
|
|
26
30
|
async def _execute(self, action_fn: Callable[[], Awaitable[str]], checker_fn: Callable[[str], bool]):
|
|
27
31
|
is_valid = False
|
|
@@ -38,36 +42,22 @@ class LLM:
|
|
|
38
42
|
return result
|
|
39
43
|
|
|
40
44
|
async def create_project(self, prompt: str) -> DataProject:
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
async def generate_sql_sample_data(
|
|
49
|
-
self, tb_client: TinyB, datasource_content: str, row_count: int = 20, context: str = ""
|
|
50
|
-
) -> str:
|
|
51
|
-
async def action_fn():
|
|
52
|
-
response = self.client.chat.completions.create(
|
|
53
|
-
model="gpt-4o-mini",
|
|
54
|
-
messages=[
|
|
55
|
-
{
|
|
56
|
-
"role": "system",
|
|
57
|
-
"content": sample_data_sql_prompt.format(
|
|
58
|
-
current_datetime=datetime.now().isoformat(), row_count=row_count, context=context
|
|
59
|
-
),
|
|
60
|
-
},
|
|
61
|
-
{"role": "user", "content": datasource_content},
|
|
62
|
-
],
|
|
45
|
+
try:
|
|
46
|
+
response = await self.user_client._req(
|
|
47
|
+
"/v0/llm/create",
|
|
48
|
+
method="POST",
|
|
49
|
+
data=f'{{"prompt": "{prompt}"}}',
|
|
50
|
+
headers={"Content-Type": "application/json"},
|
|
63
51
|
)
|
|
64
|
-
return response.
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
52
|
+
return DataProject.model_validate(response.get("result", {}))
|
|
53
|
+
except Exception:
|
|
54
|
+
return DataProject(datasources=[], pipes=[])
|
|
55
|
+
|
|
56
|
+
async def generate_sql_sample_data(self, schema: str, rows: int = 20, context: str = "") -> str:
|
|
57
|
+
response = await self.user_client._req(
|
|
58
|
+
"/v0/llm/mock",
|
|
59
|
+
method="POST",
|
|
60
|
+
data=f'{{"schema": "{urllib.parse.quote(schema)}", "rows": {rows}, "context": "{urllib.parse.quote(context)}"}}',
|
|
61
|
+
headers={"Content-Type": "application/json"},
|
|
62
|
+
)
|
|
63
|
+
return response.get("result", "")
|
tinybird/tb/modules/local.py
CHANGED
|
@@ -140,10 +140,9 @@ async def get_tinybird_local_client(path: Optional[str] = None):
|
|
|
140
140
|
|
|
141
141
|
token = ws["token"]
|
|
142
142
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
config.persist_to_file()
|
|
143
|
+
config.set_token(token)
|
|
144
|
+
config.set_host(TB_LOCAL_HOST)
|
|
145
|
+
config.set_user_token(user_token)
|
|
147
146
|
return config.get_client(host=TB_LOCAL_HOST, token=token)
|
|
148
147
|
|
|
149
148
|
|
tinybird/tb/modules/login.py
CHANGED
|
@@ -87,6 +87,9 @@ class AuthHandler(http.server.SimpleHTTPRequestHandler):
|
|
|
87
87
|
return
|
|
88
88
|
|
|
89
89
|
|
|
90
|
+
AUTH_SERVER_PORT = 49160
|
|
91
|
+
|
|
92
|
+
|
|
90
93
|
class AuthServer(socketserver.TCPServer):
|
|
91
94
|
allow_reuse_address = True
|
|
92
95
|
|
|
@@ -96,7 +99,7 @@ class AuthServer(socketserver.TCPServer):
|
|
|
96
99
|
|
|
97
100
|
|
|
98
101
|
def start_server(auth_callback):
|
|
99
|
-
with AuthServer(("",
|
|
102
|
+
with AuthServer(("", AUTH_SERVER_PORT), AuthHandler, auth_callback) as httpd:
|
|
100
103
|
httpd.timeout = 30
|
|
101
104
|
start_time = time.time()
|
|
102
105
|
while time.time() - start_time < 60: # Run for a maximum of 60 seconds
|
|
@@ -132,7 +135,7 @@ def login(host: str, workspace: str):
|
|
|
132
135
|
|
|
133
136
|
# Open the browser to the auth page
|
|
134
137
|
client_id = "T6excMo8IKguvUw4vFNYfqlt9pe6msCU"
|
|
135
|
-
callback_url = "http://localhost:
|
|
138
|
+
callback_url = f"http://localhost:{AUTH_SERVER_PORT}"
|
|
136
139
|
params = {
|
|
137
140
|
"client_id": client_id,
|
|
138
141
|
"redirect_uri": callback_url,
|
|
@@ -159,7 +162,6 @@ def login(host: str, workspace: str):
|
|
|
159
162
|
cli_config.set_token_for_host(workspace_token, host)
|
|
160
163
|
cli_config.set_user_token(user_token)
|
|
161
164
|
config.set_host(host)
|
|
162
|
-
|
|
163
165
|
cli_config.persist_to_file()
|
|
164
166
|
click.echo(FeedbackManager.success(message="✓ Authentication successful!"))
|
|
165
167
|
else:
|
tinybird/tb/modules/mock.py
CHANGED
|
@@ -45,13 +45,14 @@ async def mock(datasource: str, rows: int, context: str, folder: str) -> None:
|
|
|
45
45
|
click.echo(FeedbackManager.gray(message=f"Overriding context for {datasource_name}..."))
|
|
46
46
|
context_path.write_text(context)
|
|
47
47
|
|
|
48
|
-
|
|
49
48
|
click.echo(FeedbackManager.gray(message=f"Creating fixture for {datasource_name}..."))
|
|
50
49
|
datasource_content = datasource_path.read_text()
|
|
51
|
-
|
|
52
|
-
|
|
50
|
+
config = CLIConfig.get_project_config()
|
|
51
|
+
user_client = config.get_client()
|
|
52
|
+
user_client.token = config.get_user_token()
|
|
53
|
+
llm = LLM(client=user_client)
|
|
53
54
|
tb_client = await get_tinybird_local_client(os.path.abspath(folder))
|
|
54
|
-
sql = await llm.generate_sql_sample_data(
|
|
55
|
+
sql = await llm.generate_sql_sample_data(datasource_content, rows=rows, context=context)
|
|
55
56
|
result = await tb_client.query(f"{sql} FORMAT JSON")
|
|
56
57
|
data = result.get("data", [])[:rows]
|
|
57
58
|
fixture_name = build_fixture_name(datasource_path.absolute(), datasource_name, datasource_content)
|
tinybird/tb/modules/prompts.py
CHANGED
|
@@ -76,6 +76,29 @@ SQL >
|
|
|
76
76
|
</instructions>
|
|
77
77
|
"""
|
|
78
78
|
|
|
79
|
+
create_test_calls_prompt = """
|
|
80
|
+
You are a Tinybird expert. You will be given a pipe endpoint containing different nodes with SQL and Tinybird templating syntax. You will generate URLs to test it with different parameters combinations.
|
|
81
|
+
|
|
82
|
+
<test>
|
|
83
|
+
<test_1>:
|
|
84
|
+
name: <test_name_1>
|
|
85
|
+
description: <description_1>
|
|
86
|
+
parameters: <url_encoded_parameters_1>
|
|
87
|
+
<test_2>:
|
|
88
|
+
name: <test_name_2>
|
|
89
|
+
description: <description_2>
|
|
90
|
+
parameters: <url_encoded_parameters_2>
|
|
91
|
+
</test>
|
|
92
|
+
<instructions>
|
|
93
|
+
- The test name must be unique.
|
|
94
|
+
- The test command must be a valid Tinybird command that can be run in the terminal.
|
|
95
|
+
- The test command can have as many parameters as are needed to test the pipe.
|
|
96
|
+
- The parameter within Tinybird templating syntax looks like this one {{String(my_param_name, default_value)}}.
|
|
97
|
+
- If there are no parameters in the , you can omit parametrs and generate a single test command.
|
|
98
|
+
- Extra context: {context}
|
|
99
|
+
</instructions>
|
|
100
|
+
"""
|
|
101
|
+
|
|
79
102
|
sample_data_prompt = """Your role is to generate sample data for a given data source. A data source is a table in Tinybird. You will be given a schema for the data source, and you will need to generate a set of rows that fit that schema.
|
|
80
103
|
|
|
81
104
|
The schema will be provided to you at the end of this prompt in the following format:
|
|
@@ -114,7 +137,6 @@ You will generate {row_count} rows of data in JSON format, newlines separated. Y
|
|
|
114
137
|
|
|
115
138
|
For columns that represent timestamps, use values that are close to the current date and time. The current date and time is {current_datetime}"""
|
|
116
139
|
|
|
117
|
-
|
|
118
140
|
sample_data_sql_prompt = """
|
|
119
141
|
Given the schema for a Tinybird datasource, return a can you create a clickhouse sql query to generate some random data that matches that schema.
|
|
120
142
|
|
tinybird/tb/modules/test.py
CHANGED
|
@@ -4,23 +4,21 @@
|
|
|
4
4
|
# - But please, **do not** interleave utility functions and command definitions.
|
|
5
5
|
|
|
6
6
|
import glob
|
|
7
|
-
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Iterable, List, Optional, Tuple
|
|
8
11
|
|
|
9
12
|
import click
|
|
13
|
+
import yaml
|
|
10
14
|
|
|
11
|
-
from tinybird.client import AuthNoTokenException
|
|
12
15
|
from tinybird.feedback_manager import FeedbackManager
|
|
13
16
|
from tinybird.tb.modules.cli import cli
|
|
14
|
-
from tinybird.tb.modules.common import coro
|
|
17
|
+
from tinybird.tb.modules.common import coro
|
|
15
18
|
from tinybird.tb.modules.config import CLIConfig
|
|
16
19
|
from tinybird.tb.modules.exceptions import CLIException
|
|
17
|
-
from tinybird.tb.modules.
|
|
18
|
-
|
|
19
|
-
generate_file,
|
|
20
|
-
parse_file,
|
|
21
|
-
run_test_file,
|
|
22
|
-
test_run_summary,
|
|
23
|
-
)
|
|
20
|
+
from tinybird.tb.modules.llm import LLM, TestExpectation
|
|
21
|
+
from tinybird.tb.modules.local import get_tinybird_local_client
|
|
24
22
|
|
|
25
23
|
|
|
26
24
|
@cli.group()
|
|
@@ -30,78 +28,100 @@ def test(ctx: click.Context) -> None:
|
|
|
30
28
|
|
|
31
29
|
|
|
32
30
|
@test.command(
|
|
33
|
-
name="
|
|
34
|
-
help="
|
|
31
|
+
name="create",
|
|
32
|
+
help="Create a test for an existing endpoint",
|
|
35
33
|
)
|
|
36
|
-
@click.argument("
|
|
37
|
-
@click.option(
|
|
38
|
-
|
|
39
|
-
|
|
34
|
+
@click.argument("pipe", type=str)
|
|
35
|
+
@click.option(
|
|
36
|
+
"--folder",
|
|
37
|
+
default=".",
|
|
38
|
+
type=click.Path(exists=True, file_okay=False),
|
|
39
|
+
help="Folder where datafiles will be placed",
|
|
40
|
+
)
|
|
41
|
+
@click.option("--prompt", type=str, default=None, help="Prompt to be used to create the test")
|
|
40
42
|
@click.pass_context
|
|
41
43
|
@coro
|
|
42
|
-
async def
|
|
43
|
-
|
|
44
|
+
async def test_create(ctx: click.Context, pipe: str, prompt: Optional[str], folder: Optional[str]) -> None:
|
|
45
|
+
"""
|
|
46
|
+
Create a test for an existing endpoint
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
def generate_test_file(pipe_name: str, tests: List[TestExpectation]):
|
|
50
|
+
base = Path("tests")
|
|
51
|
+
if folder:
|
|
52
|
+
base = Path(folder) / base
|
|
53
|
+
base.mkdir(parents=True, exist_ok=True)
|
|
54
|
+
|
|
55
|
+
path = base / f"{pipe_name}.yaml"
|
|
56
|
+
with open(path, "w") as f:
|
|
57
|
+
yaml.dump(tests, f)
|
|
44
58
|
|
|
45
59
|
try:
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
if
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
60
|
+
pipe_path = Path(pipe)
|
|
61
|
+
pipe_name = pipe
|
|
62
|
+
if pipe_path.suffix == ".pipe":
|
|
63
|
+
pipe_name = pipe_path.stem
|
|
64
|
+
else:
|
|
65
|
+
pipe_path = Path("endpoints", f"{pipe}.pipe")
|
|
66
|
+
if not pipe_path.exists():
|
|
67
|
+
pipe_path = Path("pipes", f"{pipe}.pipe")
|
|
68
|
+
|
|
69
|
+
click.echo(FeedbackManager.gray(message=f"\nCreating tests for {pipe_name} endpoint..."))
|
|
70
|
+
pipe_path = Path(folder) / pipe_path
|
|
71
|
+
pipe_content = pipe_path.read_text()
|
|
72
|
+
|
|
73
|
+
llm_config = CLIConfig.get_llm_config()
|
|
74
|
+
llm = LLM(key=llm_config["api_key"])
|
|
75
|
+
|
|
76
|
+
client = await get_tinybird_local_client(os.path.abspath(folder))
|
|
77
|
+
|
|
78
|
+
test_expectations = await llm.create_test_commands(pipe_content=pipe_content, context=prompt)
|
|
79
|
+
valid_test_expectations = []
|
|
80
|
+
for test in test_expectations.tests:
|
|
81
|
+
try:
|
|
82
|
+
response = await client._req(f"/v0/pipes/{pipe_name}.ndjson?{test.parameters}")
|
|
83
|
+
except Exception:
|
|
84
|
+
continue
|
|
85
|
+
|
|
86
|
+
test.expected_result = json.dumps(response)
|
|
87
|
+
valid_test_expectations.append(test.model_dump())
|
|
88
|
+
|
|
89
|
+
if valid_test_expectations:
|
|
90
|
+
generate_test_file(pipe_name, valid_test_expectations)
|
|
91
|
+
click.echo(FeedbackManager.info(message=f"✓ /tests/{pipe_name}.yaml"))
|
|
92
|
+
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
54
93
|
except Exception as e:
|
|
55
94
|
raise CLIException(FeedbackManager.error_exception(error=e))
|
|
56
95
|
|
|
57
|
-
file_list: Iterable[str] = file if len(file) > 0 else glob.glob("./tests/**/*.y*ml", recursive=True)
|
|
58
|
-
click.echo(FeedbackManager.info_skipping_resource(resource="regression.yaml"))
|
|
59
|
-
file_list = [f for f in file_list if not f.endswith("regression.yaml")]
|
|
60
|
-
final_file_list = []
|
|
61
|
-
for f in file_list:
|
|
62
|
-
if "skip_in_branch" in f and current_ws and current_ws.get("is_branch"):
|
|
63
|
-
click.echo(FeedbackManager.info_skipping_resource(resource=f))
|
|
64
|
-
else:
|
|
65
|
-
final_file_list.append(f)
|
|
66
|
-
file_list = final_file_list
|
|
67
|
-
|
|
68
|
-
async def run_test(tb_client, test_file, results):
|
|
69
|
-
try:
|
|
70
|
-
test_result = await run_test_file(tb_client, test_file)
|
|
71
|
-
results.append(TestSummaryResults(filename=test_file, results=test_result, semver=tb_client.semver))
|
|
72
|
-
except Exception as e:
|
|
73
|
-
if verbose:
|
|
74
|
-
click.echo(FeedbackManager.error_exception(error=e))
|
|
75
|
-
raise CLIException(FeedbackManager.error_running_test(file=test_file))
|
|
76
|
-
|
|
77
|
-
test_tasks = [run_test(tb_client, test_file, results) for test_file in file_list]
|
|
78
|
-
await gather_with_concurrency(concurrency, *test_tasks)
|
|
79
|
-
|
|
80
|
-
if len(results) <= 0:
|
|
81
|
-
click.echo(FeedbackManager.warning_no_test_results())
|
|
82
|
-
else:
|
|
83
|
-
test_run_summary(results, only_fail=only_fail, verbose_level=int(verbose))
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
@test.command(name="init", help="Initialize a file list with a simple test suite.")
|
|
87
|
-
@click.argument("files", nargs=-1)
|
|
88
|
-
@click.option("--force", is_flag=True, default=False, help="Override existing files")
|
|
89
|
-
@click.pass_context
|
|
90
|
-
@coro
|
|
91
|
-
async def test_init(ctx: click.Context, files: Tuple[str, ...], force: bool) -> None:
|
|
92
|
-
if len(files) == 0:
|
|
93
|
-
files = ("tests/default.yaml",)
|
|
94
|
-
|
|
95
|
-
for file in files:
|
|
96
|
-
generate_file(file, overwrite=force)
|
|
97
|
-
|
|
98
96
|
|
|
99
|
-
@test.command(
|
|
100
|
-
|
|
97
|
+
@test.command(
|
|
98
|
+
name="run",
|
|
99
|
+
help="Run the test suite, a file, or a test. To skip test to run in branches and CI put them in a 'skip_in_branch' folder.",
|
|
100
|
+
)
|
|
101
|
+
@click.argument("file", nargs=-1)
|
|
102
|
+
@click.option(
|
|
103
|
+
"--folder",
|
|
104
|
+
default=".",
|
|
105
|
+
type=click.Path(exists=True, file_okay=False),
|
|
106
|
+
help="Folder where tests will be placed",
|
|
107
|
+
)
|
|
101
108
|
@click.pass_context
|
|
102
109
|
@coro
|
|
103
|
-
async def
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
110
|
+
async def test_run(ctx: click.Context, file: Tuple[str, ...], folder: Optional[str]) -> None:
|
|
111
|
+
client = await get_tinybird_local_client(os.path.abspath(folder))
|
|
112
|
+
file_list: Iterable[str] = file if len(file) > 0 else glob.glob("./tests/**/*.y*ml", recursive=True)
|
|
113
|
+
|
|
114
|
+
async def run_test(test_file):
|
|
115
|
+
test_file_path = Path(test_file)
|
|
116
|
+
test_file_content = yaml.safe_load(test_file_path.read_text())
|
|
117
|
+
for test in test_file_content:
|
|
118
|
+
try:
|
|
119
|
+
response = await client._req(f"/v0/pipes/{test_file_path.stem}.ndjson?{test['parameters']}")
|
|
120
|
+
if test["expected_result"] != json.dumps(response):
|
|
121
|
+
raise Exception("Test failed")
|
|
122
|
+
click.echo(FeedbackManager.success(message=f"✓ {test_file_path.name} - {test['name']}"))
|
|
123
|
+
except Exception:
|
|
124
|
+
click.echo(FeedbackManager.error(message=f"✗ {test_file_path.name} - {test['name']}"))
|
|
125
|
+
|
|
126
|
+
for test_file in file_list:
|
|
127
|
+
await run_test(test_file)
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
tinybird/__cli__.py,sha256=pgYsVLcqL16wtSn6KtKweNZYoYJdEksTgSvQAW7hH64,250
|
|
2
|
-
tinybird/check_pypi.py,sha256=RIhZcTg9dk5SFuvV3yVtFgBael_5_VYl4GzOZJojCfU,971
|
|
3
2
|
tinybird/client.py,sha256=nd97gD2-8Ap8yDonBcVwk9eXDAL43hmIYdo-Pse43RE,50738
|
|
4
3
|
tinybird/config.py,sha256=Z-BX9FrjgsLw1YwcCdF0IztLB97Zpc70VVPplO_pDSY,6089
|
|
5
4
|
tinybird/connectors.py,sha256=lkpVSUmSuViEZBa4QjTK7YmPHUop0a5UFoTrSmlVq6k,15244
|
|
@@ -15,49 +14,49 @@ tinybird/syncasync.py,sha256=fAvq0qkRgqXqXMKwbY2iJNYqLT_r6mDsh1MRpGKrdRU,27763
|
|
|
15
14
|
tinybird/tornado_template.py,sha256=o2HguxrL1Evnt8o3IvrsI8Zm6JtRQ3zhLJKf1XyR3SQ,41965
|
|
16
15
|
tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj0,3890
|
|
17
16
|
tinybird/ch_utils/engine.py,sha256=OXkBhlzGjZotjD0vaT-rFIbSGV4tpiHxE8qO_ip0SyQ,40454
|
|
18
|
-
tinybird/tb/cli.py,sha256
|
|
17
|
+
tinybird/tb/cli.py,sha256=NipQJ0V-uqJDQSb4xkTw04ALzgqSfe1twmjkBkQJWT4,784
|
|
19
18
|
tinybird/tb/modules/auth.py,sha256=hynZ-Temot8YBsySUWKSFzZlYadtFPxG3o6lCSu1n6E,9018
|
|
20
19
|
tinybird/tb/modules/branch.py,sha256=R1tTUBGyI0p_dt2IAWbuyNOvemhjCIPwYxEmOxL3zOg,38468
|
|
21
|
-
tinybird/tb/modules/build.py,sha256=
|
|
20
|
+
tinybird/tb/modules/build.py,sha256=_EvBtX5FO7fnpHP74qlrBJqm7Uk1Bx9NarJ8P-M8vvg,12901
|
|
22
21
|
tinybird/tb/modules/cicd.py,sha256=KCFfywFfvGRh24GZwqrhICiTK_arHelPs_X4EB-pXIw,7331
|
|
23
|
-
tinybird/tb/modules/cli.py,sha256=
|
|
22
|
+
tinybird/tb/modules/cli.py,sha256=pF7bxobeLJP-aN7RwJmLaYOKQZHlBnneZpYbiAbTmaM,55511
|
|
24
23
|
tinybird/tb/modules/common.py,sha256=Vubc2AIR8BfEupnT5e1Y8OYGEyvNoIcjo8th-SaUflw,80111
|
|
25
24
|
tinybird/tb/modules/config.py,sha256=ppWvACHrSLkb5hOoQLYNby2w8jR76-8Kx2NBCst7ntQ,11760
|
|
26
25
|
tinybird/tb/modules/connection.py,sha256=ZSqBGoRiJedjHKEyB_fr1ybucOHtaad8d7uqGa2Q92M,28668
|
|
27
|
-
tinybird/tb/modules/create.py,sha256=
|
|
26
|
+
tinybird/tb/modules/create.py,sha256=WL2GzJBgH3z1m6oU58uZXyUQRzVB2rH4ig4BIdOJkLw,6307
|
|
28
27
|
tinybird/tb/modules/datasource.py,sha256=tjcf5o-HYIdTkb_c1ErGUFIE-W6G992vsvCuDGcxb9Q,35818
|
|
29
28
|
tinybird/tb/modules/exceptions.py,sha256=4A2sSjCEqKUMqpP3WI00zouCWW4uLaghXXLZBSw04mY,3363
|
|
30
29
|
tinybird/tb/modules/fmt.py,sha256=UszEQO15fdzQ49QEj7Unhu68IKwSuKPsOrKhk2p2TAg,3547
|
|
31
30
|
tinybird/tb/modules/job.py,sha256=eoBVyA24lYIPonU88Jn7FF9hBKz1kScy9_w_oWreuc4,2952
|
|
32
|
-
tinybird/tb/modules/llm.py,sha256=
|
|
33
|
-
tinybird/tb/modules/local.py,sha256=
|
|
34
|
-
tinybird/tb/modules/login.py,sha256=
|
|
35
|
-
tinybird/tb/modules/mock.py,sha256=
|
|
31
|
+
tinybird/tb/modules/llm.py,sha256=j8AZl-4BaCrtpM3_ozGt2Pr0BPfPFG5qYAaFgjJabGk,2008
|
|
32
|
+
tinybird/tb/modules/local.py,sha256=ajOrQMttYLXNs-4uDW9njw7-eIAHENvlbrNbURhr6gI,6926
|
|
33
|
+
tinybird/tb/modules/login.py,sha256=vIeysdttfGDBMkf_i3cqAVNR5s0X0D6exATcRsDdWiA,5849
|
|
34
|
+
tinybird/tb/modules/mock.py,sha256=HorUdULzxCCCGsFlPvL4RJeMJvVNnKnhbxxSog2Cn1Y,2817
|
|
36
35
|
tinybird/tb/modules/pipe.py,sha256=9wnfKbp2FkmLiJgVk3qbra76ktwsUTXghu6j9cCEahQ,31058
|
|
37
|
-
tinybird/tb/modules/prompts.py,sha256=
|
|
36
|
+
tinybird/tb/modules/prompts.py,sha256=KIDoAC0FvBSSsTft_mhpY6vxliNRBYOCsqikr_Hx0HA,9655
|
|
38
37
|
tinybird/tb/modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
39
38
|
tinybird/tb/modules/table.py,sha256=hG-PRDVuFp2uph41WpoLRV1yjp3RI2fi_iGGiI0rdxU,7695
|
|
40
39
|
tinybird/tb/modules/tag.py,sha256=1qQWyk1p3Btv3LzM8VbJG-k7x2-pFuAlYCg3QL6QewI,3480
|
|
41
40
|
tinybird/tb/modules/telemetry.py,sha256=iEGnMuCuNhvF6ln__j6X9MSTwL_0Hm-GgFHHHvhfknk,10466
|
|
42
|
-
tinybird/tb/modules/test.py,sha256=
|
|
41
|
+
tinybird/tb/modules/test.py,sha256=UR7Fz8AHhIptZfxofidzNzTnfAgwhCSWaGyjKbD2TAQ,4605
|
|
43
42
|
tinybird/tb/modules/token.py,sha256=r0oeG1RpOOzHtqbUaHBiOmhE55HfNIvReAAWyKl9fJg,12695
|
|
44
43
|
tinybird/tb/modules/workspace.py,sha256=FVlh-kbiZp5Gvp6dGFxi0UD8ail77rMamXLhqdVwrZ0,10916
|
|
45
44
|
tinybird/tb/modules/workspace_members.py,sha256=08W0onEYkKLEC5TkAI07cxN9XSquEm7HnL7OkHAVDjo,8715
|
|
46
|
-
tinybird/tb/modules/datafile/build.py,sha256=
|
|
45
|
+
tinybird/tb/modules/datafile/build.py,sha256=rFdK_GerPDgPfyPfZ4EZ0-cQqWfHd6htS0ls-Yy7khk,92491
|
|
47
46
|
tinybird/tb/modules/datafile/build_common.py,sha256=74547h5ja4C66DAwDMabj75FA_BUTJxTJv-24tSFmrs,4551
|
|
48
47
|
tinybird/tb/modules/datafile/build_datasource.py,sha256=fquzEGwk9NL_0K5YYG86Xtvgn4J5YHtRUoKJxbQGO0s,17344
|
|
49
|
-
tinybird/tb/modules/datafile/build_pipe.py,sha256=
|
|
50
|
-
tinybird/tb/modules/datafile/common.py,sha256=
|
|
51
|
-
tinybird/tb/modules/datafile/diff.py,sha256
|
|
48
|
+
tinybird/tb/modules/datafile/build_pipe.py,sha256=sSSl1rQMkR4uUbFCxK_aDXZi3JwKV64YZlBdBWgGKjo,27657
|
|
49
|
+
tinybird/tb/modules/datafile/common.py,sha256=NkoCdj4p-Ak3n80DJB5a33Ucw2WTcSYa8iqw4KsRZGs,81082
|
|
50
|
+
tinybird/tb/modules/datafile/diff.py,sha256=-iaP7GvAzZtZSa8jPgVpOFlTRutxgxRBLBcGL1_RFr4,6743
|
|
52
51
|
tinybird/tb/modules/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
|
|
53
52
|
tinybird/tb/modules/datafile/fixture.py,sha256=YHlL4tojmPwm343Y8KO6r7d5Bhsk7U3lKP-oLMeBMsY,1771
|
|
54
53
|
tinybird/tb/modules/datafile/format_common.py,sha256=zNWDXvwSKC9_T5e9R92LLj9ekDflVWwsllhGQilZsnY,2184
|
|
55
54
|
tinybird/tb/modules/datafile/format_datasource.py,sha256=tsnCjONISvhFuucKNbIHkT__UmlUbcswx5mwI9hiDQc,6216
|
|
56
55
|
tinybird/tb/modules/datafile/format_pipe.py,sha256=R5tnlEccLn3KX6ehtC_H2sGQNrthuJUiVSN9z_-KGCY,7474
|
|
57
|
-
tinybird/tb/modules/datafile/parse_datasource.py,sha256=
|
|
56
|
+
tinybird/tb/modules/datafile/parse_datasource.py,sha256=YKt4Sy830p2jqXW5jN1Bf60AM5o4gZ3QcZjxnyWIeBg,1190
|
|
58
57
|
tinybird/tb/modules/datafile/parse_pipe.py,sha256=STgA12LOLUnnb_cvVvZeEE4ka-nfk0jsNzxJhWj94cY,2599
|
|
59
58
|
tinybird/tb/modules/datafile/pipe_checker.py,sha256=cp80Bru41GlyMRvyERpdJNXns2MjmtIAWFnBLF4cPXs,24667
|
|
60
|
-
tinybird/tb/modules/datafile/pull.py,sha256=
|
|
59
|
+
tinybird/tb/modules/datafile/pull.py,sha256=wBXBAZIruIyCRQZvfYxMc7h1q35NlKF-hFIF-bUm4iY,5956
|
|
61
60
|
tinybird/tb/modules/tinyunit/tinyunit.py,sha256=IkjRCvb8HnNEE84rtl0I1b9gQVpE_zCE8MvFFet51sg,11716
|
|
62
61
|
tinybird/tb/modules/tinyunit/tinyunit_lib.py,sha256=hGh1ZaXC1af7rKnX7222urkj0QJMhMWclqMy59dOqwE,1922
|
|
63
62
|
tinybird/tb_cli_modules/cicd.py,sha256=0lMkb6CVOFZl5HOwgY8mK4T4mgI7O8335UngLXtCc-c,13851
|
|
@@ -66,8 +65,8 @@ tinybird/tb_cli_modules/config.py,sha256=6NTgIdwf0X132A1j6G_YrdPep87ymZ9b5pABabK
|
|
|
66
65
|
tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
|
|
67
66
|
tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
68
67
|
tinybird/tb_cli_modules/telemetry.py,sha256=iEGnMuCuNhvF6ln__j6X9MSTwL_0Hm-GgFHHHvhfknk,10466
|
|
69
|
-
tinybird-0.0.1.
|
|
70
|
-
tinybird-0.0.1.
|
|
71
|
-
tinybird-0.0.1.
|
|
72
|
-
tinybird-0.0.1.
|
|
73
|
-
tinybird-0.0.1.
|
|
68
|
+
tinybird-0.0.1.dev15.dist-info/METADATA,sha256=D7NcYU4DyAQdBn5hevIYUm3gugIyzdVfXYT7L5DxTYQ,2405
|
|
69
|
+
tinybird-0.0.1.dev15.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
|
|
70
|
+
tinybird-0.0.1.dev15.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
|
|
71
|
+
tinybird-0.0.1.dev15.dist-info/top_level.txt,sha256=pgw6AzERHBcW3YTi2PW4arjxLkulk2msOz_SomfOEuc,45
|
|
72
|
+
tinybird-0.0.1.dev15.dist-info/RECORD,,
|
tinybird/check_pypi.py
DELETED
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
from typing import Optional
|
|
2
|
-
|
|
3
|
-
import requests
|
|
4
|
-
|
|
5
|
-
from tinybird.feedback_manager import FeedbackManager
|
|
6
|
-
from tinybird.syncasync import sync_to_async
|
|
7
|
-
from tinybird.tb_cli_modules.common import CLIException, getenv_bool
|
|
8
|
-
|
|
9
|
-
PYPY_URL = "https://pypi.org/pypi/tinybird/json"
|
|
10
|
-
requests_get = sync_to_async(requests.get, thread_sensitive=False)
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
class CheckPypi:
|
|
14
|
-
async def get_latest_version(self) -> Optional[str]:
|
|
15
|
-
version: Optional[str] = None
|
|
16
|
-
try:
|
|
17
|
-
disable_ssl: bool = getenv_bool("TB_DISABLE_SSL_CHECKS", False)
|
|
18
|
-
response: requests.Response = await requests_get(PYPY_URL, verify=not disable_ssl)
|
|
19
|
-
if response.status_code != 200:
|
|
20
|
-
raise CLIException(FeedbackManager.error_exception(error=response.content.decode("utf-8")))
|
|
21
|
-
version = response.json()["info"]["version"]
|
|
22
|
-
except Exception as e:
|
|
23
|
-
raise CLIException(FeedbackManager.error_exception(error=str(e)))
|
|
24
|
-
|
|
25
|
-
return version
|
|
File without changes
|
|
File without changes
|
|
File without changes
|