tinybird 0.0.1.dev29__py3-none-any.whl → 0.0.1.dev31__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/__cli__.py +2 -2
- tinybird/client.py +1 -1
- tinybird/config.py +1 -0
- tinybird/context.py +1 -0
- tinybird/prompts.py +218 -325
- tinybird/sql_template.py +21 -2
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/cli.py +2 -1
- tinybird/tb/modules/build.py +75 -188
- tinybird/tb/modules/build_client.py +219 -0
- tinybird/tb/modules/cli.py +53 -16
- tinybird/tb/modules/common.py +1 -26
- tinybird/tb/modules/config.py +0 -8
- tinybird/tb/modules/create.py +22 -3
- tinybird/tb/modules/datafile/build.py +2 -2
- tinybird/tb/modules/datafile/build_pipe.py +13 -1
- tinybird/tb/modules/datasource.py +1 -1
- tinybird/tb/modules/llm.py +6 -73
- tinybird/tb/modules/local.py +1 -1
- tinybird/tb/modules/local_common.py +1 -1
- tinybird/tb/modules/login.py +7 -2
- tinybird/tb/modules/mock.py +1 -1
- tinybird/tb/modules/shell.py +30 -35
- tinybird/tb/modules/test.py +41 -22
- tinybird/tb/modules/update.py +182 -0
- tinybird/tb/modules/watch.py +76 -1
- {tinybird-0.0.1.dev29.dist-info → tinybird-0.0.1.dev31.dist-info}/METADATA +2 -1
- {tinybird-0.0.1.dev29.dist-info → tinybird-0.0.1.dev31.dist-info}/RECORD +31 -30
- tinybird/tb/modules/build_server.py +0 -75
- {tinybird-0.0.1.dev29.dist-info → tinybird-0.0.1.dev31.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev29.dist-info → tinybird-0.0.1.dev31.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev29.dist-info → tinybird-0.0.1.dev31.dist-info}/top_level.txt +0 -0
tinybird/tb/modules/test.py
CHANGED
|
@@ -6,11 +6,13 @@
|
|
|
6
6
|
import difflib
|
|
7
7
|
import glob
|
|
8
8
|
import os
|
|
9
|
+
import urllib.parse
|
|
9
10
|
from pathlib import Path
|
|
10
|
-
from typing import Any, Dict,
|
|
11
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
11
12
|
|
|
12
13
|
import click
|
|
13
14
|
import yaml
|
|
15
|
+
from requests import Response
|
|
14
16
|
|
|
15
17
|
from tinybird.prompts import test_create_prompt
|
|
16
18
|
from tinybird.tb.modules.cli import cli
|
|
@@ -66,9 +68,9 @@ def test(ctx: click.Context) -> None:
|
|
|
66
68
|
type=click.Path(exists=True, file_okay=False),
|
|
67
69
|
help="Folder where datafiles will be placed",
|
|
68
70
|
)
|
|
69
|
-
@click.option("--prompt", type=str, default=
|
|
71
|
+
@click.option("--prompt", type=str, default="", help="Prompt to be used to create the test")
|
|
70
72
|
@coro
|
|
71
|
-
async def test_create(name_or_filename: str, prompt:
|
|
73
|
+
async def test_create(name_or_filename: str, prompt: str, folder: str) -> None:
|
|
72
74
|
"""
|
|
73
75
|
Create a test for an existing pipe
|
|
74
76
|
"""
|
|
@@ -80,15 +82,13 @@ async def test_create(name_or_filename: str, prompt: Optional[str], folder: str)
|
|
|
80
82
|
raise CLIException(FeedbackManager.error(message=f"Pipe {name_or_filename} not found"))
|
|
81
83
|
else:
|
|
82
84
|
pipe_folders = ("endpoints", "copies", "materializations", "sinks", "pipes")
|
|
83
|
-
|
|
84
|
-
(
|
|
85
|
+
try:
|
|
86
|
+
pipe_path = next(
|
|
85
87
|
root_path / folder / f"{name_or_filename}.pipe"
|
|
86
88
|
for folder in pipe_folders
|
|
87
89
|
if (root_path / folder / f"{name_or_filename}.pipe").exists()
|
|
88
|
-
)
|
|
89
|
-
|
|
90
|
-
)
|
|
91
|
-
if not pipe_path:
|
|
90
|
+
)
|
|
91
|
+
except Exception:
|
|
92
92
|
raise CLIException(FeedbackManager.error(message=f"Pipe {name_or_filename} not found"))
|
|
93
93
|
|
|
94
94
|
pipe_name = pipe_path.stem
|
|
@@ -97,8 +97,8 @@ async def test_create(name_or_filename: str, prompt: Optional[str], folder: str)
|
|
|
97
97
|
pipe_content = pipe_path.read_text()
|
|
98
98
|
|
|
99
99
|
client = await get_tinybird_local_client(os.path.abspath(folder))
|
|
100
|
-
|
|
101
|
-
parameters = set([param["name"] for node in
|
|
100
|
+
pipe = await client._req(f"/v0/pipes/{pipe_name}")
|
|
101
|
+
parameters = set([param["name"] for node in pipe["nodes"] for param in node["params"]])
|
|
102
102
|
|
|
103
103
|
system_prompt = test_create_prompt.format(
|
|
104
104
|
name=pipe_name,
|
|
@@ -107,24 +107,26 @@ async def test_create(name_or_filename: str, prompt: Optional[str], folder: str)
|
|
|
107
107
|
)
|
|
108
108
|
config = CLIConfig.get_project_config(folder)
|
|
109
109
|
user_token = config.get_user_token()
|
|
110
|
+
if not user_token:
|
|
111
|
+
raise CLIException(FeedbackManager.error(message="No user token found"))
|
|
110
112
|
llm = LLM(user_token=user_token, client=config.get_client())
|
|
111
113
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
tests_content = parse_xml(
|
|
114
|
+
response_llm = await llm.ask(system_prompt=system_prompt, prompt=prompt)
|
|
115
|
+
response_xml = extract_xml(response_llm, "response")
|
|
116
|
+
tests_content = parse_xml(response_xml, "test")
|
|
115
117
|
|
|
116
118
|
tests: List[Dict[str, Any]] = []
|
|
117
119
|
for test_content in tests_content:
|
|
118
|
-
test = {}
|
|
120
|
+
test: Dict[str, Any] = {}
|
|
119
121
|
test["name"] = extract_xml(test_content, "name")
|
|
120
122
|
test["description"] = extract_xml(test_content, "description")
|
|
121
|
-
|
|
122
|
-
test["parameters"] =
|
|
123
|
+
parameters_api = extract_xml(test_content, "parameters")
|
|
124
|
+
test["parameters"] = parameters_api.split("?")[1] if "?" in parameters_api else parameters_api
|
|
123
125
|
test["expected_result"] = ""
|
|
124
126
|
|
|
125
127
|
response = None
|
|
126
128
|
try:
|
|
127
|
-
response = await client
|
|
129
|
+
response = await get_pipe_data(client, pipe_name=pipe_name, test_params=test["parameters"])
|
|
128
130
|
except Exception:
|
|
129
131
|
pass
|
|
130
132
|
|
|
@@ -181,7 +183,7 @@ async def test_update(pipe: str, folder: str) -> None:
|
|
|
181
183
|
test_params = test["parameters"].split("?")[1] if "?" in test["parameters"] else test["parameters"]
|
|
182
184
|
response = None
|
|
183
185
|
try:
|
|
184
|
-
response = await client
|
|
186
|
+
response = await get_pipe_data(client, pipe_name=pipe_name, test_params=test_params)
|
|
185
187
|
except Exception:
|
|
186
188
|
continue
|
|
187
189
|
|
|
@@ -221,20 +223,21 @@ async def run_tests(name: Tuple[str, ...], folder: str) -> None:
|
|
|
221
223
|
client = await get_tinybird_local_client(os.path.abspath(folder))
|
|
222
224
|
paths = [Path(n) for n in name]
|
|
223
225
|
endpoints = [f"./tests/{p.stem}.yaml" for p in paths]
|
|
224
|
-
test_files:
|
|
226
|
+
test_files: List[str] = endpoints if len(endpoints) > 0 else glob.glob("./tests/**/*.y*ml", recursive=True)
|
|
225
227
|
|
|
226
228
|
async def run_test(test_file):
|
|
227
229
|
test_file_path = Path(test_file)
|
|
228
230
|
click.echo(FeedbackManager.info(message=f"\n* {test_file_path.stem}{test_file_path.suffix}"))
|
|
229
231
|
test_file_content = yaml.safe_load(test_file_path.read_text())
|
|
232
|
+
|
|
230
233
|
for test in test_file_content:
|
|
231
234
|
try:
|
|
232
235
|
test_params = test["parameters"].split("?")[1] if "?" in test["parameters"] else test["parameters"]
|
|
233
236
|
response = None
|
|
234
237
|
try:
|
|
235
|
-
response = await client
|
|
238
|
+
response = await get_pipe_data(client, pipe_name=test_file_path.stem, test_params=test_params)
|
|
236
239
|
except Exception:
|
|
237
|
-
|
|
240
|
+
continue
|
|
238
241
|
|
|
239
242
|
expected_result = response.text
|
|
240
243
|
if response.status_code >= 400:
|
|
@@ -271,3 +274,19 @@ async def run_tests(name: Tuple[str, ...], folder: str) -> None:
|
|
|
271
274
|
exit(1)
|
|
272
275
|
else:
|
|
273
276
|
click.echo(FeedbackManager.success(message=f"\n✓ {test_count}/{test_count} passed"))
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
async def get_pipe_data(client, pipe_name: str, test_params: str) -> Response:
|
|
280
|
+
pipe = await client._req(f"/v0/pipes/{pipe_name}")
|
|
281
|
+
output_node = next(
|
|
282
|
+
(node for node in pipe["nodes"] if node["node_type"] != "default" and node["node_type"] != "standard"),
|
|
283
|
+
{"name": "not_found"},
|
|
284
|
+
)
|
|
285
|
+
if output_node["node_type"] == "endpoint":
|
|
286
|
+
return await client._req_raw(f"/v0/pipes/{pipe_name}.ndjson?{test_params}")
|
|
287
|
+
|
|
288
|
+
params = {
|
|
289
|
+
"q": output_node["sql"],
|
|
290
|
+
"pipeline": pipe_name,
|
|
291
|
+
}
|
|
292
|
+
return await client._req_raw(f"""/v0/sql?{urllib.parse.urlencode(params)}&{test_params}""")
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
from os import getcwd
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
import click
|
|
8
|
+
|
|
9
|
+
from tinybird.client import TinyB
|
|
10
|
+
from tinybird.prompts import mock_prompt, update_prompt
|
|
11
|
+
from tinybird.tb.modules.cli import cli
|
|
12
|
+
from tinybird.tb.modules.common import check_user_token_with_client, coro, generate_datafile
|
|
13
|
+
from tinybird.tb.modules.config import CLIConfig
|
|
14
|
+
from tinybird.tb.modules.datafile.fixture import build_fixture_name, persist_fixture
|
|
15
|
+
from tinybird.tb.modules.exceptions import CLIException
|
|
16
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
17
|
+
from tinybird.tb.modules.llm import LLM
|
|
18
|
+
from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
|
|
19
|
+
from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@cli.command()
|
|
23
|
+
@click.argument("prompt")
|
|
24
|
+
@click.option(
|
|
25
|
+
"--folder",
|
|
26
|
+
default=".",
|
|
27
|
+
type=click.Path(exists=False, file_okay=False),
|
|
28
|
+
help="Folder where project files will be placed",
|
|
29
|
+
)
|
|
30
|
+
@coro
|
|
31
|
+
async def update(
|
|
32
|
+
prompt: str,
|
|
33
|
+
folder: str,
|
|
34
|
+
) -> None:
|
|
35
|
+
"""Update resources in the project."""
|
|
36
|
+
folder = folder or getcwd()
|
|
37
|
+
folder_path = Path(folder)
|
|
38
|
+
if not folder_path.exists():
|
|
39
|
+
folder_path.mkdir()
|
|
40
|
+
|
|
41
|
+
try:
|
|
42
|
+
config = CLIConfig.get_project_config(folder)
|
|
43
|
+
tb_client = config.get_client()
|
|
44
|
+
user_token: Optional[str] = None
|
|
45
|
+
try:
|
|
46
|
+
user_token = config.get_user_token()
|
|
47
|
+
if not user_token:
|
|
48
|
+
raise CLIException("No user token found")
|
|
49
|
+
await check_user_token_with_client(tb_client, token=user_token)
|
|
50
|
+
except Exception as e:
|
|
51
|
+
click.echo(
|
|
52
|
+
FeedbackManager.error(message=f"This action requires authentication. Run 'tb login' first. Error: {e}")
|
|
53
|
+
)
|
|
54
|
+
return
|
|
55
|
+
|
|
56
|
+
local_client = await get_tinybird_local_client(folder)
|
|
57
|
+
|
|
58
|
+
click.echo(FeedbackManager.highlight(message="\n» Updating resources..."))
|
|
59
|
+
datasources_updated = await update_resources(tb_client, user_token, prompt, folder)
|
|
60
|
+
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
61
|
+
|
|
62
|
+
if datasources_updated and user_token:
|
|
63
|
+
click.echo(FeedbackManager.highlight(message="\n» Generating fixtures..."))
|
|
64
|
+
|
|
65
|
+
datasource_files = [f for f in os.listdir(Path(folder) / "datasources") if f.endswith(".datasource")]
|
|
66
|
+
for datasource_file in datasource_files:
|
|
67
|
+
datasource_path = Path(folder) / "datasources" / datasource_file
|
|
68
|
+
llm = LLM(user_token=user_token, client=tb_client)
|
|
69
|
+
datasource_name = datasource_path.stem
|
|
70
|
+
datasource_content = datasource_path.read_text()
|
|
71
|
+
has_json_path = "`json:" in datasource_content
|
|
72
|
+
if has_json_path:
|
|
73
|
+
prompt = f"<datasource_schema>{datasource_content}</datasource_schema>\n<user_input>{prompt}</user_input>"
|
|
74
|
+
response = await llm.ask(system_prompt=mock_prompt(rows=20), prompt=prompt)
|
|
75
|
+
sql = extract_xml(response, "sql")
|
|
76
|
+
sql = sql.split("FORMAT")[0]
|
|
77
|
+
result = await local_client.query(f"{sql} FORMAT JSON")
|
|
78
|
+
data = result.get("data", [])
|
|
79
|
+
fixture_name = build_fixture_name(
|
|
80
|
+
datasource_path.absolute().as_posix(), datasource_name, datasource_content
|
|
81
|
+
)
|
|
82
|
+
if data:
|
|
83
|
+
persist_fixture(fixture_name, data, folder)
|
|
84
|
+
click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}"))
|
|
85
|
+
|
|
86
|
+
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
87
|
+
except Exception as e:
|
|
88
|
+
click.echo(FeedbackManager.error(message=f"Error: {str(e)}"))
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
async def update_resources(
|
|
92
|
+
tb_client: TinyB,
|
|
93
|
+
user_token: str,
|
|
94
|
+
prompt: str,
|
|
95
|
+
folder: str,
|
|
96
|
+
):
|
|
97
|
+
datasource_paths = [
|
|
98
|
+
Path(folder) / "datasources" / f for f in os.listdir(Path(folder) / "datasources") if f.endswith(".datasource")
|
|
99
|
+
]
|
|
100
|
+
pipes_paths = [
|
|
101
|
+
Path(folder) / "endpoints" / f for f in os.listdir(Path(folder) / "endpoints") if f.endswith(".pipe")
|
|
102
|
+
]
|
|
103
|
+
resources_xml = "\n".join(
|
|
104
|
+
[
|
|
105
|
+
f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
|
|
106
|
+
for resource_type, resource_name, resource_content in [
|
|
107
|
+
("datasource", ds.stem, ds.read_text()) for ds in datasource_paths
|
|
108
|
+
]
|
|
109
|
+
+ [
|
|
110
|
+
(
|
|
111
|
+
"pipe",
|
|
112
|
+
pipe.stem,
|
|
113
|
+
pipe.read_text(),
|
|
114
|
+
)
|
|
115
|
+
for pipe in pipes_paths
|
|
116
|
+
]
|
|
117
|
+
]
|
|
118
|
+
)
|
|
119
|
+
llm = LLM(user_token=user_token, client=tb_client)
|
|
120
|
+
result = await llm.ask(system_prompt=update_prompt(resources_xml), prompt=prompt)
|
|
121
|
+
result = extract_xml(result, "response")
|
|
122
|
+
resources = parse_xml(result, "resource")
|
|
123
|
+
datasources = []
|
|
124
|
+
pipes = []
|
|
125
|
+
for resource_xml in resources:
|
|
126
|
+
resource_type = extract_xml(resource_xml, "type")
|
|
127
|
+
name = extract_xml(resource_xml, "name")
|
|
128
|
+
content = extract_xml(resource_xml, "content")
|
|
129
|
+
resource = {
|
|
130
|
+
"name": name,
|
|
131
|
+
"content": content,
|
|
132
|
+
}
|
|
133
|
+
if resource_type.lower() == "datasource":
|
|
134
|
+
datasources.append(resource)
|
|
135
|
+
elif resource_type.lower() == "pipe":
|
|
136
|
+
pipes.append(resource)
|
|
137
|
+
|
|
138
|
+
for ds in datasources:
|
|
139
|
+
content = ds["content"].replace("```", "")
|
|
140
|
+
filename = f"{ds['name']}.datasource"
|
|
141
|
+
generate_datafile(
|
|
142
|
+
content,
|
|
143
|
+
filename=filename,
|
|
144
|
+
data=None,
|
|
145
|
+
_format="ndjson",
|
|
146
|
+
force=True,
|
|
147
|
+
folder=folder,
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
for pipe in pipes:
|
|
151
|
+
content = pipe["content"].replace("```", "")
|
|
152
|
+
generate_pipe_file(pipe["name"], content, folder)
|
|
153
|
+
|
|
154
|
+
return len(datasources) > 0
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def generate_pipe_file(name: str, content: str, folder: str):
|
|
158
|
+
def is_copy(content: str) -> bool:
|
|
159
|
+
return re.search(r"TYPE copy", content, re.IGNORECASE) is not None
|
|
160
|
+
|
|
161
|
+
def is_materialization(content: str) -> bool:
|
|
162
|
+
return re.search(r"TYPE materialized", content, re.IGNORECASE) is not None
|
|
163
|
+
|
|
164
|
+
def is_sink(content: str) -> bool:
|
|
165
|
+
return re.search(r"TYPE sink", content, re.IGNORECASE) is not None
|
|
166
|
+
|
|
167
|
+
if is_copy(content):
|
|
168
|
+
pathname = "copies"
|
|
169
|
+
elif is_materialization(content):
|
|
170
|
+
pathname = "materializations"
|
|
171
|
+
elif is_sink(content):
|
|
172
|
+
pathname = "sinks"
|
|
173
|
+
else:
|
|
174
|
+
pathname = "endpoints"
|
|
175
|
+
|
|
176
|
+
base = Path(folder) / pathname
|
|
177
|
+
if not base.exists():
|
|
178
|
+
base = Path()
|
|
179
|
+
f = base / (f"{name}.pipe")
|
|
180
|
+
with open(f"{f}", "w") as file:
|
|
181
|
+
file.write(content)
|
|
182
|
+
click.echo(FeedbackManager.info_file_created(file=f.relative_to(folder)))
|
tinybird/tb/modules/watch.py
CHANGED
|
@@ -1,10 +1,18 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
import os
|
|
3
3
|
import time
|
|
4
|
+
from pathlib import Path
|
|
4
5
|
from typing import Any, Callable, List, Optional, Union
|
|
5
6
|
|
|
6
7
|
import click
|
|
7
|
-
from watchdog.events import
|
|
8
|
+
from watchdog.events import (
|
|
9
|
+
DirDeletedEvent,
|
|
10
|
+
DirMovedEvent,
|
|
11
|
+
FileDeletedEvent,
|
|
12
|
+
FileMovedEvent,
|
|
13
|
+
FileSystemEventHandler,
|
|
14
|
+
PatternMatchingEventHandler,
|
|
15
|
+
)
|
|
8
16
|
from watchdog.observers import Observer
|
|
9
17
|
|
|
10
18
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
@@ -110,3 +118,70 @@ def watch_files(
|
|
|
110
118
|
observer.stop()
|
|
111
119
|
|
|
112
120
|
observer.join()
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class WatchProjectHandler(PatternMatchingEventHandler):
|
|
124
|
+
def __init__(self, shell: Shell, folder: str, process: Callable[[List[str]], None]):
|
|
125
|
+
self.shell = shell
|
|
126
|
+
self.process = process
|
|
127
|
+
super().__init__(
|
|
128
|
+
patterns=[f"{folder}/**/*.datasource", f"{folder}/**/*.pipe"],
|
|
129
|
+
ignore_patterns=[f"{folder}/vendor/"],
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
def should_process(self, event: Any) -> Optional[str]:
|
|
133
|
+
if event.is_directory:
|
|
134
|
+
return None
|
|
135
|
+
|
|
136
|
+
if os.path.exists(event.src_path):
|
|
137
|
+
return event.src_path
|
|
138
|
+
|
|
139
|
+
if os.path.exists(event.dest_path):
|
|
140
|
+
return event.dest_path
|
|
141
|
+
|
|
142
|
+
return event.src_path
|
|
143
|
+
|
|
144
|
+
def _process(self) -> None:
|
|
145
|
+
click.echo(FeedbackManager.highlight(message="» Rebuilding..."))
|
|
146
|
+
time_start = time.time()
|
|
147
|
+
self.process()
|
|
148
|
+
time_end = time.time()
|
|
149
|
+
elapsed_time = time_end - time_start
|
|
150
|
+
click.echo(
|
|
151
|
+
FeedbackManager.success(message="\n✓ ")
|
|
152
|
+
+ FeedbackManager.gray(message=f"Rebuild completed in {elapsed_time:.1f}s")
|
|
153
|
+
)
|
|
154
|
+
self.shell.reprint_prompt()
|
|
155
|
+
|
|
156
|
+
def on_modified(self, event: Any) -> None:
|
|
157
|
+
if path := self.should_process(event):
|
|
158
|
+
filename = Path(path).name
|
|
159
|
+
click.echo(FeedbackManager.highlight(message=f"\n\n⟲ Changes detected in {filename}\n"))
|
|
160
|
+
self._process()
|
|
161
|
+
|
|
162
|
+
def on_deleted(self, event: Union[DirDeletedEvent, FileDeletedEvent]) -> None:
|
|
163
|
+
filename = Path(event.src_path).name
|
|
164
|
+
if event.is_directory:
|
|
165
|
+
click.echo(FeedbackManager.highlight(message=f"\n\n⟲ Deleted directory: {filename}\n"))
|
|
166
|
+
else:
|
|
167
|
+
click.echo(FeedbackManager.highlight(message=f"\n\n⟲ Deleted file: {filename}\n"))
|
|
168
|
+
self._process()
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def watch_project(
|
|
172
|
+
shell: Shell,
|
|
173
|
+
process: Callable,
|
|
174
|
+
folder: str,
|
|
175
|
+
) -> None:
|
|
176
|
+
event_handler = WatchProjectHandler(shell=shell, folder=folder, process=process)
|
|
177
|
+
observer = Observer()
|
|
178
|
+
observer.schedule(event_handler, path=folder, recursive=True)
|
|
179
|
+
observer.start()
|
|
180
|
+
|
|
181
|
+
try:
|
|
182
|
+
while True:
|
|
183
|
+
time.sleep(1)
|
|
184
|
+
except KeyboardInterrupt:
|
|
185
|
+
observer.stop()
|
|
186
|
+
|
|
187
|
+
observer.join()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: tinybird
|
|
3
|
-
Version: 0.0.1.
|
|
3
|
+
Version: 0.0.1.dev31
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/cli/introduction.html
|
|
6
6
|
Author: Tinybird
|
|
@@ -8,6 +8,7 @@ Author-email: support@tinybird.co
|
|
|
8
8
|
Requires-Python: >=3.9, <3.13
|
|
9
9
|
Description-Content-Type: text/x-rst
|
|
10
10
|
Requires-Dist: aiofiles (==24.1.0)
|
|
11
|
+
Requires-Dist: anthropic (==0.42.0)
|
|
11
12
|
Requires-Dist: click (<8.2,>=8.1.6)
|
|
12
13
|
Requires-Dist: clickhouse-toolset (==0.33.dev0)
|
|
13
14
|
Requires-Dist: colorama (==0.4.6)
|
|
@@ -1,58 +1,59 @@
|
|
|
1
|
-
tinybird/__cli__.py,sha256=
|
|
2
|
-
tinybird/client.py,sha256=
|
|
3
|
-
tinybird/config.py,sha256=
|
|
1
|
+
tinybird/__cli__.py,sha256=esPl5QDTzuQgHe5FuxWLm-fURFigGGwjnYLh9GuWUw4,232
|
|
2
|
+
tinybird/client.py,sha256=P-bc7s7SbrnzkZkSsb_TO58q4Jxy-bBD5S5qpYiXiZg,51517
|
|
3
|
+
tinybird/config.py,sha256=ENRNyEMXHj_P882o31iFz0hTveziLabVRrxiWE5RRBE,6233
|
|
4
4
|
tinybird/connectors.py,sha256=lkpVSUmSuViEZBa4QjTK7YmPHUop0a5UFoTrSmlVq6k,15244
|
|
5
|
-
tinybird/context.py,sha256=
|
|
5
|
+
tinybird/context.py,sha256=eeFneyn_oTUoTIfst59CnSkskchvnWuRvyI7UKRFq14,1407
|
|
6
6
|
tinybird/datatypes.py,sha256=XNypumfqNjsvLJ5iNXnbVHRvAJe0aQwI3lS6Cxox-e0,10979
|
|
7
7
|
tinybird/feedback_manager.py,sha256=cNUbt0Jxim02UiIdlyP12DJfXfFFzxDCfJK9XRWZ9A0,67488
|
|
8
8
|
tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
|
|
9
|
-
tinybird/prompts.py,sha256=
|
|
9
|
+
tinybird/prompts.py,sha256=xrZMZykphO9DRfW5yvCJNZMhPq0v1zab3d9jKy9plhU,24064
|
|
10
10
|
tinybird/sql.py,sha256=eulpRe05ZFrKFrxYawgxDxxrktFE8uL6hSL1gHIWKyg,46166
|
|
11
|
-
tinybird/sql_template.py,sha256=
|
|
11
|
+
tinybird/sql_template.py,sha256=GmMLAI10MTqjQo9qztuQHLRWs67teozsWDxUBdvkAn4,93668
|
|
12
12
|
tinybird/sql_template_fmt.py,sha256=1z-PuqSZXtzso8Z_mPqUc-NxIxUrNUcVIPezNieZk-M,10196
|
|
13
13
|
tinybird/sql_toolset.py,sha256=HAo6xtCjlbhikdpK6xfMC_RvEG6uzRropwhKSHEkIIc,14343
|
|
14
14
|
tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
|
|
15
15
|
tinybird/tornado_template.py,sha256=oflXyoL2LSCegvl6bAzqw2JIqRaN5WPjhYYDtQcfuOE,41869
|
|
16
16
|
tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj0,3890
|
|
17
17
|
tinybird/ch_utils/engine.py,sha256=OXkBhlzGjZotjD0vaT-rFIbSGV4tpiHxE8qO_ip0SyQ,40454
|
|
18
|
-
tinybird/tb/__cli__.py,sha256=
|
|
19
|
-
tinybird/tb/cli.py,sha256=
|
|
18
|
+
tinybird/tb/__cli__.py,sha256=SgyAWbfPDLAaYyoq9rFOPpcXyLPQqCPh2m8Bt_6lxaI,251
|
|
19
|
+
tinybird/tb/cli.py,sha256=dmrf8O9cqBuOYUmVBbNDSNQpHUhBg6IkBPaISAj4kHo,923
|
|
20
20
|
tinybird/tb/modules/auth.py,sha256=EzRWFmwRkXNhUmRaruEVFLdkbUg8xMSix0cAWl5D4Jg,9029
|
|
21
|
-
tinybird/tb/modules/build.py,sha256=
|
|
22
|
-
tinybird/tb/modules/
|
|
21
|
+
tinybird/tb/modules/build.py,sha256=J_YCaJIkvoM1CMpsYQ4gM1LinaqJmJmvfcmhxEmWq_g,3725
|
|
22
|
+
tinybird/tb/modules/build_client.py,sha256=SoGaGPBYT_0BoRp2lxNyBBvQQ1GFAMpx_o62jb91DUI,8046
|
|
23
23
|
tinybird/tb/modules/cicd.py,sha256=SjCyvvy0WUnsjFs2biwwXvcf0Ddpmghhd8-SnMyfsRM,5355
|
|
24
|
-
tinybird/tb/modules/cli.py,sha256=
|
|
25
|
-
tinybird/tb/modules/common.py,sha256=
|
|
26
|
-
tinybird/tb/modules/config.py,sha256=
|
|
24
|
+
tinybird/tb/modules/cli.py,sha256=i5KQF1kJqd3lflYl0e3tnKrMZBIAOlMF1T5pxa6e7tM,22124
|
|
25
|
+
tinybird/tb/modules/common.py,sha256=e4U7AT0dUBG6O-7Iq2CVN1UHPd6-ZCFucyW0L5gBi4g,70592
|
|
26
|
+
tinybird/tb/modules/config.py,sha256=mie3oMVTf5YOUFEiLs88P16U4LkJafJjSpjwyAkFHog,10979
|
|
27
27
|
tinybird/tb/modules/connection.py,sha256=FhDM-OAnLN2epbO2YonpjJQhHqBjyuanBsZmKlDXrqg,28679
|
|
28
|
-
tinybird/tb/modules/create.py,sha256=
|
|
29
|
-
tinybird/tb/modules/datasource.py,sha256=
|
|
28
|
+
tinybird/tb/modules/create.py,sha256=HclKekBKTaobVQeYq9ooSmuBwlrebi-YsFOFJJMGTZs,10860
|
|
29
|
+
tinybird/tb/modules/datasource.py,sha256=PmIoDSLqBNHQvguX9t1T3z9lsG3HB67cmwhQO5_8Lc4,32918
|
|
30
30
|
tinybird/tb/modules/deploy.py,sha256=j1MuiGsWplG1BQs8eG6-STcYyzvHqVESPCOo6P-dcF8,9242
|
|
31
31
|
tinybird/tb/modules/exceptions.py,sha256=4A2sSjCEqKUMqpP3WI00zouCWW4uLaghXXLZBSw04mY,3363
|
|
32
32
|
tinybird/tb/modules/feedback_manager.py,sha256=e8tqehRR0Buhs8O0n8N2Sg2vnnBVb1NLtnZqkPrYD_A,68379
|
|
33
33
|
tinybird/tb/modules/fmt.py,sha256=poh6_cwVGSf-sBu6LKWuO2TANL_J8Sgm25sPpwxa3Aw,3558
|
|
34
34
|
tinybird/tb/modules/job.py,sha256=956Pj8BEEsiD2GZsV9RKKVM3I_CveOLgS82lykO5ukk,2963
|
|
35
|
-
tinybird/tb/modules/llm.py,sha256=
|
|
35
|
+
tinybird/tb/modules/llm.py,sha256=mguBJaTJ74a5mQXcBQwbCjKugfvcFm_Wzg2e_hSkooQ,953
|
|
36
36
|
tinybird/tb/modules/llm_utils.py,sha256=hIfBU7vMUHUt25pljim3WdZdJTNr9hDo3mHezqI5B7w,766
|
|
37
|
-
tinybird/tb/modules/local.py,sha256=
|
|
38
|
-
tinybird/tb/modules/local_common.py,sha256=
|
|
39
|
-
tinybird/tb/modules/login.py,sha256=
|
|
40
|
-
tinybird/tb/modules/mock.py,sha256=
|
|
37
|
+
tinybird/tb/modules/local.py,sha256=bSBBk2A0yj4fQHxsdzbZAH1Q_xrqT7vFOIPoz7TNabo,5260
|
|
38
|
+
tinybird/tb/modules/local_common.py,sha256=Na4zHQP2H9oE840KYUn69_f0PcqExQtaKFikaiNf43s,2296
|
|
39
|
+
tinybird/tb/modules/login.py,sha256=0cS-f3MsQFHc6xjw8FRWJm4EJBH9C7Ri68EcO_tiwes,6508
|
|
40
|
+
tinybird/tb/modules/mock.py,sha256=eSfD-m4w5NLGxjfGRlYplQhC0FKpanysIweG_8GsbNE,3665
|
|
41
41
|
tinybird/tb/modules/pipe.py,sha256=eYmMBiSj1Ur_hXUs74YZ9mCSAyiICDmkuKuTemlxPUY,27018
|
|
42
42
|
tinybird/tb/modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
43
|
-
tinybird/tb/modules/shell.py,sha256=
|
|
43
|
+
tinybird/tb/modules/shell.py,sha256=84WiRppf4bLZSSEscSqFov-7RpDEQ8u0DFovqlnXauk,13093
|
|
44
44
|
tinybird/tb/modules/table.py,sha256=4XrtjM-N0zfNtxVkbvLDQQazno1EPXnxTyo7llivfXk,11035
|
|
45
45
|
tinybird/tb/modules/tag.py,sha256=anPmMUBc-TbFovlpFi8GPkKA18y7Y0GczMsMms5TZsU,3502
|
|
46
46
|
tinybird/tb/modules/telemetry.py,sha256=iEGnMuCuNhvF6ln__j6X9MSTwL_0Hm-GgFHHHvhfknk,10466
|
|
47
|
-
tinybird/tb/modules/test.py,sha256=
|
|
47
|
+
tinybird/tb/modules/test.py,sha256=2UhRiMRPP2vEsa2ayM_ujLc1cB8gHkKHCLJ50GbkQxc,11577
|
|
48
48
|
tinybird/tb/modules/token.py,sha256=AePr-QMv_vtWwZDWQ92Zp0kPrCjze61i4npiPhoLMZg,12717
|
|
49
|
-
tinybird/tb/modules/
|
|
49
|
+
tinybird/tb/modules/update.py,sha256=wXzLhBzKD1kWEN1HejOVkKpFQqomZGfP1AiCvE-K04A,6827
|
|
50
|
+
tinybird/tb/modules/watch.py,sha256=na1H0lVCyMTCgzm2h8FHyKWhWN5poAehWgJeXT2rLXg,6281
|
|
50
51
|
tinybird/tb/modules/workspace.py,sha256=6icAgnTvfL3d1kx4L1Z1cGXCD_2Yx0fNRjbZHNxRbYc,10927
|
|
51
52
|
tinybird/tb/modules/workspace_members.py,sha256=Ai6iCOzXX1zQ8q9iXIFSFHsBJlT-8Q28DaG5Ie-UweY,8726
|
|
52
|
-
tinybird/tb/modules/datafile/build.py,sha256=
|
|
53
|
+
tinybird/tb/modules/datafile/build.py,sha256=pwgsIuvHwb2cdsl3IWOAPyj6S9vB3jn_BXGRcKT7I2Y,57577
|
|
53
54
|
tinybird/tb/modules/datafile/build_common.py,sha256=IXl-Z51zUi1dypV7meNenX0iu2UmowNeqgG6WHyMHlk,4562
|
|
54
55
|
tinybird/tb/modules/datafile/build_datasource.py,sha256=4aP8_DYCRGghXntZSeWDNJxjps1QRVa7WHoYCzQwQts,17355
|
|
55
|
-
tinybird/tb/modules/datafile/build_pipe.py,sha256=
|
|
56
|
+
tinybird/tb/modules/datafile/build_pipe.py,sha256=Jgv3YKIvMfjPiSIdw1k2mpaoDdAWMiMRaSHwRgyI97E,28258
|
|
56
57
|
tinybird/tb/modules/datafile/common.py,sha256=TSJozXsXzxvkAC_N7dV-Iv_cy2AIGHUNtCYgwlQAbtU,75762
|
|
57
58
|
tinybird/tb/modules/datafile/diff.py,sha256=-0J7PsBO64T7LOZSkZ4ZFHHCPvT7cKItnJkbz2PkndU,6754
|
|
58
59
|
tinybird/tb/modules/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
|
|
@@ -72,8 +73,8 @@ tinybird/tb_cli_modules/config.py,sha256=6u6B5QCdiQLbJkCkwtnKGs9H3nP-KXXhC75mF7B
|
|
|
72
73
|
tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
|
|
73
74
|
tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
74
75
|
tinybird/tb_cli_modules/telemetry.py,sha256=iEGnMuCuNhvF6ln__j6X9MSTwL_0Hm-GgFHHHvhfknk,10466
|
|
75
|
-
tinybird-0.0.1.
|
|
76
|
-
tinybird-0.0.1.
|
|
77
|
-
tinybird-0.0.1.
|
|
78
|
-
tinybird-0.0.1.
|
|
79
|
-
tinybird-0.0.1.
|
|
76
|
+
tinybird-0.0.1.dev31.dist-info/METADATA,sha256=nl0L_9CisXRnrHeEukw_5Z733F7vlRvHMVgHPdVrcL0,2482
|
|
77
|
+
tinybird-0.0.1.dev31.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
|
|
78
|
+
tinybird-0.0.1.dev31.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
|
|
79
|
+
tinybird-0.0.1.dev31.dist-info/top_level.txt,sha256=pgw6AzERHBcW3YTi2PW4arjxLkulk2msOz_SomfOEuc,45
|
|
80
|
+
tinybird-0.0.1.dev31.dist-info/RECORD,,
|
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
import asyncio
|
|
2
|
-
import glob
|
|
3
|
-
import json
|
|
4
|
-
import logging
|
|
5
|
-
from pathlib import Path
|
|
6
|
-
from typing import List
|
|
7
|
-
|
|
8
|
-
import click
|
|
9
|
-
import requests
|
|
10
|
-
|
|
11
|
-
from tinybird.tb.modules.cli import cli
|
|
12
|
-
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
13
|
-
from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def project_files(project_path: Path) -> List[str]:
|
|
17
|
-
project_file_extensions = ("datasource", "pipe")
|
|
18
|
-
project_files = []
|
|
19
|
-
for extension in project_file_extensions:
|
|
20
|
-
for project_file in glob.glob(f"{project_path}/**/*.{extension}", recursive=True):
|
|
21
|
-
logging.debug(f"Found project file: {project_file}")
|
|
22
|
-
project_files.append(project_file)
|
|
23
|
-
return project_files
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
@cli.command()
|
|
27
|
-
@click.argument("project_path", type=click.Path(exists=True), default=Path.cwd())
|
|
28
|
-
def build_server(project_path: Path) -> None:
|
|
29
|
-
"""
|
|
30
|
-
Validate and build the project server side.
|
|
31
|
-
"""
|
|
32
|
-
|
|
33
|
-
MULTIPART_BOUNDARY_DATA_PROJECT = "data_project://"
|
|
34
|
-
DATAFILE_TYPE_TO_CONTENT_TYPE = {
|
|
35
|
-
".datasource": "text/plain",
|
|
36
|
-
".pipe": "text/plain",
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
tb_client = asyncio.run(get_tinybird_local_client(str(project_path)))
|
|
40
|
-
TINYBIRD_API_URL = tb_client.host + "/v1/build"
|
|
41
|
-
TINYBIRD_API_KEY = tb_client.token
|
|
42
|
-
|
|
43
|
-
files = [
|
|
44
|
-
("context://", ("cli-version", "1.0.0", "text/plain")),
|
|
45
|
-
]
|
|
46
|
-
fds = []
|
|
47
|
-
for file_path in project_files(project_path):
|
|
48
|
-
relative_path = str(Path(file_path).relative_to(project_path))
|
|
49
|
-
fd = open(file_path, "rb")
|
|
50
|
-
fds.append(fd)
|
|
51
|
-
content_type = DATAFILE_TYPE_TO_CONTENT_TYPE.get(Path(file_path).suffix, "application/unknown")
|
|
52
|
-
files.append((MULTIPART_BOUNDARY_DATA_PROJECT, (relative_path, fd, content_type)))
|
|
53
|
-
|
|
54
|
-
try:
|
|
55
|
-
HEADERS = {"Authorization": f"Bearer {TINYBIRD_API_KEY}"}
|
|
56
|
-
|
|
57
|
-
r = requests.post(TINYBIRD_API_URL, files=files, headers=HEADERS)
|
|
58
|
-
result = r.json()
|
|
59
|
-
logging.debug(json.dumps(result, indent=2))
|
|
60
|
-
|
|
61
|
-
build_result = result.get("result")
|
|
62
|
-
if build_result == "success":
|
|
63
|
-
click.echo(FeedbackManager.success(message="Build completed successfully"))
|
|
64
|
-
elif build_result == "failed":
|
|
65
|
-
click.echo(FeedbackManager.error(message="Build failed"))
|
|
66
|
-
build_errors = result.get("errors")
|
|
67
|
-
for build_error in build_errors:
|
|
68
|
-
click.echo(
|
|
69
|
-
FeedbackManager.error(message=f"{build_error.get('filename')}\n\n{build_error.get('error')}")
|
|
70
|
-
)
|
|
71
|
-
else:
|
|
72
|
-
click.echo(FeedbackManager.error(message=f"Unknown build result. Error: {result.get('error')}"))
|
|
73
|
-
finally:
|
|
74
|
-
for fd in fds:
|
|
75
|
-
fd.close()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|