tinybird 0.0.1.dev174__py3-none-any.whl → 0.0.1.dev176__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/prompts.py +2 -2
- tinybird/sql_template.py +3 -3
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/modules/create.py +162 -142
- tinybird/tb/modules/datafile/common.py +79 -15
- tinybird/tb/modules/datafile/format_common.py +1 -1
- tinybird/tb/modules/datasource.py +4 -4
- tinybird/tb/modules/tinyunit/tinyunit.py +1 -1
- {tinybird-0.0.1.dev174.dist-info → tinybird-0.0.1.dev176.dist-info}/METADATA +2 -2
- {tinybird-0.0.1.dev174.dist-info → tinybird-0.0.1.dev176.dist-info}/RECORD +13 -13
- {tinybird-0.0.1.dev174.dist-info → tinybird-0.0.1.dev176.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev174.dist-info → tinybird-0.0.1.dev176.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev174.dist-info → tinybird-0.0.1.dev176.dist-info}/top_level.txt +0 -0
tinybird/prompts.py
CHANGED
|
@@ -461,8 +461,7 @@ Use the following feedback to regenerate the response:
|
|
|
461
461
|
"""
|
|
462
462
|
|
|
463
463
|
return f"""
|
|
464
|
-
Given the schema for a Tinybird datasource, return a
|
|
465
|
-
|
|
464
|
+
Given the schema for a Tinybird datasource, return a valid clickhouse sql query to generate some random data that matches that schema.
|
|
466
465
|
Response format MUST be just a valid clickhouse sql query.
|
|
467
466
|
|
|
468
467
|
<example>
|
|
@@ -722,6 +721,7 @@ datasource_instructions = """
|
|
|
722
721
|
- Use MergeTree engine by default.
|
|
723
722
|
- Use AggregatingMergeTree engine when the datasource is the target of a materialized pipe.
|
|
724
723
|
- Use always json paths to define the schema. Example: `user_id` String `json:$.user_id`,
|
|
724
|
+
- Array columns are supported with a special syntax. Example: `items` Array(String) `json:$.items[:]`
|
|
725
725
|
- If the datasource is using an S3 or GCS connection, they need to set IMPORT_CONNECTION_NAME, IMPORT_BUCKET_URI and IMPORT_SCHEDULE (GCS @on-demand only, S3 supports @auto too)
|
|
726
726
|
</datasource_file_instructions>
|
|
727
727
|
"""
|
tinybird/sql_template.py
CHANGED
|
@@ -1657,7 +1657,7 @@ def get_var_data(content, node_id=None):
|
|
|
1657
1657
|
# It will be overriden in later definitions or left as is otherwise.
|
|
1658
1658
|
# args[0] check is used to avoid adding unnamed parameters found in
|
|
1659
1659
|
# templates like: `split_to_array('')`
|
|
1660
|
-
if
|
|
1660
|
+
if args and isinstance(args[0], list):
|
|
1661
1661
|
raise ValueError(f'"{args[0]}" can not be used as a variable name')
|
|
1662
1662
|
if len(args) > 0 and args[0] not in vars and args[0]:
|
|
1663
1663
|
vars[args[0]] = {
|
|
@@ -1669,7 +1669,7 @@ def get_var_data(content, node_id=None):
|
|
|
1669
1669
|
if "default" not in kwargs:
|
|
1670
1670
|
default = kwargs.get("default", args[2] if len(args) > 2 and args[2] else None)
|
|
1671
1671
|
kwargs["default"] = check_default_value(default)
|
|
1672
|
-
if
|
|
1672
|
+
if args:
|
|
1673
1673
|
if isinstance(args[0], list):
|
|
1674
1674
|
raise ValueError(f'"{args[0]}" can not be used as a variable name')
|
|
1675
1675
|
vars[args[0]] = {
|
|
@@ -1678,7 +1678,7 @@ def get_var_data(content, node_id=None):
|
|
|
1678
1678
|
}
|
|
1679
1679
|
elif func in parameter_types:
|
|
1680
1680
|
# avoid variable names to be None
|
|
1681
|
-
if
|
|
1681
|
+
if args and args[0] is not None:
|
|
1682
1682
|
# if this is a cast use the function name to get the type
|
|
1683
1683
|
if "default" not in kwargs:
|
|
1684
1684
|
default = kwargs.get("default", args[1] if len(args) > 1 else None)
|
tinybird/tb/__cli__.py
CHANGED
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/forward/commands'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '0.0.1.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '0.0.1.dev176'
|
|
8
|
+
__revision__ = '09be7b9'
|
tinybird/tb/modules/create.py
CHANGED
|
@@ -2,7 +2,7 @@ import glob
|
|
|
2
2
|
import os
|
|
3
3
|
import re
|
|
4
4
|
from pathlib import Path
|
|
5
|
-
from typing import Any, Dict,
|
|
5
|
+
from typing import Any, Dict, List, Optional
|
|
6
6
|
|
|
7
7
|
import click
|
|
8
8
|
|
|
@@ -69,7 +69,7 @@ async def create(
|
|
|
69
69
|
|
|
70
70
|
try:
|
|
71
71
|
tb_client = config.get_client(token=ctx_config.get("token"), host=ctx_config.get("host"))
|
|
72
|
-
user_token:
|
|
72
|
+
user_token: str = ""
|
|
73
73
|
created_something = False
|
|
74
74
|
if prompt:
|
|
75
75
|
user_token = ctx_config.get("user_token")
|
|
@@ -81,32 +81,40 @@ async def create(
|
|
|
81
81
|
create_project_structure(folder)
|
|
82
82
|
click.echo(FeedbackManager.success(message="✓ Scaffolding completed!\n"))
|
|
83
83
|
created_something = True
|
|
84
|
-
result =
|
|
85
|
-
|
|
86
|
-
if data or (prompt and user_token):
|
|
84
|
+
result: List[Path] = []
|
|
85
|
+
if data or prompt:
|
|
87
86
|
click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
|
|
88
|
-
|
|
87
|
+
|
|
88
|
+
data_result: List[Path] = []
|
|
89
|
+
if data:
|
|
90
|
+
data_result = await create_resources_from_data(data, project, ctx_config)
|
|
91
|
+
result.extend(data_result)
|
|
92
|
+
|
|
93
|
+
prompt_result: List[Path] = []
|
|
94
|
+
if prompt:
|
|
95
|
+
prompt_result = await create_resources_from_prompt(tb_client, user_token, prompt, project)
|
|
96
|
+
result.extend(prompt_result)
|
|
97
|
+
readme_path = Path(root_folder) / "README.md"
|
|
98
|
+
if readme_path.exists():
|
|
99
|
+
click.echo(FeedbackManager.highlight(message="\n» Updating README.md..."))
|
|
100
|
+
else:
|
|
101
|
+
click.echo(FeedbackManager.highlight(message="\n» Creating README.md..."))
|
|
102
|
+
readme_path.touch()
|
|
103
|
+
llm = LLM(user_token=str(user_token), host=tb_client.host)
|
|
104
|
+
readme_user_prompt = prompt or ""
|
|
105
|
+
all_resources_xml = get_resources_xml(project)
|
|
106
|
+
readme_response = llm.ask(
|
|
107
|
+
system_prompt=readme_prompt(
|
|
108
|
+
readme_path.read_text(), tb_client.host, "$TB_ADMIN_TOKEN", all_resources_xml
|
|
109
|
+
),
|
|
110
|
+
prompt=readme_user_prompt,
|
|
111
|
+
)
|
|
112
|
+
readme_result = extract_xml(readme_response, "readme")
|
|
113
|
+
readme_path.write_text(readme_result)
|
|
114
|
+
created_something = True
|
|
115
|
+
|
|
116
|
+
if data or prompt:
|
|
89
117
|
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
90
|
-
if prompt:
|
|
91
|
-
readme_path = Path(root_folder) / "README.md"
|
|
92
|
-
if readme_path.exists():
|
|
93
|
-
click.echo(FeedbackManager.highlight(message="\n» Updating README.md..."))
|
|
94
|
-
else:
|
|
95
|
-
click.echo(FeedbackManager.highlight(message="\n» Creating README.md..."))
|
|
96
|
-
readme_path.touch()
|
|
97
|
-
llm = LLM(user_token=str(user_token), host=tb_client.host)
|
|
98
|
-
readme_user_prompt = prompt or ""
|
|
99
|
-
all_resources_xml = get_resources_xml(project)
|
|
100
|
-
readme_response = llm.ask(
|
|
101
|
-
system_prompt=readme_prompt(
|
|
102
|
-
readme_path.read_text(), tb_client.host, "$TB_ADMIN_TOKEN", all_resources_xml
|
|
103
|
-
),
|
|
104
|
-
prompt=readme_user_prompt,
|
|
105
|
-
)
|
|
106
|
-
readme_result = extract_xml(readme_response, "readme")
|
|
107
|
-
readme_path.write_text(readme_result)
|
|
108
|
-
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
109
|
-
created_something = True
|
|
110
118
|
|
|
111
119
|
if not already_has_cicd(root_folder):
|
|
112
120
|
click.echo(FeedbackManager.highlight(message="\n» Creating CI/CD files for GitHub and GitLab..."))
|
|
@@ -125,17 +133,18 @@ async def create(
|
|
|
125
133
|
click.echo(FeedbackManager.highlight(message="\n» Generating fixtures..."))
|
|
126
134
|
|
|
127
135
|
if data:
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
136
|
+
for ds_path in [ds for ds in data_result if ds.suffix == ".datasource"]:
|
|
137
|
+
data_path = Path(data)
|
|
138
|
+
data_content = data_path.read_text()
|
|
139
|
+
ds_name = ds_path.stem
|
|
140
|
+
data_format = data_path.suffix.lstrip(".")
|
|
141
|
+
datasource_path = Path(folder) / "datasources" / f"{ds_name}.datasource"
|
|
142
|
+
click.echo(FeedbackManager.info(message=f"✓ /fixtures/{ds_name}"))
|
|
143
|
+
persist_fixture(ds_name, data_content, folder, format=data_format)
|
|
144
|
+
created_something = True
|
|
145
|
+
elif prompt and prompt_result:
|
|
146
|
+
ds_results = [path for path in prompt_result if path.suffix == ".datasource"]
|
|
147
|
+
for datasource_path in ds_results:
|
|
139
148
|
datasource_name = datasource_path.stem
|
|
140
149
|
datasource_content = datasource_path.read_text()
|
|
141
150
|
has_json_path = "`json:" in datasource_content
|
|
@@ -157,7 +166,7 @@ async def create(
|
|
|
157
166
|
click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}"))
|
|
158
167
|
created_something = True
|
|
159
168
|
|
|
160
|
-
if not created_something:
|
|
169
|
+
if not created_something and not len(result) > 0:
|
|
161
170
|
click.echo(FeedbackManager.warning(message="△ No resources created\n"))
|
|
162
171
|
except Exception as e:
|
|
163
172
|
raise CLICreateException(FeedbackManager.error(message=str(e)))
|
|
@@ -178,10 +187,19 @@ def validate_project_structure(folder: str) -> bool:
|
|
|
178
187
|
return len(datasources) > 0 or len(pipes) > 0
|
|
179
188
|
|
|
180
189
|
|
|
181
|
-
def should_generate_fixtures(result:
|
|
190
|
+
def should_generate_fixtures(result: List[Path]) -> List[Path]:
|
|
182
191
|
if not result:
|
|
183
|
-
return
|
|
184
|
-
return
|
|
192
|
+
return []
|
|
193
|
+
return [
|
|
194
|
+
path
|
|
195
|
+
for path in result
|
|
196
|
+
if path.suffix == ".datasource"
|
|
197
|
+
# we only want to generate fixtures for MergeTree or Null engines
|
|
198
|
+
and (
|
|
199
|
+
re.search(r'ENGINE\s+(?:"MergeTree"|MergeTree|"Null"|Null)', path.read_text())
|
|
200
|
+
or not re.search(r"ENGINE\s+", path.read_text())
|
|
201
|
+
)
|
|
202
|
+
]
|
|
185
203
|
|
|
186
204
|
|
|
187
205
|
def already_has_cicd(folder: str) -> bool:
|
|
@@ -205,113 +223,86 @@ def create_project_structure(folder: str):
|
|
|
205
223
|
click.echo(FeedbackManager.info_path_created(path=x))
|
|
206
224
|
|
|
207
225
|
|
|
208
|
-
async def
|
|
226
|
+
async def create_resources_from_prompt(
|
|
209
227
|
tb_client: TinyB,
|
|
210
|
-
user_token:
|
|
211
|
-
|
|
212
|
-
prompt: Optional[str],
|
|
228
|
+
user_token: str,
|
|
229
|
+
prompt: str,
|
|
213
230
|
project: Project,
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
""",
|
|
233
|
-
project.folder,
|
|
234
|
-
)
|
|
235
|
-
result = (
|
|
236
|
-
f"<response><resource><type>datasource</type><name>{name}</name><content></content></resource></response>"
|
|
237
|
-
)
|
|
238
|
-
created_any_resource = True
|
|
239
|
-
|
|
240
|
-
elif prompt and user_token:
|
|
241
|
-
datasource_paths = [Path(ds_file) for ds_file in project.get_datasource_files()]
|
|
242
|
-
pipes_paths = [Path(pipe_file) for pipe_file in project.get_pipe_files()]
|
|
243
|
-
connections_paths = [Path(conn_file) for conn_file in project.get_connection_files()]
|
|
244
|
-
resources_xml = "\n".join(
|
|
245
|
-
[
|
|
246
|
-
f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
|
|
247
|
-
for resource_type, resource_name, resource_content in [
|
|
248
|
-
("datasource", ds.stem, ds.read_text()) for ds in datasource_paths
|
|
249
|
-
]
|
|
250
|
-
+ [
|
|
251
|
-
(
|
|
252
|
-
"pipe",
|
|
253
|
-
pipe.stem,
|
|
254
|
-
pipe.read_text(),
|
|
255
|
-
)
|
|
256
|
-
for pipe in pipes_paths
|
|
257
|
-
]
|
|
258
|
-
+ [
|
|
259
|
-
(
|
|
260
|
-
"connection",
|
|
261
|
-
conn.stem,
|
|
262
|
-
conn.read_text(),
|
|
263
|
-
)
|
|
264
|
-
for conn in connections_paths
|
|
265
|
-
]
|
|
231
|
+
) -> List[Path]:
|
|
232
|
+
result: List[Path] = []
|
|
233
|
+
datasource_paths = [Path(ds_file) for ds_file in project.get_datasource_files()]
|
|
234
|
+
pipes_paths = [Path(pipe_file) for pipe_file in project.get_pipe_files()]
|
|
235
|
+
connections_paths = [Path(conn_file) for conn_file in project.get_connection_files()]
|
|
236
|
+
resources_xml = "\n".join(
|
|
237
|
+
[
|
|
238
|
+
f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
|
|
239
|
+
for resource_type, resource_name, resource_content in [
|
|
240
|
+
("datasource", ds.stem, ds.read_text()) for ds in datasource_paths
|
|
241
|
+
]
|
|
242
|
+
+ [
|
|
243
|
+
(
|
|
244
|
+
"pipe",
|
|
245
|
+
pipe.stem,
|
|
246
|
+
pipe.read_text(),
|
|
247
|
+
)
|
|
248
|
+
for pipe in pipes_paths
|
|
266
249
|
]
|
|
250
|
+
+ [
|
|
251
|
+
(
|
|
252
|
+
"connection",
|
|
253
|
+
conn.stem,
|
|
254
|
+
conn.read_text(),
|
|
255
|
+
)
|
|
256
|
+
for conn in connections_paths
|
|
257
|
+
]
|
|
258
|
+
]
|
|
259
|
+
)
|
|
260
|
+
llm = LLM(user_token=user_token, host=tb_client.host)
|
|
261
|
+
prompt_result = llm.ask(system_prompt=create_prompt(resources_xml), prompt=prompt)
|
|
262
|
+
prompt_result = extract_xml(prompt_result, "response")
|
|
263
|
+
resources = parse_xml(prompt_result, "resource")
|
|
264
|
+
datasources = []
|
|
265
|
+
pipes = []
|
|
266
|
+
connections = []
|
|
267
|
+
for resource_xml in resources:
|
|
268
|
+
resource_type = extract_xml(resource_xml, "type")
|
|
269
|
+
name = extract_xml(resource_xml, "name")
|
|
270
|
+
content = extract_xml(resource_xml, "content")
|
|
271
|
+
resource = {
|
|
272
|
+
"name": name,
|
|
273
|
+
"content": content,
|
|
274
|
+
}
|
|
275
|
+
if resource_type.lower() == "datasource":
|
|
276
|
+
datasources.append(resource)
|
|
277
|
+
elif resource_type.lower() == "pipe":
|
|
278
|
+
pipes.append(resource)
|
|
279
|
+
elif resource_type.lower() == "connection":
|
|
280
|
+
connections.append(resource)
|
|
281
|
+
|
|
282
|
+
for ds in datasources:
|
|
283
|
+
content = ds["content"].replace("```", "")
|
|
284
|
+
filename = f"{ds['name']}.datasource"
|
|
285
|
+
ds_file = generate_datafile(
|
|
286
|
+
content,
|
|
287
|
+
filename=filename,
|
|
288
|
+
data=None,
|
|
289
|
+
_format="ndjson",
|
|
290
|
+
force=True,
|
|
291
|
+
folder=project.folder,
|
|
267
292
|
)
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
pipes = []
|
|
274
|
-
connections = []
|
|
275
|
-
for resource_xml in resources:
|
|
276
|
-
resource_type = extract_xml(resource_xml, "type")
|
|
277
|
-
name = extract_xml(resource_xml, "name")
|
|
278
|
-
content = extract_xml(resource_xml, "content")
|
|
279
|
-
resource = {
|
|
280
|
-
"name": name,
|
|
281
|
-
"content": content,
|
|
282
|
-
}
|
|
283
|
-
if resource_type.lower() == "datasource":
|
|
284
|
-
datasources.append(resource)
|
|
285
|
-
elif resource_type.lower() == "pipe":
|
|
286
|
-
pipes.append(resource)
|
|
287
|
-
elif resource_type.lower() == "connection":
|
|
288
|
-
connections.append(resource)
|
|
289
|
-
|
|
290
|
-
for ds in datasources:
|
|
291
|
-
content = ds["content"].replace("```", "")
|
|
292
|
-
filename = f"{ds['name']}.datasource"
|
|
293
|
-
generate_datafile(
|
|
294
|
-
content,
|
|
295
|
-
filename=filename,
|
|
296
|
-
data=None,
|
|
297
|
-
_format="ndjson",
|
|
298
|
-
force=True,
|
|
299
|
-
folder=project.folder,
|
|
300
|
-
)
|
|
301
|
-
created_any_resource = True
|
|
293
|
+
result.append(ds_file)
|
|
294
|
+
for pipe in pipes:
|
|
295
|
+
content = pipe["content"].replace("```", "")
|
|
296
|
+
pipe_file = generate_pipe_file(pipe["name"], content, project.folder)
|
|
297
|
+
result.append(pipe_file)
|
|
302
298
|
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
299
|
+
for conn in connections:
|
|
300
|
+
content = conn["content"].replace("```", "")
|
|
301
|
+
filename = f"{conn['name']}.connection"
|
|
302
|
+
conn_file = generate_connection_file(conn["name"], content, project.folder)
|
|
303
|
+
result.append(conn_file)
|
|
307
304
|
|
|
308
|
-
|
|
309
|
-
content = conn["content"].replace("```", "")
|
|
310
|
-
filename = f"{conn['name']}.connection"
|
|
311
|
-
generate_connection_file(conn["name"], content, project.folder)
|
|
312
|
-
created_any_resource = True
|
|
313
|
-
|
|
314
|
-
return result, created_any_resource
|
|
305
|
+
return result
|
|
315
306
|
|
|
316
307
|
|
|
317
308
|
def init_git(folder: str):
|
|
@@ -454,3 +445,32 @@ def get_resources_xml(project: Project) -> str:
|
|
|
454
445
|
]
|
|
455
446
|
)
|
|
456
447
|
return resources_xml
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
async def create_resources_from_data(
|
|
451
|
+
data: str,
|
|
452
|
+
project: Project,
|
|
453
|
+
config: Dict[str, Any],
|
|
454
|
+
) -> List[Path]:
|
|
455
|
+
folder_path = project.path
|
|
456
|
+
local_client = await get_tinybird_local_client(config)
|
|
457
|
+
path = folder_path / data
|
|
458
|
+
result: List[Path] = []
|
|
459
|
+
format = path.suffix.lstrip(".")
|
|
460
|
+
ds_file = await _generate_datafile(str(path), local_client, format=format, force=True)
|
|
461
|
+
result.append(ds_file)
|
|
462
|
+
name = ds_file.stem
|
|
463
|
+
no_pipes = len(project.get_pipe_files()) == 0
|
|
464
|
+
if no_pipes:
|
|
465
|
+
pipe_file = generate_pipe_file(
|
|
466
|
+
f"{name}_endpoint",
|
|
467
|
+
f"""
|
|
468
|
+
NODE endpoint
|
|
469
|
+
SQL >
|
|
470
|
+
SELECT * from {name}
|
|
471
|
+
TYPE ENDPOINT
|
|
472
|
+
""",
|
|
473
|
+
project.folder,
|
|
474
|
+
)
|
|
475
|
+
result.append(pipe_file)
|
|
476
|
+
return result
|
|
@@ -110,6 +110,19 @@ class PipeNodeTypes:
|
|
|
110
110
|
DATA_SINK = "sink"
|
|
111
111
|
COPY = "copy"
|
|
112
112
|
STREAM = "stream"
|
|
113
|
+
EMPTY = ""
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
VALID_PIPE_NODE_TYPES = {
|
|
117
|
+
PipeNodeTypes.MATERIALIZED,
|
|
118
|
+
PipeNodeTypes.STANDARD,
|
|
119
|
+
PipeNodeTypes.COPY,
|
|
120
|
+
PipeNodeTypes.EMPTY,
|
|
121
|
+
PipeNodeTypes.ENDPOINT,
|
|
122
|
+
PipeNodeTypes.STREAM,
|
|
123
|
+
PipeNodeTypes.DATA_SINK,
|
|
124
|
+
}
|
|
125
|
+
VISIBLE_PIPE_NODE_TYPES = {PipeNodeTypes.MATERIALIZED, PipeNodeTypes.COPY, PipeNodeTypes.ENDPOINT}
|
|
113
126
|
|
|
114
127
|
|
|
115
128
|
class DataFileExtensions:
|
|
@@ -130,10 +143,43 @@ class CopyModes:
|
|
|
130
143
|
return node_mode.lower() in CopyModes.valid_modes
|
|
131
144
|
|
|
132
145
|
|
|
133
|
-
class
|
|
146
|
+
class Parameters:
|
|
147
|
+
ACCEPTED_ATTRIBUTES: set[str] = set()
|
|
148
|
+
MANDATORY_ATTRIBUTES: set[str] = set()
|
|
149
|
+
_PARAMS_ALIASES: dict[str, str] = {"name": "node", "mode": "copy_mode"}
|
|
150
|
+
|
|
151
|
+
@classmethod
|
|
152
|
+
def valid_params(cls) -> set[str]:
|
|
153
|
+
return cls.ACCEPTED_ATTRIBUTES
|
|
154
|
+
|
|
155
|
+
@classmethod
|
|
156
|
+
def required_params(cls) -> set[str]:
|
|
157
|
+
return cls.MANDATORY_ATTRIBUTES
|
|
158
|
+
|
|
159
|
+
@staticmethod
|
|
160
|
+
def canonical_name(name: str):
|
|
161
|
+
return Parameters._PARAMS_ALIASES.get(name, name)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
class PipeParameters(Parameters):
|
|
165
|
+
MANDATORY_ATTRIBUTES = {"name", "sql", "type"}
|
|
166
|
+
ACCEPTED_ATTRIBUTES = {"description"}.union(MANDATORY_ATTRIBUTES)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
class CopyParameters(Parameters):
|
|
134
170
|
TARGET_DATASOURCE = "target_datasource"
|
|
135
171
|
COPY_SCHEDULE = "copy_schedule"
|
|
136
172
|
COPY_MODE = "copy_mode"
|
|
173
|
+
COPY_MODE_ALIAS = "mode" # we need this because bot MODE and COPY_MODE go to `mode` variable inside the node
|
|
174
|
+
MANDATORY_ATTRIBUTES = PipeParameters.MANDATORY_ATTRIBUTES.union({TARGET_DATASOURCE})
|
|
175
|
+
ACCEPTED_ATTRIBUTES = PipeParameters.ACCEPTED_ATTRIBUTES.union(MANDATORY_ATTRIBUTES).union(
|
|
176
|
+
{COPY_SCHEDULE, COPY_MODE_ALIAS}
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
class MaterializedParameters(Parameters):
|
|
181
|
+
MANDATORY_ATTRIBUTES = PipeParameters.MANDATORY_ATTRIBUTES.union({"datasource"})
|
|
182
|
+
ACCEPTED_ATTRIBUTES = PipeParameters.ACCEPTED_ATTRIBUTES.union(MANDATORY_ATTRIBUTES)
|
|
137
183
|
|
|
138
184
|
|
|
139
185
|
DATAFILE_NEW_LINE = "\n"
|
|
@@ -228,8 +274,10 @@ class Datafile:
|
|
|
228
274
|
self.kind = kind
|
|
229
275
|
|
|
230
276
|
def validate_copy_node(self, node: Dict[str, Any]):
|
|
231
|
-
if
|
|
232
|
-
raise DatafileValidationError(
|
|
277
|
+
if missing := [param for param in CopyParameters.required_params() if param not in node]:
|
|
278
|
+
raise DatafileValidationError(
|
|
279
|
+
f"Some copy node params have been provided, but the following required ones are missing: {missing}"
|
|
280
|
+
)
|
|
233
281
|
# copy mode must be append or replace
|
|
234
282
|
if node.get("mode") and node["mode"] not in ["append", "replace"]:
|
|
235
283
|
raise DatafileValidationError("COPY node mode must be append or replace")
|
|
@@ -240,6 +288,22 @@ class Datafile:
|
|
|
240
288
|
and not croniter.is_valid(node["copy_schedule"])
|
|
241
289
|
):
|
|
242
290
|
raise DatafileValidationError("COPY node schedule must be @on-demand or a valid cron expression.")
|
|
291
|
+
for key in node.keys():
|
|
292
|
+
if key not in CopyParameters.valid_params():
|
|
293
|
+
raise DatafileValidationError(
|
|
294
|
+
f"Copy node {repr(node['name'])} has an invalid attribute ({CopyParameters.canonical_name(key)})"
|
|
295
|
+
)
|
|
296
|
+
|
|
297
|
+
def validate_materialized_node(self, node: Dict[str, Any]):
|
|
298
|
+
if missing := [param for param in MaterializedParameters.required_params() if param not in node]:
|
|
299
|
+
raise DatafileValidationError(
|
|
300
|
+
f"Some materialized node params have been provided, but the following required ones are missing: {missing}"
|
|
301
|
+
)
|
|
302
|
+
for key in node.keys():
|
|
303
|
+
if key not in MaterializedParameters.valid_params():
|
|
304
|
+
raise DatafileValidationError(
|
|
305
|
+
f"Materialized node {repr(node['name'])} has an invalid attribute ({MaterializedParameters.canonical_name(key)})"
|
|
306
|
+
)
|
|
243
307
|
|
|
244
308
|
def validate(self):
|
|
245
309
|
if self.kind == DatafileKind.pipe:
|
|
@@ -250,19 +314,21 @@ class Datafile:
|
|
|
250
314
|
# [x] Only one materialized node
|
|
251
315
|
# [x] Only one node of any specific type
|
|
252
316
|
# (rbarbadillo): there's a HUGE amount of validations in api_pipes.py, we should somehow merge them
|
|
253
|
-
for node in self.nodes:
|
|
254
|
-
if "sql" not in node:
|
|
255
|
-
raise DatafileValidationError(f"SQL missing for node {repr(node['name'])}")
|
|
256
317
|
non_standard_nodes_count = 0
|
|
257
318
|
for node in self.nodes:
|
|
258
|
-
|
|
319
|
+
node_type = node.get("type", "").lower()
|
|
320
|
+
if node_type not in {PipeNodeTypes.STANDARD, ""}:
|
|
259
321
|
non_standard_nodes_count += 1
|
|
260
322
|
if non_standard_nodes_count > 1:
|
|
261
323
|
raise DatafileValidationError("Multiple non-standard nodes in pipe. There can only be one")
|
|
262
|
-
if
|
|
263
|
-
|
|
264
|
-
if
|
|
324
|
+
if node_type == PipeNodeTypes.MATERIALIZED:
|
|
325
|
+
self.validate_materialized_node(node)
|
|
326
|
+
if node_type == PipeNodeTypes.COPY:
|
|
265
327
|
self.validate_copy_node(node)
|
|
328
|
+
if node_type not in VALID_PIPE_NODE_TYPES:
|
|
329
|
+
raise DatafileValidationError(
|
|
330
|
+
f"Invalid node type ({node_type}) in pipe {repr(node['name'])}. Allowed node types: {VISIBLE_PIPE_NODE_TYPES}"
|
|
331
|
+
)
|
|
266
332
|
for token in self.tokens:
|
|
267
333
|
if token["permission"].upper() != "READ":
|
|
268
334
|
raise DatafileValidationError(
|
|
@@ -289,17 +355,15 @@ class Datafile:
|
|
|
289
355
|
f"Invalid permission {token['permission']} for token {token['token_name']}. Only READ and APPEND are allowed for datasources"
|
|
290
356
|
)
|
|
291
357
|
# Validate Kafka params
|
|
292
|
-
if any(param in node for param in KAFKA_PARAMS) and
|
|
293
|
-
param in
|
|
358
|
+
if any(param in node for param in KAFKA_PARAMS) and (
|
|
359
|
+
missing := [param for param in REQUIRED_KAFKA_PARAMS if param not in node]
|
|
294
360
|
):
|
|
295
|
-
missing = [param for param in REQUIRED_KAFKA_PARAMS if param not in node]
|
|
296
361
|
raise DatafileValidationError(
|
|
297
362
|
f"Some Kafka params have been provided, but the following required ones are missing: {missing}"
|
|
298
363
|
)
|
|
299
364
|
# Validate S3 params
|
|
300
365
|
if any(param in node for param in BLOB_STORAGE_PARAMS):
|
|
301
|
-
if
|
|
302
|
-
missing = [param for param in REQUIRED_BLOB_STORAGE_PARAMS if param not in node]
|
|
366
|
+
if missing := [param for param in REQUIRED_BLOB_STORAGE_PARAMS if param not in node]:
|
|
303
367
|
raise DatafileValidationError(
|
|
304
368
|
f"Some connection params have been provided, but the following required ones are missing: {missing}"
|
|
305
369
|
)
|
|
@@ -53,7 +53,7 @@ def format_include(file_parts: List[str], doc: Datafile, unroll_includes: bool =
|
|
|
53
53
|
assert doc.raw is not None
|
|
54
54
|
|
|
55
55
|
include = [line for line in doc.raw if "INCLUDE" in line and ".incl" in line]
|
|
56
|
-
if
|
|
56
|
+
if include:
|
|
57
57
|
file_parts.append(include[0])
|
|
58
58
|
file_parts.append(DATAFILE_NEW_LINE)
|
|
59
59
|
return file_parts
|
|
@@ -107,8 +107,8 @@ async def datasource_ls(ctx: Context, match: Optional[str], format_: str):
|
|
|
107
107
|
|
|
108
108
|
|
|
109
109
|
@datasource.command(name="append")
|
|
110
|
-
@click.argument("datasource_name")
|
|
111
|
-
@click.argument("url", nargs=-1)
|
|
110
|
+
@click.argument("datasource_name", required=True)
|
|
111
|
+
@click.argument("url", nargs=-1, required=True)
|
|
112
112
|
@click.option("--concurrency", help="How many files to submit concurrently", default=1, hidden=True)
|
|
113
113
|
@click.pass_context
|
|
114
114
|
@coro
|
|
@@ -137,8 +137,8 @@ async def datasource_append(
|
|
|
137
137
|
|
|
138
138
|
|
|
139
139
|
@datasource.command(name="replace")
|
|
140
|
-
@click.argument("datasource_name")
|
|
141
|
-
@click.argument("url", nargs=-1)
|
|
140
|
+
@click.argument("datasource_name", required=True)
|
|
141
|
+
@click.argument("url", nargs=-1, required=True)
|
|
142
142
|
@click.option("--sql-condition", default=None, help="SQL WHERE condition to replace data", hidden=True)
|
|
143
143
|
@click.option("--skip-incompatible-partition-key", is_flag=True, default=False, hidden=True)
|
|
144
144
|
@click.pass_context
|
|
@@ -317,7 +317,7 @@ def test_run_summary(results: List[TestSummaryResults], only_fail: bool = False,
|
|
|
317
317
|
if test.error:
|
|
318
318
|
click.secho(test.error, fg=test.status.color, bold=True, nl=True, err=True)
|
|
319
319
|
|
|
320
|
-
if
|
|
320
|
+
if total_counts:
|
|
321
321
|
click.echo("\nTotals:")
|
|
322
322
|
for key_status, value_total in total_counts.items():
|
|
323
323
|
code_summary = f"Total {key_status.description}: {value_total}"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: tinybird
|
|
3
|
-
Version: 0.0.1.
|
|
3
|
+
Version: 0.0.1.dev176
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/forward/commands
|
|
6
6
|
Author: Tinybird
|
|
@@ -14,7 +14,7 @@ Requires-Dist: click<8.2,>=8.1.6
|
|
|
14
14
|
Requires-Dist: clickhouse-toolset==0.34.dev0
|
|
15
15
|
Requires-Dist: colorama==0.4.6
|
|
16
16
|
Requires-Dist: cryptography~=41.0.0
|
|
17
|
-
Requires-Dist: croniter==1.3.
|
|
17
|
+
Requires-Dist: croniter==1.3.15
|
|
18
18
|
Requires-Dist: docker==7.1.0
|
|
19
19
|
Requires-Dist: GitPython~=3.1.32
|
|
20
20
|
Requires-Dist: humanfriendly~=8.2
|
|
@@ -3,16 +3,16 @@ tinybird/context.py,sha256=FfqYfrGX_I7PKGTQo93utaKPDNVYWelg4Hsp3evX5wM,1291
|
|
|
3
3
|
tinybird/datatypes.py,sha256=r4WCvspmrXTJHiPjjyOTiZyZl31FO3Ynkwq4LQsYm6E,11059
|
|
4
4
|
tinybird/feedback_manager.py,sha256=1INQFfRfuMCb9lfB8KNf4r6qC2khW568hoHjtk-wshI,69305
|
|
5
5
|
tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
|
|
6
|
-
tinybird/prompts.py,sha256=
|
|
6
|
+
tinybird/prompts.py,sha256=uFzlVsGgWFwEOBhXkaLL4zY-wz8jCQgTFiMEfoFcgro,36550
|
|
7
7
|
tinybird/sql.py,sha256=C_B81wwv3BsqyXGhF5oTk9DcTUkrp7NwIFqSzd3Dmjc,47854
|
|
8
|
-
tinybird/sql_template.py,sha256=
|
|
8
|
+
tinybird/sql_template.py,sha256=hWW8JawSWLl9GeWPYkC_Yrxj7P0MHEVMJ0Px9bedEgM,99817
|
|
9
9
|
tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,10196
|
|
10
10
|
tinybird/sql_toolset.py,sha256=KORVbNAUTfW1qo3U9oe7Z59xQ0QMsFhB0ji3HzY2JVo,15324
|
|
11
11
|
tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
|
|
12
12
|
tinybird/tornado_template.py,sha256=jjNVDMnkYFWXflmT8KU_Ssbo5vR8KQq3EJMk5vYgXRw,41959
|
|
13
13
|
tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj0,3890
|
|
14
14
|
tinybird/ch_utils/engine.py,sha256=X4tE9OrfaUy6kO9cqVEzyI9cDcmOF3IAssRRzsTsfEQ,40781
|
|
15
|
-
tinybird/tb/__cli__.py,sha256=
|
|
15
|
+
tinybird/tb/__cli__.py,sha256=qwSD_BO6xPIvA1xz7Y11FszqnysrtuKrXLPMCIWrhHw,247
|
|
16
16
|
tinybird/tb/check_pypi.py,sha256=rW4QmDRbtgKdUUwJCnBkVjmTjZSZGN-XgZhx7vMkC0w,1009
|
|
17
17
|
tinybird/tb/cli.py,sha256=u3eGOhX0MHkuT6tiwaZ0_3twqLmqKXDAOxF7yV_Nn9Q,1075
|
|
18
18
|
tinybird/tb/client.py,sha256=CSBl_JRuioPyY0H8Ac96dJ9wQXDXfrvK2lwqlOxKGoY,55715
|
|
@@ -24,8 +24,8 @@ tinybird/tb/modules/common.py,sha256=lu1Z3VYImwocaHvqOW2FzBJP6F3Ev7RuXsItkCZ6jpo
|
|
|
24
24
|
tinybird/tb/modules/config.py,sha256=ziqW_t_mRVvWOd85VoB4vKyvgMkEfpXDf9H4v38p2xc,11422
|
|
25
25
|
tinybird/tb/modules/connection.py,sha256=7oOR7x4PhBcm1ETFFCH2YJ_3oeGXjAbmx1cnZX9_L70,9014
|
|
26
26
|
tinybird/tb/modules/copy.py,sha256=2Mm4FWKehOG7CoOhiF1m9UZJgJn0W1_cMolqju8ONYg,5805
|
|
27
|
-
tinybird/tb/modules/create.py,sha256=
|
|
28
|
-
tinybird/tb/modules/datasource.py,sha256=
|
|
27
|
+
tinybird/tb/modules/create.py,sha256=jz5jswVntvsnNN-IbCXoKw9pUKySx_IubpP0rxi7Iy0,17528
|
|
28
|
+
tinybird/tb/modules/datasource.py,sha256=mZ9N9LxbNA9QoKSMbxcMiQkdaQkfFqLeFB9_88e3_68,17843
|
|
29
29
|
tinybird/tb/modules/deployment.py,sha256=j-6zGvlAlr__ERRJg98-jEtwpR3ZbUJQeRJIj5YS94I,27070
|
|
30
30
|
tinybird/tb/modules/deprecations.py,sha256=rrszC1f_JJeJ8mUxGoCxckQTJFBCR8wREf4XXXN-PRc,4507
|
|
31
31
|
tinybird/tb/modules/dev_server.py,sha256=57FCKuWpErwYUYgHspYDkLWEm9F4pbvVOtMrFXX1fVU,10129
|
|
@@ -60,11 +60,11 @@ tinybird/tb/modules/datafile/build.py,sha256=d_h3pRFDPFrDKGhpFx2iejY25GuB2k8yfNo
|
|
|
60
60
|
tinybird/tb/modules/datafile/build_common.py,sha256=LU24kAQmxDJIyoIapDaYG-SU3P4FrMG9UBf8m9PgVSI,4565
|
|
61
61
|
tinybird/tb/modules/datafile/build_datasource.py,sha256=nXEQ0qHdq2ai7jJTv8H2d7eeDPBYzLn8VY7zMtOYb8M,17382
|
|
62
62
|
tinybird/tb/modules/datafile/build_pipe.py,sha256=6Cwjf3BKEF3-oQ9PipsQfK-Z43nSwtA4qJAUoysI7Uc,11385
|
|
63
|
-
tinybird/tb/modules/datafile/common.py,sha256=
|
|
63
|
+
tinybird/tb/modules/datafile/common.py,sha256=AYmU8IMKPEzdXNOtWcIrIn_voNncPADR_HzsxERi50g,92197
|
|
64
64
|
tinybird/tb/modules/datafile/diff.py,sha256=MTmj53RYjER4neLgWVjabn-FKVFgh8h8uYiBo55lFQg,6757
|
|
65
65
|
tinybird/tb/modules/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
|
|
66
66
|
tinybird/tb/modules/datafile/fixture.py,sha256=DrRWivcvo_1rn7LlVUnHcXccdgx9yVj63mzBkUwCzk8,1420
|
|
67
|
-
tinybird/tb/modules/datafile/format_common.py,sha256=
|
|
67
|
+
tinybird/tb/modules/datafile/format_common.py,sha256=1V1ZQuphp8EH2hT-IfZQEgz_0b9QqY177nwX8aMUte4,1962
|
|
68
68
|
tinybird/tb/modules/datafile/format_datasource.py,sha256=iWbeXruxC7OBmjNgurWt6ymcJlYzxfKwkGnhpcoSKEo,6190
|
|
69
69
|
tinybird/tb/modules/datafile/format_pipe.py,sha256=DUGdmlzI146YDqTwW-7kSIOXocz4AH-md_LFGUm9hrc,7436
|
|
70
70
|
tinybird/tb/modules/datafile/parse_datasource.py,sha256=p-qSdmDF7xbXt4PrQQc8q8aFa93kVbLpU4hIi6d3QH4,1764
|
|
@@ -72,7 +72,7 @@ tinybird/tb/modules/datafile/parse_pipe.py,sha256=WuQlYW51tflbcp2iFapJ7bxa9IkegD
|
|
|
72
72
|
tinybird/tb/modules/datafile/pipe_checker.py,sha256=xv7vyjN5dPc2hcw9RnLBq2VkR4nte-8bhYDT10qceQY,24620
|
|
73
73
|
tinybird/tb/modules/datafile/playground.py,sha256=94tOydeg5iQ3TQAdEWQWxLhx5Emz6xh0bEwLSao44-Y,56568
|
|
74
74
|
tinybird/tb/modules/datafile/pull.py,sha256=l6bIglY8b-tTIWgEYezf4kXjS0QHAVz4iOWLuNwe7Ps,5970
|
|
75
|
-
tinybird/tb/modules/tinyunit/tinyunit.py,sha256=
|
|
75
|
+
tinybird/tb/modules/tinyunit/tinyunit.py,sha256=qc3OHiXYSOv3TQCFRYrMnR47xJbUH0LnV4FBcBtWN1s,11713
|
|
76
76
|
tinybird/tb/modules/tinyunit/tinyunit_lib.py,sha256=hGh1ZaXC1af7rKnX7222urkj0QJMhMWclqMy59dOqwE,1922
|
|
77
77
|
tinybird/tb_cli_modules/cicd.py,sha256=0lMkb6CVOFZl5HOwgY8mK4T4mgI7O8335UngLXtCc-c,13851
|
|
78
78
|
tinybird/tb_cli_modules/common.py,sha256=G08f1_x5YSB6n-ncRj4tB2jhtjAjlWN-QD4xMWbTNQU,83033
|
|
@@ -80,8 +80,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
|
|
|
80
80
|
tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
|
|
81
81
|
tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
82
82
|
tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
|
|
83
|
-
tinybird-0.0.1.
|
|
84
|
-
tinybird-0.0.1.
|
|
85
|
-
tinybird-0.0.1.
|
|
86
|
-
tinybird-0.0.1.
|
|
87
|
-
tinybird-0.0.1.
|
|
83
|
+
tinybird-0.0.1.dev176.dist-info/METADATA,sha256=4wJLDkqnQ1E1ynrHIK3402UWiSP4VA3hzNguApVbFz4,1608
|
|
84
|
+
tinybird-0.0.1.dev176.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
|
85
|
+
tinybird-0.0.1.dev176.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
|
|
86
|
+
tinybird-0.0.1.dev176.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
|
|
87
|
+
tinybird-0.0.1.dev176.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|