tinybird 0.0.1.dev174__py3-none-any.whl → 0.0.1.dev176__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of tinybird might be problematic. Click here for more details.

tinybird/prompts.py CHANGED
@@ -461,8 +461,7 @@ Use the following feedback to regenerate the response:
461
461
  """
462
462
 
463
463
  return f"""
464
- Given the schema for a Tinybird datasource, return a can you create a clickhouse sql query to generate some random data that matches that schema.
465
-
464
+ Given the schema for a Tinybird datasource, return a valid clickhouse sql query to generate some random data that matches that schema.
466
465
  Response format MUST be just a valid clickhouse sql query.
467
466
 
468
467
  <example>
@@ -722,6 +721,7 @@ datasource_instructions = """
722
721
  - Use MergeTree engine by default.
723
722
  - Use AggregatingMergeTree engine when the datasource is the target of a materialized pipe.
724
723
  - Use always json paths to define the schema. Example: `user_id` String `json:$.user_id`,
724
+ - Array columns are supported with a special syntax. Example: `items` Array(String) `json:$.items[:]`
725
725
  - If the datasource is using an S3 or GCS connection, they need to set IMPORT_CONNECTION_NAME, IMPORT_BUCKET_URI and IMPORT_SCHEDULE (GCS @on-demand only, S3 supports @auto too)
726
726
  </datasource_file_instructions>
727
727
  """
tinybird/sql_template.py CHANGED
@@ -1657,7 +1657,7 @@ def get_var_data(content, node_id=None):
1657
1657
  # It will be overriden in later definitions or left as is otherwise.
1658
1658
  # args[0] check is used to avoid adding unnamed parameters found in
1659
1659
  # templates like: `split_to_array('')`
1660
- if len(args) and isinstance(args[0], list):
1660
+ if args and isinstance(args[0], list):
1661
1661
  raise ValueError(f'"{args[0]}" can not be used as a variable name')
1662
1662
  if len(args) > 0 and args[0] not in vars and args[0]:
1663
1663
  vars[args[0]] = {
@@ -1669,7 +1669,7 @@ def get_var_data(content, node_id=None):
1669
1669
  if "default" not in kwargs:
1670
1670
  default = kwargs.get("default", args[2] if len(args) > 2 and args[2] else None)
1671
1671
  kwargs["default"] = check_default_value(default)
1672
- if len(args):
1672
+ if args:
1673
1673
  if isinstance(args[0], list):
1674
1674
  raise ValueError(f'"{args[0]}" can not be used as a variable name')
1675
1675
  vars[args[0]] = {
@@ -1678,7 +1678,7 @@ def get_var_data(content, node_id=None):
1678
1678
  }
1679
1679
  elif func in parameter_types:
1680
1680
  # avoid variable names to be None
1681
- if len(args) and args[0] is not None:
1681
+ if args and args[0] is not None:
1682
1682
  # if this is a cast use the function name to get the type
1683
1683
  if "default" not in kwargs:
1684
1684
  default = kwargs.get("default", args[1] if len(args) > 1 else None)
tinybird/tb/__cli__.py CHANGED
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
4
4
  __url__ = 'https://www.tinybird.co/docs/forward/commands'
5
5
  __author__ = 'Tinybird'
6
6
  __author_email__ = 'support@tinybird.co'
7
- __version__ = '0.0.1.dev174'
8
- __revision__ = '39f0b89'
7
+ __version__ = '0.0.1.dev176'
8
+ __revision__ = '09be7b9'
@@ -2,7 +2,7 @@ import glob
2
2
  import os
3
3
  import re
4
4
  from pathlib import Path
5
- from typing import Any, Dict, Optional, Tuple
5
+ from typing import Any, Dict, List, Optional
6
6
 
7
7
  import click
8
8
 
@@ -69,7 +69,7 @@ async def create(
69
69
 
70
70
  try:
71
71
  tb_client = config.get_client(token=ctx_config.get("token"), host=ctx_config.get("host"))
72
- user_token: Optional[str] = None
72
+ user_token: str = ""
73
73
  created_something = False
74
74
  if prompt:
75
75
  user_token = ctx_config.get("user_token")
@@ -81,32 +81,40 @@ async def create(
81
81
  create_project_structure(folder)
82
82
  click.echo(FeedbackManager.success(message="✓ Scaffolding completed!\n"))
83
83
  created_something = True
84
- result = ""
85
-
86
- if data or (prompt and user_token):
84
+ result: List[Path] = []
85
+ if data or prompt:
87
86
  click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
88
- result, created_something = await create_resources(tb_client, user_token, data, prompt, project, ctx_config)
87
+
88
+ data_result: List[Path] = []
89
+ if data:
90
+ data_result = await create_resources_from_data(data, project, ctx_config)
91
+ result.extend(data_result)
92
+
93
+ prompt_result: List[Path] = []
94
+ if prompt:
95
+ prompt_result = await create_resources_from_prompt(tb_client, user_token, prompt, project)
96
+ result.extend(prompt_result)
97
+ readme_path = Path(root_folder) / "README.md"
98
+ if readme_path.exists():
99
+ click.echo(FeedbackManager.highlight(message="\n» Updating README.md..."))
100
+ else:
101
+ click.echo(FeedbackManager.highlight(message="\n» Creating README.md..."))
102
+ readme_path.touch()
103
+ llm = LLM(user_token=str(user_token), host=tb_client.host)
104
+ readme_user_prompt = prompt or ""
105
+ all_resources_xml = get_resources_xml(project)
106
+ readme_response = llm.ask(
107
+ system_prompt=readme_prompt(
108
+ readme_path.read_text(), tb_client.host, "$TB_ADMIN_TOKEN", all_resources_xml
109
+ ),
110
+ prompt=readme_user_prompt,
111
+ )
112
+ readme_result = extract_xml(readme_response, "readme")
113
+ readme_path.write_text(readme_result)
114
+ created_something = True
115
+
116
+ if data or prompt:
89
117
  click.echo(FeedbackManager.success(message="✓ Done!\n"))
90
- if prompt:
91
- readme_path = Path(root_folder) / "README.md"
92
- if readme_path.exists():
93
- click.echo(FeedbackManager.highlight(message="\n» Updating README.md..."))
94
- else:
95
- click.echo(FeedbackManager.highlight(message="\n» Creating README.md..."))
96
- readme_path.touch()
97
- llm = LLM(user_token=str(user_token), host=tb_client.host)
98
- readme_user_prompt = prompt or ""
99
- all_resources_xml = get_resources_xml(project)
100
- readme_response = llm.ask(
101
- system_prompt=readme_prompt(
102
- readme_path.read_text(), tb_client.host, "$TB_ADMIN_TOKEN", all_resources_xml
103
- ),
104
- prompt=readme_user_prompt,
105
- )
106
- readme_result = extract_xml(readme_response, "readme")
107
- readme_path.write_text(readme_result)
108
- click.echo(FeedbackManager.success(message="✓ Done!\n"))
109
- created_something = True
110
118
 
111
119
  if not already_has_cicd(root_folder):
112
120
  click.echo(FeedbackManager.highlight(message="\n» Creating CI/CD files for GitHub and GitLab..."))
@@ -125,17 +133,18 @@ async def create(
125
133
  click.echo(FeedbackManager.highlight(message="\n» Generating fixtures..."))
126
134
 
127
135
  if data:
128
- ds_name = os.path.basename(data.split(".")[0])
129
- data_path = Path(data)
130
- data_content = data_path.read_text()
131
- data_format = data_path.suffix.lstrip(".")
132
- datasource_path = Path(folder) / "datasources" / f"{ds_name}.datasource"
133
- click.echo(FeedbackManager.info(message=f" /fixtures/{ds_name}"))
134
- persist_fixture(ds_name, data_content, folder, format=data_format)
135
- created_something = True
136
- elif prompt and user_token:
137
- for datasource_file in project.get_datasource_files():
138
- datasource_path = Path(datasource_file)
136
+ for ds_path in [ds for ds in data_result if ds.suffix == ".datasource"]:
137
+ data_path = Path(data)
138
+ data_content = data_path.read_text()
139
+ ds_name = ds_path.stem
140
+ data_format = data_path.suffix.lstrip(".")
141
+ datasource_path = Path(folder) / "datasources" / f"{ds_name}.datasource"
142
+ click.echo(FeedbackManager.info(message=f"✓ /fixtures/{ds_name}"))
143
+ persist_fixture(ds_name, data_content, folder, format=data_format)
144
+ created_something = True
145
+ elif prompt and prompt_result:
146
+ ds_results = [path for path in prompt_result if path.suffix == ".datasource"]
147
+ for datasource_path in ds_results:
139
148
  datasource_name = datasource_path.stem
140
149
  datasource_content = datasource_path.read_text()
141
150
  has_json_path = "`json:" in datasource_content
@@ -157,7 +166,7 @@ async def create(
157
166
  click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}"))
158
167
  created_something = True
159
168
 
160
- if not created_something:
169
+ if not created_something and not len(result) > 0:
161
170
  click.echo(FeedbackManager.warning(message="△ No resources created\n"))
162
171
  except Exception as e:
163
172
  raise CLICreateException(FeedbackManager.error(message=str(e)))
@@ -178,10 +187,19 @@ def validate_project_structure(folder: str) -> bool:
178
187
  return len(datasources) > 0 or len(pipes) > 0
179
188
 
180
189
 
181
- def should_generate_fixtures(result: str) -> bool:
190
+ def should_generate_fixtures(result: List[Path]) -> List[Path]:
182
191
  if not result:
183
- return False
184
- return "<type>datasource</type>" in result
192
+ return []
193
+ return [
194
+ path
195
+ for path in result
196
+ if path.suffix == ".datasource"
197
+ # we only want to generate fixtures for MergeTree or Null engines
198
+ and (
199
+ re.search(r'ENGINE\s+(?:"MergeTree"|MergeTree|"Null"|Null)', path.read_text())
200
+ or not re.search(r"ENGINE\s+", path.read_text())
201
+ )
202
+ ]
185
203
 
186
204
 
187
205
  def already_has_cicd(folder: str) -> bool:
@@ -205,113 +223,86 @@ def create_project_structure(folder: str):
205
223
  click.echo(FeedbackManager.info_path_created(path=x))
206
224
 
207
225
 
208
- async def create_resources(
226
+ async def create_resources_from_prompt(
209
227
  tb_client: TinyB,
210
- user_token: Optional[str],
211
- data: Optional[str],
212
- prompt: Optional[str],
228
+ user_token: str,
229
+ prompt: str,
213
230
  project: Project,
214
- config: Dict[str, Any],
215
- ) -> Tuple[str, bool]:
216
- result = ""
217
- created_any_resource = False
218
- folder_path = project.path
219
- if data:
220
- local_client = await get_tinybird_local_client(config)
221
- path = folder_path / data
222
- format = path.suffix.lstrip(".")
223
- await _generate_datafile(str(path), local_client, format=format, force=True)
224
- name = path.stem
225
- generate_pipe_file(
226
- f"{name}_endpoint",
227
- f"""
228
- NODE endpoint
229
- SQL >
230
- SELECT * from {name}
231
- TYPE ENDPOINT
232
- """,
233
- project.folder,
234
- )
235
- result = (
236
- f"<response><resource><type>datasource</type><name>{name}</name><content></content></resource></response>"
237
- )
238
- created_any_resource = True
239
-
240
- elif prompt and user_token:
241
- datasource_paths = [Path(ds_file) for ds_file in project.get_datasource_files()]
242
- pipes_paths = [Path(pipe_file) for pipe_file in project.get_pipe_files()]
243
- connections_paths = [Path(conn_file) for conn_file in project.get_connection_files()]
244
- resources_xml = "\n".join(
245
- [
246
- f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
247
- for resource_type, resource_name, resource_content in [
248
- ("datasource", ds.stem, ds.read_text()) for ds in datasource_paths
249
- ]
250
- + [
251
- (
252
- "pipe",
253
- pipe.stem,
254
- pipe.read_text(),
255
- )
256
- for pipe in pipes_paths
257
- ]
258
- + [
259
- (
260
- "connection",
261
- conn.stem,
262
- conn.read_text(),
263
- )
264
- for conn in connections_paths
265
- ]
231
+ ) -> List[Path]:
232
+ result: List[Path] = []
233
+ datasource_paths = [Path(ds_file) for ds_file in project.get_datasource_files()]
234
+ pipes_paths = [Path(pipe_file) for pipe_file in project.get_pipe_files()]
235
+ connections_paths = [Path(conn_file) for conn_file in project.get_connection_files()]
236
+ resources_xml = "\n".join(
237
+ [
238
+ f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
239
+ for resource_type, resource_name, resource_content in [
240
+ ("datasource", ds.stem, ds.read_text()) for ds in datasource_paths
241
+ ]
242
+ + [
243
+ (
244
+ "pipe",
245
+ pipe.stem,
246
+ pipe.read_text(),
247
+ )
248
+ for pipe in pipes_paths
266
249
  ]
250
+ + [
251
+ (
252
+ "connection",
253
+ conn.stem,
254
+ conn.read_text(),
255
+ )
256
+ for conn in connections_paths
257
+ ]
258
+ ]
259
+ )
260
+ llm = LLM(user_token=user_token, host=tb_client.host)
261
+ prompt_result = llm.ask(system_prompt=create_prompt(resources_xml), prompt=prompt)
262
+ prompt_result = extract_xml(prompt_result, "response")
263
+ resources = parse_xml(prompt_result, "resource")
264
+ datasources = []
265
+ pipes = []
266
+ connections = []
267
+ for resource_xml in resources:
268
+ resource_type = extract_xml(resource_xml, "type")
269
+ name = extract_xml(resource_xml, "name")
270
+ content = extract_xml(resource_xml, "content")
271
+ resource = {
272
+ "name": name,
273
+ "content": content,
274
+ }
275
+ if resource_type.lower() == "datasource":
276
+ datasources.append(resource)
277
+ elif resource_type.lower() == "pipe":
278
+ pipes.append(resource)
279
+ elif resource_type.lower() == "connection":
280
+ connections.append(resource)
281
+
282
+ for ds in datasources:
283
+ content = ds["content"].replace("```", "")
284
+ filename = f"{ds['name']}.datasource"
285
+ ds_file = generate_datafile(
286
+ content,
287
+ filename=filename,
288
+ data=None,
289
+ _format="ndjson",
290
+ force=True,
291
+ folder=project.folder,
267
292
  )
268
- llm = LLM(user_token=user_token, host=tb_client.host)
269
- result = llm.ask(system_prompt=create_prompt(resources_xml), prompt=prompt)
270
- result = extract_xml(result, "response")
271
- resources = parse_xml(result, "resource")
272
- datasources = []
273
- pipes = []
274
- connections = []
275
- for resource_xml in resources:
276
- resource_type = extract_xml(resource_xml, "type")
277
- name = extract_xml(resource_xml, "name")
278
- content = extract_xml(resource_xml, "content")
279
- resource = {
280
- "name": name,
281
- "content": content,
282
- }
283
- if resource_type.lower() == "datasource":
284
- datasources.append(resource)
285
- elif resource_type.lower() == "pipe":
286
- pipes.append(resource)
287
- elif resource_type.lower() == "connection":
288
- connections.append(resource)
289
-
290
- for ds in datasources:
291
- content = ds["content"].replace("```", "")
292
- filename = f"{ds['name']}.datasource"
293
- generate_datafile(
294
- content,
295
- filename=filename,
296
- data=None,
297
- _format="ndjson",
298
- force=True,
299
- folder=project.folder,
300
- )
301
- created_any_resource = True
293
+ result.append(ds_file)
294
+ for pipe in pipes:
295
+ content = pipe["content"].replace("```", "")
296
+ pipe_file = generate_pipe_file(pipe["name"], content, project.folder)
297
+ result.append(pipe_file)
302
298
 
303
- for pipe in pipes:
304
- content = pipe["content"].replace("```", "")
305
- generate_pipe_file(pipe["name"], content, project.folder)
306
- created_any_resource = True
299
+ for conn in connections:
300
+ content = conn["content"].replace("```", "")
301
+ filename = f"{conn['name']}.connection"
302
+ conn_file = generate_connection_file(conn["name"], content, project.folder)
303
+ result.append(conn_file)
307
304
 
308
- for conn in connections:
309
- content = conn["content"].replace("```", "")
310
- filename = f"{conn['name']}.connection"
311
- generate_connection_file(conn["name"], content, project.folder)
312
- created_any_resource = True
313
-
314
- return result, created_any_resource
305
+ return result
315
306
 
316
307
 
317
308
  def init_git(folder: str):
@@ -454,3 +445,32 @@ def get_resources_xml(project: Project) -> str:
454
445
  ]
455
446
  )
456
447
  return resources_xml
448
+
449
+
450
+ async def create_resources_from_data(
451
+ data: str,
452
+ project: Project,
453
+ config: Dict[str, Any],
454
+ ) -> List[Path]:
455
+ folder_path = project.path
456
+ local_client = await get_tinybird_local_client(config)
457
+ path = folder_path / data
458
+ result: List[Path] = []
459
+ format = path.suffix.lstrip(".")
460
+ ds_file = await _generate_datafile(str(path), local_client, format=format, force=True)
461
+ result.append(ds_file)
462
+ name = ds_file.stem
463
+ no_pipes = len(project.get_pipe_files()) == 0
464
+ if no_pipes:
465
+ pipe_file = generate_pipe_file(
466
+ f"{name}_endpoint",
467
+ f"""
468
+ NODE endpoint
469
+ SQL >
470
+ SELECT * from {name}
471
+ TYPE ENDPOINT
472
+ """,
473
+ project.folder,
474
+ )
475
+ result.append(pipe_file)
476
+ return result
@@ -110,6 +110,19 @@ class PipeNodeTypes:
110
110
  DATA_SINK = "sink"
111
111
  COPY = "copy"
112
112
  STREAM = "stream"
113
+ EMPTY = ""
114
+
115
+
116
+ VALID_PIPE_NODE_TYPES = {
117
+ PipeNodeTypes.MATERIALIZED,
118
+ PipeNodeTypes.STANDARD,
119
+ PipeNodeTypes.COPY,
120
+ PipeNodeTypes.EMPTY,
121
+ PipeNodeTypes.ENDPOINT,
122
+ PipeNodeTypes.STREAM,
123
+ PipeNodeTypes.DATA_SINK,
124
+ }
125
+ VISIBLE_PIPE_NODE_TYPES = {PipeNodeTypes.MATERIALIZED, PipeNodeTypes.COPY, PipeNodeTypes.ENDPOINT}
113
126
 
114
127
 
115
128
  class DataFileExtensions:
@@ -130,10 +143,43 @@ class CopyModes:
130
143
  return node_mode.lower() in CopyModes.valid_modes
131
144
 
132
145
 
133
- class CopyParameters:
146
+ class Parameters:
147
+ ACCEPTED_ATTRIBUTES: set[str] = set()
148
+ MANDATORY_ATTRIBUTES: set[str] = set()
149
+ _PARAMS_ALIASES: dict[str, str] = {"name": "node", "mode": "copy_mode"}
150
+
151
+ @classmethod
152
+ def valid_params(cls) -> set[str]:
153
+ return cls.ACCEPTED_ATTRIBUTES
154
+
155
+ @classmethod
156
+ def required_params(cls) -> set[str]:
157
+ return cls.MANDATORY_ATTRIBUTES
158
+
159
+ @staticmethod
160
+ def canonical_name(name: str):
161
+ return Parameters._PARAMS_ALIASES.get(name, name)
162
+
163
+
164
+ class PipeParameters(Parameters):
165
+ MANDATORY_ATTRIBUTES = {"name", "sql", "type"}
166
+ ACCEPTED_ATTRIBUTES = {"description"}.union(MANDATORY_ATTRIBUTES)
167
+
168
+
169
+ class CopyParameters(Parameters):
134
170
  TARGET_DATASOURCE = "target_datasource"
135
171
  COPY_SCHEDULE = "copy_schedule"
136
172
  COPY_MODE = "copy_mode"
173
+ COPY_MODE_ALIAS = "mode" # we need this because bot MODE and COPY_MODE go to `mode` variable inside the node
174
+ MANDATORY_ATTRIBUTES = PipeParameters.MANDATORY_ATTRIBUTES.union({TARGET_DATASOURCE})
175
+ ACCEPTED_ATTRIBUTES = PipeParameters.ACCEPTED_ATTRIBUTES.union(MANDATORY_ATTRIBUTES).union(
176
+ {COPY_SCHEDULE, COPY_MODE_ALIAS}
177
+ )
178
+
179
+
180
+ class MaterializedParameters(Parameters):
181
+ MANDATORY_ATTRIBUTES = PipeParameters.MANDATORY_ATTRIBUTES.union({"datasource"})
182
+ ACCEPTED_ATTRIBUTES = PipeParameters.ACCEPTED_ATTRIBUTES.union(MANDATORY_ATTRIBUTES)
137
183
 
138
184
 
139
185
  DATAFILE_NEW_LINE = "\n"
@@ -228,8 +274,10 @@ class Datafile:
228
274
  self.kind = kind
229
275
 
230
276
  def validate_copy_node(self, node: Dict[str, Any]):
231
- if "target_datasource" not in node:
232
- raise DatafileValidationError("COPY node missing target datasource")
277
+ if missing := [param for param in CopyParameters.required_params() if param not in node]:
278
+ raise DatafileValidationError(
279
+ f"Some copy node params have been provided, but the following required ones are missing: {missing}"
280
+ )
233
281
  # copy mode must be append or replace
234
282
  if node.get("mode") and node["mode"] not in ["append", "replace"]:
235
283
  raise DatafileValidationError("COPY node mode must be append or replace")
@@ -240,6 +288,22 @@ class Datafile:
240
288
  and not croniter.is_valid(node["copy_schedule"])
241
289
  ):
242
290
  raise DatafileValidationError("COPY node schedule must be @on-demand or a valid cron expression.")
291
+ for key in node.keys():
292
+ if key not in CopyParameters.valid_params():
293
+ raise DatafileValidationError(
294
+ f"Copy node {repr(node['name'])} has an invalid attribute ({CopyParameters.canonical_name(key)})"
295
+ )
296
+
297
+ def validate_materialized_node(self, node: Dict[str, Any]):
298
+ if missing := [param for param in MaterializedParameters.required_params() if param not in node]:
299
+ raise DatafileValidationError(
300
+ f"Some materialized node params have been provided, but the following required ones are missing: {missing}"
301
+ )
302
+ for key in node.keys():
303
+ if key not in MaterializedParameters.valid_params():
304
+ raise DatafileValidationError(
305
+ f"Materialized node {repr(node['name'])} has an invalid attribute ({MaterializedParameters.canonical_name(key)})"
306
+ )
243
307
 
244
308
  def validate(self):
245
309
  if self.kind == DatafileKind.pipe:
@@ -250,19 +314,21 @@ class Datafile:
250
314
  # [x] Only one materialized node
251
315
  # [x] Only one node of any specific type
252
316
  # (rbarbadillo): there's a HUGE amount of validations in api_pipes.py, we should somehow merge them
253
- for node in self.nodes:
254
- if "sql" not in node:
255
- raise DatafileValidationError(f"SQL missing for node {repr(node['name'])}")
256
317
  non_standard_nodes_count = 0
257
318
  for node in self.nodes:
258
- if node.get("type", "").lower() not in {PipeNodeTypes.STANDARD, ""}:
319
+ node_type = node.get("type", "").lower()
320
+ if node_type not in {PipeNodeTypes.STANDARD, ""}:
259
321
  non_standard_nodes_count += 1
260
322
  if non_standard_nodes_count > 1:
261
323
  raise DatafileValidationError("Multiple non-standard nodes in pipe. There can only be one")
262
- if node.get("type", "").lower() == PipeNodeTypes.MATERIALIZED and "datasource" not in node:
263
- raise DatafileValidationError(f"Materialized node {repr(node['name'])} missing target datasource")
264
- if node.get("type", "").lower() == PipeNodeTypes.COPY:
324
+ if node_type == PipeNodeTypes.MATERIALIZED:
325
+ self.validate_materialized_node(node)
326
+ if node_type == PipeNodeTypes.COPY:
265
327
  self.validate_copy_node(node)
328
+ if node_type not in VALID_PIPE_NODE_TYPES:
329
+ raise DatafileValidationError(
330
+ f"Invalid node type ({node_type}) in pipe {repr(node['name'])}. Allowed node types: {VISIBLE_PIPE_NODE_TYPES}"
331
+ )
266
332
  for token in self.tokens:
267
333
  if token["permission"].upper() != "READ":
268
334
  raise DatafileValidationError(
@@ -289,17 +355,15 @@ class Datafile:
289
355
  f"Invalid permission {token['permission']} for token {token['token_name']}. Only READ and APPEND are allowed for datasources"
290
356
  )
291
357
  # Validate Kafka params
292
- if any(param in node for param in KAFKA_PARAMS) and not all(
293
- param in node for param in REQUIRED_KAFKA_PARAMS
358
+ if any(param in node for param in KAFKA_PARAMS) and (
359
+ missing := [param for param in REQUIRED_KAFKA_PARAMS if param not in node]
294
360
  ):
295
- missing = [param for param in REQUIRED_KAFKA_PARAMS if param not in node]
296
361
  raise DatafileValidationError(
297
362
  f"Some Kafka params have been provided, but the following required ones are missing: {missing}"
298
363
  )
299
364
  # Validate S3 params
300
365
  if any(param in node for param in BLOB_STORAGE_PARAMS):
301
- if not all(param in node for param in REQUIRED_BLOB_STORAGE_PARAMS):
302
- missing = [param for param in REQUIRED_BLOB_STORAGE_PARAMS if param not in node]
366
+ if missing := [param for param in REQUIRED_BLOB_STORAGE_PARAMS if param not in node]:
303
367
  raise DatafileValidationError(
304
368
  f"Some connection params have been provided, but the following required ones are missing: {missing}"
305
369
  )
@@ -53,7 +53,7 @@ def format_include(file_parts: List[str], doc: Datafile, unroll_includes: bool =
53
53
  assert doc.raw is not None
54
54
 
55
55
  include = [line for line in doc.raw if "INCLUDE" in line and ".incl" in line]
56
- if len(include):
56
+ if include:
57
57
  file_parts.append(include[0])
58
58
  file_parts.append(DATAFILE_NEW_LINE)
59
59
  return file_parts
@@ -107,8 +107,8 @@ async def datasource_ls(ctx: Context, match: Optional[str], format_: str):
107
107
 
108
108
 
109
109
  @datasource.command(name="append")
110
- @click.argument("datasource_name")
111
- @click.argument("url", nargs=-1)
110
+ @click.argument("datasource_name", required=True)
111
+ @click.argument("url", nargs=-1, required=True)
112
112
  @click.option("--concurrency", help="How many files to submit concurrently", default=1, hidden=True)
113
113
  @click.pass_context
114
114
  @coro
@@ -137,8 +137,8 @@ async def datasource_append(
137
137
 
138
138
 
139
139
  @datasource.command(name="replace")
140
- @click.argument("datasource_name")
141
- @click.argument("url", nargs=-1)
140
+ @click.argument("datasource_name", required=True)
141
+ @click.argument("url", nargs=-1, required=True)
142
142
  @click.option("--sql-condition", default=None, help="SQL WHERE condition to replace data", hidden=True)
143
143
  @click.option("--skip-incompatible-partition-key", is_flag=True, default=False, hidden=True)
144
144
  @click.pass_context
@@ -317,7 +317,7 @@ def test_run_summary(results: List[TestSummaryResults], only_fail: bool = False,
317
317
  if test.error:
318
318
  click.secho(test.error, fg=test.status.color, bold=True, nl=True, err=True)
319
319
 
320
- if len(total_counts):
320
+ if total_counts:
321
321
  click.echo("\nTotals:")
322
322
  for key_status, value_total in total_counts.items():
323
323
  code_summary = f"Total {key_status.description}: {value_total}"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: tinybird
3
- Version: 0.0.1.dev174
3
+ Version: 0.0.1.dev176
4
4
  Summary: Tinybird Command Line Tool
5
5
  Home-page: https://www.tinybird.co/docs/forward/commands
6
6
  Author: Tinybird
@@ -14,7 +14,7 @@ Requires-Dist: click<8.2,>=8.1.6
14
14
  Requires-Dist: clickhouse-toolset==0.34.dev0
15
15
  Requires-Dist: colorama==0.4.6
16
16
  Requires-Dist: cryptography~=41.0.0
17
- Requires-Dist: croniter==1.3.8
17
+ Requires-Dist: croniter==1.3.15
18
18
  Requires-Dist: docker==7.1.0
19
19
  Requires-Dist: GitPython~=3.1.32
20
20
  Requires-Dist: humanfriendly~=8.2
@@ -3,16 +3,16 @@ tinybird/context.py,sha256=FfqYfrGX_I7PKGTQo93utaKPDNVYWelg4Hsp3evX5wM,1291
3
3
  tinybird/datatypes.py,sha256=r4WCvspmrXTJHiPjjyOTiZyZl31FO3Ynkwq4LQsYm6E,11059
4
4
  tinybird/feedback_manager.py,sha256=1INQFfRfuMCb9lfB8KNf4r6qC2khW568hoHjtk-wshI,69305
5
5
  tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
6
- tinybird/prompts.py,sha256=yLXzHB7_P0zjriIvI9UsfnIRwmcwLSEA1tcJMzYJqjw,36456
6
+ tinybird/prompts.py,sha256=uFzlVsGgWFwEOBhXkaLL4zY-wz8jCQgTFiMEfoFcgro,36550
7
7
  tinybird/sql.py,sha256=C_B81wwv3BsqyXGhF5oTk9DcTUkrp7NwIFqSzd3Dmjc,47854
8
- tinybird/sql_template.py,sha256=mK0yeRFctbXTAu0VNMjIzoFBJoh9PoniAVgEatA5SG4,99832
8
+ tinybird/sql_template.py,sha256=hWW8JawSWLl9GeWPYkC_Yrxj7P0MHEVMJ0Px9bedEgM,99817
9
9
  tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,10196
10
10
  tinybird/sql_toolset.py,sha256=KORVbNAUTfW1qo3U9oe7Z59xQ0QMsFhB0ji3HzY2JVo,15324
11
11
  tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
12
12
  tinybird/tornado_template.py,sha256=jjNVDMnkYFWXflmT8KU_Ssbo5vR8KQq3EJMk5vYgXRw,41959
13
13
  tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj0,3890
14
14
  tinybird/ch_utils/engine.py,sha256=X4tE9OrfaUy6kO9cqVEzyI9cDcmOF3IAssRRzsTsfEQ,40781
15
- tinybird/tb/__cli__.py,sha256=qwBoknIQp-vIaHCpdb58D9NUc-GpfFDJ91MCeKskmaI,247
15
+ tinybird/tb/__cli__.py,sha256=qwSD_BO6xPIvA1xz7Y11FszqnysrtuKrXLPMCIWrhHw,247
16
16
  tinybird/tb/check_pypi.py,sha256=rW4QmDRbtgKdUUwJCnBkVjmTjZSZGN-XgZhx7vMkC0w,1009
17
17
  tinybird/tb/cli.py,sha256=u3eGOhX0MHkuT6tiwaZ0_3twqLmqKXDAOxF7yV_Nn9Q,1075
18
18
  tinybird/tb/client.py,sha256=CSBl_JRuioPyY0H8Ac96dJ9wQXDXfrvK2lwqlOxKGoY,55715
@@ -24,8 +24,8 @@ tinybird/tb/modules/common.py,sha256=lu1Z3VYImwocaHvqOW2FzBJP6F3Ev7RuXsItkCZ6jpo
24
24
  tinybird/tb/modules/config.py,sha256=ziqW_t_mRVvWOd85VoB4vKyvgMkEfpXDf9H4v38p2xc,11422
25
25
  tinybird/tb/modules/connection.py,sha256=7oOR7x4PhBcm1ETFFCH2YJ_3oeGXjAbmx1cnZX9_L70,9014
26
26
  tinybird/tb/modules/copy.py,sha256=2Mm4FWKehOG7CoOhiF1m9UZJgJn0W1_cMolqju8ONYg,5805
27
- tinybird/tb/modules/create.py,sha256=_s_vdb9pjq1ORajE_gFMxSGVuns1imNgiD5H8KuI2gQ,17231
28
- tinybird/tb/modules/datasource.py,sha256=V314rkpdVxVMjsp5qcSCTqDlmp4Vu--qM07BoWh-aqs,17783
27
+ tinybird/tb/modules/create.py,sha256=jz5jswVntvsnNN-IbCXoKw9pUKySx_IubpP0rxi7Iy0,17528
28
+ tinybird/tb/modules/datasource.py,sha256=mZ9N9LxbNA9QoKSMbxcMiQkdaQkfFqLeFB9_88e3_68,17843
29
29
  tinybird/tb/modules/deployment.py,sha256=j-6zGvlAlr__ERRJg98-jEtwpR3ZbUJQeRJIj5YS94I,27070
30
30
  tinybird/tb/modules/deprecations.py,sha256=rrszC1f_JJeJ8mUxGoCxckQTJFBCR8wREf4XXXN-PRc,4507
31
31
  tinybird/tb/modules/dev_server.py,sha256=57FCKuWpErwYUYgHspYDkLWEm9F4pbvVOtMrFXX1fVU,10129
@@ -60,11 +60,11 @@ tinybird/tb/modules/datafile/build.py,sha256=d_h3pRFDPFrDKGhpFx2iejY25GuB2k8yfNo
60
60
  tinybird/tb/modules/datafile/build_common.py,sha256=LU24kAQmxDJIyoIapDaYG-SU3P4FrMG9UBf8m9PgVSI,4565
61
61
  tinybird/tb/modules/datafile/build_datasource.py,sha256=nXEQ0qHdq2ai7jJTv8H2d7eeDPBYzLn8VY7zMtOYb8M,17382
62
62
  tinybird/tb/modules/datafile/build_pipe.py,sha256=6Cwjf3BKEF3-oQ9PipsQfK-Z43nSwtA4qJAUoysI7Uc,11385
63
- tinybird/tb/modules/datafile/common.py,sha256=WOHlXXPLs1RkdPLXA7qLHOGGey7d1Kn8PPdOe-3zVtE,89720
63
+ tinybird/tb/modules/datafile/common.py,sha256=AYmU8IMKPEzdXNOtWcIrIn_voNncPADR_HzsxERi50g,92197
64
64
  tinybird/tb/modules/datafile/diff.py,sha256=MTmj53RYjER4neLgWVjabn-FKVFgh8h8uYiBo55lFQg,6757
65
65
  tinybird/tb/modules/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
66
66
  tinybird/tb/modules/datafile/fixture.py,sha256=DrRWivcvo_1rn7LlVUnHcXccdgx9yVj63mzBkUwCzk8,1420
67
- tinybird/tb/modules/datafile/format_common.py,sha256=WaNV4tXrQU5gjV6MJP-5TGqg_Bre6ilNS8emvFl-X3c,1967
67
+ tinybird/tb/modules/datafile/format_common.py,sha256=1V1ZQuphp8EH2hT-IfZQEgz_0b9QqY177nwX8aMUte4,1962
68
68
  tinybird/tb/modules/datafile/format_datasource.py,sha256=iWbeXruxC7OBmjNgurWt6ymcJlYzxfKwkGnhpcoSKEo,6190
69
69
  tinybird/tb/modules/datafile/format_pipe.py,sha256=DUGdmlzI146YDqTwW-7kSIOXocz4AH-md_LFGUm9hrc,7436
70
70
  tinybird/tb/modules/datafile/parse_datasource.py,sha256=p-qSdmDF7xbXt4PrQQc8q8aFa93kVbLpU4hIi6d3QH4,1764
@@ -72,7 +72,7 @@ tinybird/tb/modules/datafile/parse_pipe.py,sha256=WuQlYW51tflbcp2iFapJ7bxa9IkegD
72
72
  tinybird/tb/modules/datafile/pipe_checker.py,sha256=xv7vyjN5dPc2hcw9RnLBq2VkR4nte-8bhYDT10qceQY,24620
73
73
  tinybird/tb/modules/datafile/playground.py,sha256=94tOydeg5iQ3TQAdEWQWxLhx5Emz6xh0bEwLSao44-Y,56568
74
74
  tinybird/tb/modules/datafile/pull.py,sha256=l6bIglY8b-tTIWgEYezf4kXjS0QHAVz4iOWLuNwe7Ps,5970
75
- tinybird/tb/modules/tinyunit/tinyunit.py,sha256=tVynlV16gt1ex0fDmCPmcABQYyzjVnnycHq1Ou3kU0c,11718
75
+ tinybird/tb/modules/tinyunit/tinyunit.py,sha256=qc3OHiXYSOv3TQCFRYrMnR47xJbUH0LnV4FBcBtWN1s,11713
76
76
  tinybird/tb/modules/tinyunit/tinyunit_lib.py,sha256=hGh1ZaXC1af7rKnX7222urkj0QJMhMWclqMy59dOqwE,1922
77
77
  tinybird/tb_cli_modules/cicd.py,sha256=0lMkb6CVOFZl5HOwgY8mK4T4mgI7O8335UngLXtCc-c,13851
78
78
  tinybird/tb_cli_modules/common.py,sha256=G08f1_x5YSB6n-ncRj4tB2jhtjAjlWN-QD4xMWbTNQU,83033
@@ -80,8 +80,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
80
80
  tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
81
81
  tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
82
82
  tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
83
- tinybird-0.0.1.dev174.dist-info/METADATA,sha256=ByV0Ug-OYQaawT9rhCxOq05EaNrW5dKguHZLqws1rw0,1607
84
- tinybird-0.0.1.dev174.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
85
- tinybird-0.0.1.dev174.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
86
- tinybird-0.0.1.dev174.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
87
- tinybird-0.0.1.dev174.dist-info/RECORD,,
83
+ tinybird-0.0.1.dev176.dist-info/METADATA,sha256=4wJLDkqnQ1E1ynrHIK3402UWiSP4VA3hzNguApVbFz4,1608
84
+ tinybird-0.0.1.dev176.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
85
+ tinybird-0.0.1.dev176.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
86
+ tinybird-0.0.1.dev176.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
87
+ tinybird-0.0.1.dev176.dist-info/RECORD,,