tinybird 0.0.1.dev234__py3-none-any.whl → 0.0.1.dev235__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/check_pypi.py +3 -8
- tinybird/tb/cli.py +0 -6
- tinybird/tb/client.py +314 -340
- tinybird/tb/config.py +4 -5
- tinybird/tb/modules/build.py +21 -24
- tinybird/tb/modules/cicd.py +2 -2
- tinybird/tb/modules/cli.py +18 -28
- tinybird/tb/modules/common.py +123 -138
- tinybird/tb/modules/config.py +2 -4
- tinybird/tb/modules/connection.py +21 -26
- tinybird/tb/modules/copy.py +7 -9
- tinybird/tb/modules/create.py +18 -21
- tinybird/tb/modules/datafile/build.py +39 -39
- tinybird/tb/modules/datafile/build_common.py +9 -9
- tinybird/tb/modules/datafile/build_datasource.py +24 -24
- tinybird/tb/modules/datafile/build_pipe.py +11 -13
- tinybird/tb/modules/datafile/diff.py +12 -12
- tinybird/tb/modules/datafile/format_datasource.py +5 -5
- tinybird/tb/modules/datafile/format_pipe.py +6 -6
- tinybird/tb/modules/datafile/playground.py +42 -42
- tinybird/tb/modules/datafile/pull.py +24 -26
- tinybird/tb/modules/datasource.py +42 -56
- tinybird/tb/modules/endpoint.py +14 -19
- tinybird/tb/modules/info.py +14 -15
- tinybird/tb/modules/infra.py +43 -48
- tinybird/tb/modules/job.py +7 -10
- tinybird/tb/modules/local.py +6 -12
- tinybird/tb/modules/local_common.py +4 -4
- tinybird/tb/modules/login.py +9 -10
- tinybird/tb/modules/materialization.py +7 -10
- tinybird/tb/modules/mock.py +8 -9
- tinybird/tb/modules/open.py +1 -3
- tinybird/tb/modules/pipe.py +2 -4
- tinybird/tb/modules/secret.py +12 -16
- tinybird/tb/modules/shell.py +7 -20
- tinybird/tb/modules/sink.py +6 -8
- tinybird/tb/modules/test.py +9 -14
- tinybird/tb/modules/tinyunit/tinyunit.py +3 -3
- tinybird/tb/modules/token.py +16 -24
- tinybird/tb/modules/watch.py +3 -7
- tinybird/tb/modules/workspace.py +26 -37
- tinybird/tb/modules/workspace_members.py +16 -23
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev235.dist-info}/METADATA +1 -1
- tinybird-0.0.1.dev235.dist-info/RECORD +89 -0
- tinybird-0.0.1.dev234.dist-info/RECORD +0 -89
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev235.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev235.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev235.dist-info}/top_level.txt +0 -0
tinybird/tb/modules/config.py
CHANGED
|
@@ -256,9 +256,7 @@ class CLIConfig:
|
|
|
256
256
|
def set_workspace_token(self, workspace_id: str, token: str) -> None:
|
|
257
257
|
pass
|
|
258
258
|
|
|
259
|
-
|
|
260
|
-
self, workspace_id: Optional[str] = None, host: Optional[str] = None
|
|
261
|
-
) -> Optional[str]:
|
|
259
|
+
def get_workspace_token(self, workspace_id: Optional[str] = None, host: Optional[str] = None) -> Optional[str]:
|
|
262
260
|
"""Returns the token for the specific workspace on a host.
|
|
263
261
|
- If no workspace passed, it uses the current active workspace.
|
|
264
262
|
- If no host is passed, it uses the current one.
|
|
@@ -282,7 +280,7 @@ class CLIConfig:
|
|
|
282
280
|
|
|
283
281
|
client: tbc.TinyB = self.get_client(token=self.get_user_token(), host=host)
|
|
284
282
|
|
|
285
|
-
info: Dict[str, Any] =
|
|
283
|
+
info: Dict[str, Any] = client.user_workspaces_and_branches(version="v1")
|
|
286
284
|
workspaces: List[Dict[str, Any]] = info["workspaces"]
|
|
287
285
|
|
|
288
286
|
result: Optional[str] = next(
|
|
@@ -15,7 +15,6 @@ from tinybird.tb.modules.cli import cli
|
|
|
15
15
|
from tinybird.tb.modules.common import (
|
|
16
16
|
DataConnectorType,
|
|
17
17
|
_get_setting_value,
|
|
18
|
-
coro,
|
|
19
18
|
echo_safe_humanfriendly_tables_format_smart_table,
|
|
20
19
|
get_gcs_connection_name,
|
|
21
20
|
get_gcs_svc_account_creds,
|
|
@@ -117,13 +116,12 @@ def connection(ctx: Context) -> None:
|
|
|
117
116
|
@connection.command(name="ls")
|
|
118
117
|
@click.option("--service", help="Filter by service")
|
|
119
118
|
@click.pass_context
|
|
120
|
-
|
|
121
|
-
async def connection_ls(ctx: Context, service: Optional[DataConnectorType] = None) -> None:
|
|
119
|
+
def connection_ls(ctx: Context, service: Optional[DataConnectorType] = None) -> None:
|
|
122
120
|
"""List connections."""
|
|
123
121
|
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
124
122
|
client: TinyB = obj["client"]
|
|
125
123
|
|
|
126
|
-
connections =
|
|
124
|
+
connections = client.connections(connector=service, skip_bigquery=True)
|
|
127
125
|
columns = []
|
|
128
126
|
table = []
|
|
129
127
|
|
|
@@ -155,8 +153,7 @@ def connection_create(ctx: Context) -> None:
|
|
|
155
153
|
|
|
156
154
|
@connection_create.command(name="s3", short_help="Creates a AWS S3 connection.")
|
|
157
155
|
@click.pass_context
|
|
158
|
-
|
|
159
|
-
async def connection_create_s3(ctx: Context) -> None:
|
|
156
|
+
def connection_create_s3(ctx: Context) -> None:
|
|
160
157
|
"""
|
|
161
158
|
Creates a AWS S3 connection.
|
|
162
159
|
|
|
@@ -167,7 +164,7 @@ async def connection_create_s3(ctx: Context) -> None:
|
|
|
167
164
|
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
168
165
|
client: TinyB = obj["client"]
|
|
169
166
|
|
|
170
|
-
if obj["env"] == "local" and not
|
|
167
|
+
if obj["env"] == "local" and not client.check_aws_credentials():
|
|
171
168
|
click.echo(
|
|
172
169
|
FeedbackManager.error(
|
|
173
170
|
message="No AWS credentials found. Please run `tb local restart --use-aws-creds` to pass your credentials. "
|
|
@@ -194,7 +191,7 @@ async def connection_create_s3(ctx: Context) -> None:
|
|
|
194
191
|
)
|
|
195
192
|
|
|
196
193
|
connection_name = get_s3_connection_name(project.folder)
|
|
197
|
-
role_arn, region, bucket_name =
|
|
194
|
+
role_arn, region, bucket_name = run_aws_iamrole_connection_flow(
|
|
198
195
|
client,
|
|
199
196
|
service=service,
|
|
200
197
|
environment=obj["env"],
|
|
@@ -206,7 +203,7 @@ async def connection_create_s3(ctx: Context) -> None:
|
|
|
206
203
|
if obj["env"] == "local":
|
|
207
204
|
save_secret_to_env_file(project=project, name=secret_name, value=role_arn)
|
|
208
205
|
else:
|
|
209
|
-
|
|
206
|
+
client.create_secret(name=secret_name, value=role_arn)
|
|
210
207
|
|
|
211
208
|
create_in_cloud = (
|
|
212
209
|
click.confirm(FeedbackManager.prompt_connection_in_cloud_confirmation(), default=True)
|
|
@@ -223,7 +220,7 @@ async def connection_create_s3(ctx: Context) -> None:
|
|
|
223
220
|
host=host,
|
|
224
221
|
staging=False,
|
|
225
222
|
)
|
|
226
|
-
prod_role_arn, _, _ =
|
|
223
|
+
prod_role_arn, _, _ = production_aws_iamrole_only(
|
|
227
224
|
prod_client,
|
|
228
225
|
service=service,
|
|
229
226
|
region=region,
|
|
@@ -232,9 +229,9 @@ async def connection_create_s3(ctx: Context) -> None:
|
|
|
232
229
|
connection_name=connection_name,
|
|
233
230
|
policy=access_type.lower(),
|
|
234
231
|
)
|
|
235
|
-
|
|
232
|
+
prod_client.create_secret(name=secret_name, value=prod_role_arn)
|
|
236
233
|
|
|
237
|
-
connection_file_path =
|
|
234
|
+
connection_file_path = generate_aws_iamrole_connection_file_with_secret(
|
|
238
235
|
name=connection_name,
|
|
239
236
|
service=service,
|
|
240
237
|
role_arn_secret_name=secret_name,
|
|
@@ -260,8 +257,7 @@ async def connection_create_s3(ctx: Context) -> None:
|
|
|
260
257
|
|
|
261
258
|
@connection_create.command(name="gcs", short_help="Creates a Google Cloud Storage connection.")
|
|
262
259
|
@click.pass_context
|
|
263
|
-
|
|
264
|
-
async def connection_create_gcs(ctx: Context) -> None:
|
|
260
|
+
def connection_create_gcs(ctx: Context) -> None:
|
|
265
261
|
"""
|
|
266
262
|
Creates a Google Cloud Storage connection.
|
|
267
263
|
|
|
@@ -275,16 +271,16 @@ async def connection_create_gcs(ctx: Context) -> None:
|
|
|
275
271
|
service = DataConnectorType.GCLOUD_STORAGE
|
|
276
272
|
click.echo(FeedbackManager.prompt_gcs_connection_header())
|
|
277
273
|
connection_name = get_gcs_connection_name(project.folder)
|
|
278
|
-
|
|
274
|
+
run_gcp_svc_account_connection_flow(environment=obj["env"])
|
|
279
275
|
creds_json = get_gcs_svc_account_creds()
|
|
280
276
|
unique_suffix = uuid.uuid4().hex[:8] # Use first 8 chars of a UUID for brevity
|
|
281
277
|
secret_name = f"gcs_svc_account_creds_{connection_name}_{unique_suffix}"
|
|
282
278
|
if obj["env"] == "local":
|
|
283
279
|
save_secret_to_env_file(project=project, name=secret_name, value=creds_json)
|
|
284
280
|
else:
|
|
285
|
-
|
|
281
|
+
client.create_secret(name=secret_name, value=creds_json)
|
|
286
282
|
|
|
287
|
-
connection_path =
|
|
283
|
+
connection_path = generate_gcs_connection_file_with_secrets(
|
|
288
284
|
name=connection_name,
|
|
289
285
|
service=service,
|
|
290
286
|
svc_account_creds=secret_name,
|
|
@@ -308,7 +304,7 @@ async def connection_create_gcs(ctx: Context) -> None:
|
|
|
308
304
|
)
|
|
309
305
|
creds_json = get_gcs_svc_account_creds()
|
|
310
306
|
secret_name = f"gcs_svc_account_creds_{connection_name}_{unique_suffix}"
|
|
311
|
-
|
|
307
|
+
prod_client.create_secret(name=secret_name, value=creds_json)
|
|
312
308
|
|
|
313
309
|
click.echo(
|
|
314
310
|
FeedbackManager.prompt_gcs_success(
|
|
@@ -320,18 +316,17 @@ async def connection_create_gcs(ctx: Context) -> None:
|
|
|
320
316
|
|
|
321
317
|
@connection_create.command(name="kafka", short_help="Creates a Kafka connection.")
|
|
322
318
|
@click.pass_context
|
|
323
|
-
|
|
324
|
-
async def connection_create_kafka_cmd(ctx: Context) -> None:
|
|
319
|
+
def connection_create_kafka_cmd(ctx: Context) -> None:
|
|
325
320
|
"""
|
|
326
321
|
Creates a Kafka connection.
|
|
327
322
|
|
|
328
323
|
\b
|
|
329
324
|
$ tb connection create kafka
|
|
330
325
|
"""
|
|
331
|
-
|
|
326
|
+
connection_create_kafka(ctx)
|
|
332
327
|
|
|
333
328
|
|
|
334
|
-
|
|
329
|
+
def connection_create_kafka(ctx: Context) -> Tuple[str, str, str, str, str, str, str, str, List[str]]:
|
|
335
330
|
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
336
331
|
click.echo(FeedbackManager.gray(message="\n» Creating Kafka connection..."))
|
|
337
332
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
@@ -439,11 +434,11 @@ async def connection_create_kafka(ctx: Context) -> Tuple[str, str, str, str, str
|
|
|
439
434
|
staging=False,
|
|
440
435
|
)
|
|
441
436
|
if tb_secret_bootstrap_servers:
|
|
442
|
-
|
|
437
|
+
prod_client.create_secret(name=tb_secret_bootstrap_servers, value=bootstrap_servers)
|
|
443
438
|
if tb_secret_key:
|
|
444
|
-
|
|
439
|
+
prod_client.create_secret(name=tb_secret_key, value=key)
|
|
445
440
|
if tb_secret_secret:
|
|
446
|
-
|
|
441
|
+
prod_client.create_secret(name=tb_secret_secret, value=secret)
|
|
447
442
|
click.echo(FeedbackManager.success(message="✓ Secrets created!"))
|
|
448
443
|
|
|
449
444
|
schema_registry_url = click.prompt(
|
|
@@ -471,7 +466,7 @@ async def connection_create_kafka(ctx: Context) -> Tuple[str, str, str, str, str
|
|
|
471
466
|
raise CLIConnectionException(FeedbackManager.error(message="Invalid Kafka connection"))
|
|
472
467
|
|
|
473
468
|
click.echo(FeedbackManager.success(message="✓ Connection is valid"))
|
|
474
|
-
|
|
469
|
+
generate_kafka_connection_with_secrets(
|
|
475
470
|
name=name,
|
|
476
471
|
bootstrap_servers=bootstrap_servers,
|
|
477
472
|
tb_secret_bootstrap_servers=tb_secret_bootstrap_servers,
|
tinybird/tb/modules/copy.py
CHANGED
|
@@ -13,7 +13,7 @@ from click import Context
|
|
|
13
13
|
from tinybird.datafile.common import get_name_version
|
|
14
14
|
from tinybird.tb.client import AuthNoTokenException, TinyB
|
|
15
15
|
from tinybird.tb.modules.cli import cli
|
|
16
|
-
from tinybird.tb.modules.common import
|
|
16
|
+
from tinybird.tb.modules.common import echo_safe_humanfriendly_tables_format_smart_table, wait_job
|
|
17
17
|
from tinybird.tb.modules.exceptions import CLIPipeException
|
|
18
18
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
19
19
|
|
|
@@ -34,12 +34,11 @@ def copy(ctx):
|
|
|
34
34
|
help="Force a type of the output",
|
|
35
35
|
)
|
|
36
36
|
@click.pass_context
|
|
37
|
-
|
|
38
|
-
async def copy_ls(ctx: Context, match: str, format_: str):
|
|
37
|
+
def copy_ls(ctx: Context, match: str, format_: str):
|
|
39
38
|
"""List copy pipes"""
|
|
40
39
|
|
|
41
40
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
42
|
-
pipes =
|
|
41
|
+
pipes = client.pipes(dependencies=False, node_attrs="name", attrs="name,updated_at,type")
|
|
43
42
|
copies = [p for p in pipes if p.get("type") == "copy"]
|
|
44
43
|
copies = sorted(copies, key=lambda p: p["updated_at"])
|
|
45
44
|
columns = ["name", "updated at", "nodes"]
|
|
@@ -84,8 +83,7 @@ async def copy_ls(ctx: Context, match: str, format_: str):
|
|
|
84
83
|
help="Key and value of the params you want the Copy pipe to be called with. For example: tb pipe copy run <my_copy_pipe> --param foo=bar",
|
|
85
84
|
)
|
|
86
85
|
@click.pass_context
|
|
87
|
-
|
|
88
|
-
async def copy_run(ctx: click.Context, pipe_name_or_id: str, wait: bool, mode: str, param: Optional[Tuple[str]]):
|
|
86
|
+
def copy_run(ctx: click.Context, pipe_name_or_id: str, wait: bool, mode: str, param: Optional[Tuple[str]]):
|
|
89
87
|
"""Run an on-demand copy pipe"""
|
|
90
88
|
|
|
91
89
|
params = dict(key_value.split("=") for key_value in param) if param else {}
|
|
@@ -93,18 +91,18 @@ async def copy_run(ctx: click.Context, pipe_name_or_id: str, wait: bool, mode: s
|
|
|
93
91
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
94
92
|
|
|
95
93
|
try:
|
|
96
|
-
response =
|
|
94
|
+
response = client.pipe_run(pipe_name_or_id, "copy", params, mode)
|
|
97
95
|
job_id = response["job"]["job_id"]
|
|
98
96
|
job_url = response["job"]["job_url"]
|
|
99
97
|
target_datasource_id = response["tags"]["copy_target_datasource"]
|
|
100
|
-
target_datasource =
|
|
98
|
+
target_datasource = client.get_datasource(target_datasource_id)
|
|
101
99
|
target_datasource_name = target_datasource["name"]
|
|
102
100
|
click.echo(FeedbackManager.gray(message="Job URL: ") + FeedbackManager.info(message=f"{job_url}"))
|
|
103
101
|
click.echo(FeedbackManager.success(message=f"✓ Copy to '{target_datasource_name}' job created"))
|
|
104
102
|
|
|
105
103
|
if wait:
|
|
106
104
|
click.echo("\n")
|
|
107
|
-
|
|
105
|
+
wait_job(client, job_id, job_url, FeedbackManager.highlight(message="» Copying data"))
|
|
108
106
|
click.echo(FeedbackManager.success(message=f"✓ Data copied to '{target_datasource_name}'"))
|
|
109
107
|
|
|
110
108
|
except AuthNoTokenException:
|
tinybird/tb/modules/create.py
CHANGED
|
@@ -13,7 +13,7 @@ from tinybird.prompts import claude_rules_prompt, create_prompt, readme_prompt,
|
|
|
13
13
|
from tinybird.tb.client import TinyB
|
|
14
14
|
from tinybird.tb.modules.cicd import init_cicd
|
|
15
15
|
from tinybird.tb.modules.cli import cli
|
|
16
|
-
from tinybird.tb.modules.common import _generate_datafile,
|
|
16
|
+
from tinybird.tb.modules.common import _generate_datafile, generate_datafile
|
|
17
17
|
from tinybird.tb.modules.config import CLIConfig
|
|
18
18
|
from tinybird.tb.modules.datafile.fixture import persist_fixture
|
|
19
19
|
from tinybird.tb.modules.exceptions import CLICreateException
|
|
@@ -42,8 +42,7 @@ from tinybird.tb.modules.project import Project
|
|
|
42
42
|
@click.option("--folder", type=str, default=None, help="Folder to create the project in")
|
|
43
43
|
@click.option("--agent", type=str, default="cursor", help="Agent to use for rules")
|
|
44
44
|
@click.pass_context
|
|
45
|
-
|
|
46
|
-
async def create(
|
|
45
|
+
def create(
|
|
47
46
|
ctx: click.Context, data: Optional[str], prompt: Optional[str], rows: int, folder: Optional[str], agent: str
|
|
48
47
|
) -> None:
|
|
49
48
|
"""Initialize a new project."""
|
|
@@ -93,14 +92,14 @@ async def create(
|
|
|
93
92
|
data_result: List[Path] = []
|
|
94
93
|
if data:
|
|
95
94
|
if urlparse(data).scheme in ("http", "https"):
|
|
96
|
-
data_result =
|
|
95
|
+
data_result = create_resources_from_url(data, project, ctx_config)
|
|
97
96
|
else:
|
|
98
|
-
data_result =
|
|
97
|
+
data_result = create_resources_from_data(data, project, ctx_config)
|
|
99
98
|
result.extend(data_result)
|
|
100
99
|
|
|
101
100
|
prompt_result: List[Path] = []
|
|
102
101
|
if prompt:
|
|
103
|
-
prompt_result =
|
|
102
|
+
prompt_result = create_resources_from_prompt(tb_client, user_token, prompt, project)
|
|
104
103
|
result.extend(prompt_result)
|
|
105
104
|
readme_path = folder_path / "README.md"
|
|
106
105
|
if readme_path.exists():
|
|
@@ -135,7 +134,7 @@ async def create(
|
|
|
135
134
|
if not already_has_cicd(root_folder):
|
|
136
135
|
click.echo(FeedbackManager.highlight(message="\n» Creating CI/CD files for GitHub and GitLab..."))
|
|
137
136
|
init_git(root_folder)
|
|
138
|
-
|
|
137
|
+
init_cicd(root_folder, data_project_dir=os.path.relpath(folder))
|
|
139
138
|
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
140
139
|
created_something = True
|
|
141
140
|
|
|
@@ -160,7 +159,7 @@ async def create(
|
|
|
160
159
|
for ds_path in [ds for ds in data_result if ds.suffix == ".datasource"]:
|
|
161
160
|
parsed_url = urlparse(data)
|
|
162
161
|
if parsed_url.scheme in ("http", "https"):
|
|
163
|
-
response = requests.get(data)
|
|
162
|
+
response = requests.get(data)
|
|
164
163
|
data_content = response.text
|
|
165
164
|
data_format = parsed_url.path.split(".")[-1]
|
|
166
165
|
else:
|
|
@@ -182,7 +181,7 @@ async def create(
|
|
|
182
181
|
datasource_content = datasource_path.read_text()
|
|
183
182
|
has_json_path = "`json:" in datasource_content
|
|
184
183
|
if has_json_path:
|
|
185
|
-
mock_data =
|
|
184
|
+
mock_data = create_mock_data(
|
|
186
185
|
datasource_name,
|
|
187
186
|
datasource_content,
|
|
188
187
|
rows,
|
|
@@ -291,7 +290,7 @@ def create_project_structure(folder: str):
|
|
|
291
290
|
pass
|
|
292
291
|
|
|
293
292
|
|
|
294
|
-
|
|
293
|
+
def create_resources_from_prompt(
|
|
295
294
|
tb_client: TinyB,
|
|
296
295
|
user_token: str,
|
|
297
296
|
prompt: str,
|
|
@@ -445,7 +444,7 @@ def generate_connection_file(name: str, content: str, folder: str, skip_feedback
|
|
|
445
444
|
return f.relative_to(folder)
|
|
446
445
|
|
|
447
446
|
|
|
448
|
-
|
|
447
|
+
def generate_aws_iamrole_connection_file_with_secret(
|
|
449
448
|
name: str, service: str, role_arn_secret_name: str, region: str, folder: str, with_default_secret: bool = False
|
|
450
449
|
) -> Path:
|
|
451
450
|
if with_default_secret:
|
|
@@ -460,9 +459,7 @@ S3_REGION {region}
|
|
|
460
459
|
return file_path
|
|
461
460
|
|
|
462
461
|
|
|
463
|
-
|
|
464
|
-
name: str, service: str, svc_account_creds: str, folder: str
|
|
465
|
-
) -> Path:
|
|
462
|
+
def generate_gcs_connection_file_with_secrets(name: str, service: str, svc_account_creds: str, folder: str) -> Path:
|
|
466
463
|
content = f"""TYPE {service}
|
|
467
464
|
GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON {{{{ tb_secret("{svc_account_creds}") }}}}
|
|
468
465
|
"""
|
|
@@ -538,20 +535,20 @@ def get_resources_xml(project: Project) -> str:
|
|
|
538
535
|
return resources_xml
|
|
539
536
|
|
|
540
537
|
|
|
541
|
-
|
|
538
|
+
def create_resources_from_data(
|
|
542
539
|
data: str,
|
|
543
540
|
project: Project,
|
|
544
541
|
config: Dict[str, Any],
|
|
545
542
|
skip_pipes: bool = False,
|
|
546
543
|
) -> List[Path]:
|
|
547
|
-
local_client =
|
|
544
|
+
local_client = get_tinybird_local_client(config)
|
|
548
545
|
folder_path = project.path
|
|
549
546
|
path = folder_path / data
|
|
550
547
|
if not path.exists():
|
|
551
548
|
path = Path(data)
|
|
552
549
|
result: List[Path] = []
|
|
553
550
|
format = path.suffix.lstrip(".")
|
|
554
|
-
ds_file =
|
|
551
|
+
ds_file = _generate_datafile(str(path), local_client, format=format, force=True, folder=project.folder)
|
|
555
552
|
result.append(ds_file)
|
|
556
553
|
name = ds_file.stem
|
|
557
554
|
no_pipes = len(project.get_pipe_files()) == 0
|
|
@@ -570,13 +567,13 @@ TYPE ENDPOINT
|
|
|
570
567
|
return result
|
|
571
568
|
|
|
572
569
|
|
|
573
|
-
|
|
570
|
+
def create_resources_from_url(
|
|
574
571
|
url: str, project: Project, config: Dict[str, Any], skip_pipes: bool = False
|
|
575
572
|
) -> List[Path]:
|
|
576
573
|
result: List[Path] = []
|
|
577
|
-
local_client =
|
|
574
|
+
local_client = get_tinybird_local_client(config)
|
|
578
575
|
format = url.split(".")[-1]
|
|
579
|
-
ds_file =
|
|
576
|
+
ds_file = _generate_datafile(url, local_client, format=format, force=True, folder=project.folder)
|
|
580
577
|
result.append(ds_file)
|
|
581
578
|
name = ds_file.stem
|
|
582
579
|
no_pipes = len(project.get_pipe_files()) == 0
|
|
@@ -595,7 +592,7 @@ TYPE ENDPOINT
|
|
|
595
592
|
return result
|
|
596
593
|
|
|
597
594
|
|
|
598
|
-
|
|
595
|
+
def generate_kafka_connection_with_secrets(
|
|
599
596
|
name: str,
|
|
600
597
|
bootstrap_servers: str,
|
|
601
598
|
key: str,
|