tinybird 0.0.1.dev213__tar.gz → 0.0.1.dev214__tar.gz

{tinybird-0.0.1.dev213 → tinybird-0.0.1.dev214}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev213
+Version: 0.0.1.dev214
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev213/tinybird/tb/modules → tinybird-0.0.1.dev214/tinybird}/datafile/common.py

@@ -25,7 +25,7 @@ from croniter import croniter
 from mypy_extensions import KwArg, VarArg
 
 from tinybird.ch_utils.engine import ENABLED_ENGINES
-from tinybird.tb.modules.datafile.exceptions import IncludeFileNotFoundException, ParseException, ValidationException
+from tinybird.datafile.exceptions import IncludeFileNotFoundException, ParseException, ValidationException
 from tinybird.tb.modules.exceptions import CLIPipeException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 
@@ -473,19 +473,19 @@ def parse_indexes_structure(indexes: Optional[List[str]]) -> List[TableIndex]:
     >>> parse_indexes_structure(["index_name u64 * length(s)"])
     Traceback (most recent call last):
     ...
-    tinybird.tb.modules.datafile.common.IndexesSyntaxError: Invalid INDEX syntax at 1:1.
+    tinybird.datafile.common.IndexesSyntaxError: Invalid INDEX syntax at 1:1.
     Usage: `[INDEX] name expr TYPE type_full GRANULARITY granularity`.
 
     >>> parse_indexes_structure(["index_name a TYPE set(100) GRANULARITY 100, index_name_bf mapValues(d) TYPE bloom_filter(0.001) GRANULARITY 16"])
     Traceback (most recent call last):
     ...
-    tinybird.tb.modules.datafile.common.IndexesSyntaxError: Invalid INDEX syntax at 1:1.
+    tinybird.datafile.common.IndexesSyntaxError: Invalid INDEX syntax at 1:1.
     Usage: `[INDEX] name expr TYPE type_full GRANULARITY granularity`.
 
     >>> parse_indexes_structure(["", "    ", "     wrong_index_syntax,"])
     Traceback (most recent call last):
     ...
-    tinybird.tb.modules.datafile.common.IndexesSyntaxError: Invalid INDEX syntax at 3:6.
+    tinybird.datafile.common.IndexesSyntaxError: Invalid INDEX syntax at 3:6.
     Usage: `[INDEX] name expr TYPE type_full GRANULARITY granularity`.
 
     >>> parse_indexes_structure(["my_index m['key'] TYPE ngrambf_v1(1, 1024, 1, 42) GRANULARITY 1"])
@@ -1164,49 +1164,49 @@ def parse_table_structure(schema: str) -> List[Dict[str, Any]]:
         >>> parse_table_structure('col')  # Missing type
         Traceback (most recent call last):
         ...
-        tinybird.tb.modules.datafile.common.SchemaSyntaxError: Column name and either type or DEFAULT are required at 1:1.
+        tinybird.datafile.common.SchemaSyntaxError: Column name and either type or DEFAULT are required at 1:1.
 
         >>> parse_table_structure('`col Int32')  # Unclosed backtick
         Traceback (most recent call last):
         ...
-        tinybird.tb.modules.datafile.common.SchemaSyntaxError: Expected closing backtick at 1:5.
+        tinybird.datafile.common.SchemaSyntaxError: Expected closing backtick at 1:5.
 
         >>> parse_table_structure('col Int32 DEFAULT')  # Missing DEFAULT value
         Traceback (most recent call last):
         ...
-        tinybird.tb.modules.datafile.common.SchemaSyntaxError: Missing mandatory value for DEFAULT at 1:18.
+        tinybird.datafile.common.SchemaSyntaxError: Missing mandatory value for DEFAULT at 1:18.
 
         >>> parse_table_structure('col Int32 CODEC')  # Missing CODEC parameters
         Traceback (most recent call last):
         ...
-        tinybird.tb.modules.datafile.common.SchemaSyntaxError: Missing mandatory value for CODEC at 1:16.
+        tinybird.datafile.common.SchemaSyntaxError: Missing mandatory value for CODEC at 1:16.
 
         >>> parse_table_structure('col#name Int32')  # Invalid character in name
         Traceback (most recent call last):
         ...
-        tinybird.tb.modules.datafile.common.SchemaSyntaxError: Column name contains invalid character '#' at 1:4.
+        tinybird.datafile.common.SchemaSyntaxError: Column name contains invalid character '#' at 1:4.
         Hint: use backticks.
 
         >>> parse_table_structure('col Int32 MATERIALIZED expr')  # Unsupported MATERIALIZED
         Traceback (most recent call last):
         ...
-        tinybird.tb.modules.datafile.common.SchemaSyntaxError: MATERIALIZED columns are not supported at 1:11.
+        tinybird.datafile.common.SchemaSyntaxError: MATERIALIZED columns are not supported at 1:11.
 
         >>> parse_table_structure('col Int32 TTL timestamp + INTERVAL 1 DAY')  # Unsupported TTL
         Traceback (most recent call last):
         ...
-        tinybird.tb.modules.datafile.common.SchemaSyntaxError: column TTL is not supported at 1:11.
+        tinybird.datafile.common.SchemaSyntaxError: column TTL is not supported at 1:11.
 
         >>> parse_table_structure('col Int32 NULL')  # Unsupported NULL
         Traceback (most recent call last):
         ...
-        tinybird.tb.modules.datafile.common.SchemaSyntaxError: NULL column syntax not supported at 1:11.
+        tinybird.datafile.common.SchemaSyntaxError: NULL column syntax not supported at 1:11.
         Hint: use Nullable(...).
 
         >>> parse_table_structure('col Int32 NOT NULL')  # Unsupported NOT NULL
         Traceback (most recent call last):
         ...
-        tinybird.tb.modules.datafile.common.SchemaSyntaxError: NOT NULL column syntax not supported at 1:11.
+        tinybird.datafile.common.SchemaSyntaxError: NOT NULL column syntax not supported at 1:11.
         Hint: Columns are not nullable by default.
 
         >>> parse_table_structure('''
@@ -1215,7 +1215,7 @@ def parse_table_structure(schema: str) -> List[Dict[str, Any]]:
         ...             ZSTD''')  # Unclosed CODEC parenthesis across lines
         Traceback (most recent call last):
         ...
-        tinybird.tb.modules.datafile.common.SchemaSyntaxError: Expected closing ) at 4:17.
+        tinybird.datafile.common.SchemaSyntaxError: Expected closing ) at 4:17.
 
         >>> parse_table_structure('''
         ...     timestamp DateTime
@@ -1223,7 +1223,7 @@ def parse_table_structure(schema: str) -> List[Dict[str, Any]]:
         ...         CODEC(ZSTD)''')  # Missing DEFAULT value with following CODEC
         Traceback (most recent call last):
         ...
-        tinybird.tb.modules.datafile.common.SchemaSyntaxError: Missing mandatory value for DEFAULT at 3:16.
+        tinybird.datafile.common.SchemaSyntaxError: Missing mandatory value for DEFAULT at 3:16.
 
         >>> parse_table_structure('''
         ...     col String
@@ -1232,7 +1232,7 @@ def parse_table_structure(schema: str) -> List[Dict[str, Any]]:
         ...                 now()''')  # MATERIALIZED with heavy indentation
         Traceback (most recent call last):
         ...
-        tinybird.tb.modules.datafile.common.SchemaSyntaxError: MATERIALIZED columns are not supported at 4:13.
+        tinybird.datafile.common.SchemaSyntaxError: MATERIALIZED columns are not supported at 4:13.
 
         >>> parse_table_structure('''
         ...     `column.with.dots`
@@ -1241,7 +1241,7 @@ def parse_table_structure(schema: str) -> List[Dict[str, Any]]:
         ...                      timestamp + INTERVAL 1 DAY''')  # TTL with increasing indentation
         Traceback (most recent call last):
         ...
-        tinybird.tb.modules.datafile.common.SchemaSyntaxError: column TTL is not supported at 4:18.
+        tinybird.datafile.common.SchemaSyntaxError: column TTL is not supported at 4:18.
     """
     return _parse_table_structure(schema)
 
@@ -1666,15 +1666,6 @@ def parse(
             "shared_with": shared_with,  # Not supported yet
             "export_service": export_service,  # Deprecated
             "forward_query": sql("forward_query"),
-            "export_connection_name": assign_var("export_connection_name"),
-            "export_schedule": assign_var("export_schedule"),
-            "export_bucket_uri": assign_var("export_bucket_uri"),
-            "export_file_template": assign_var("export_file_template"),
-            "export_format": assign_var("export_format"),
-            "export_strategy": assign_var("export_strategy"),
-            "export_compression": assign_var("export_compression"),
-            "export_write_strategy": assign_var("export_write_strategy"),
-            "export_kafka_topic": assign_var("export_kafka_topic"),
             # ENGINE_* commands are added dynamically after this dict's definition
         },
         DatafileKind.pipe: {
@@ -1693,6 +1684,15 @@ def parse(
             "include": include,
             "sql": sql("sql"),
             "version": version,
+            "export_connection_name": assign_var("export_connection_name"),
+            "export_schedule": assign_var("export_schedule"),
+            "export_bucket_uri": assign_var("export_bucket_uri"),
+            "export_file_template": assign_var("export_file_template"),
+            "export_format": assign_var("export_format"),
+            "export_strategy": assign_var("export_strategy"),
+            "export_compression": assign_var("export_compression"),
+            "export_write_strategy": assign_var("export_write_strategy"),
+            "export_kafka_topic": assign_var("export_kafka_topic"),
         },
         DatafileKind.connection: {
             "description": description,

tinybird-0.0.1.dev214/tinybird/datafile/parse_connection.py

@@ -0,0 +1,60 @@
+import os
+from typing import Dict, Optional
+
+import click
+
+from tinybird.datafile.common import (
+    DatafileKind,
+    DatafileSyntaxError,
+    ParseResult,
+    format_filename,
+    parse,
+)
+from tinybird.datafile.exceptions import ParseException
+from tinybird.sql_template import render_template_with_secrets
+from tinybird.tb.modules.feedback_manager import FeedbackManager
+
+
+def parse_connection(
+    filename: str,
+    replace_includes: bool = True,
+    content: Optional[str] = None,
+    skip_eval: bool = False,
+    hide_folders: bool = False,
+    add_context_to_datafile_syntax_errors: bool = True,
+    secrets: Optional[Dict[str, str]] = None,
+) -> ParseResult:
+    basepath = ""
+    if not content:
+        with open(filename) as file:
+            s = file.read()
+        basepath = os.path.dirname(filename)
+    else:
+        s = content
+
+    s = render_template_with_secrets(filename, s, secrets=secrets or {})
+    filename = format_filename(filename, hide_folders)
+    try:
+        doc, warnings = parse(
+            s,
+            default_node="default",
+            basepath=basepath,
+            replace_includes=replace_includes,
+            skip_eval=skip_eval,
+            kind=DatafileKind.connection,
+        )
+        doc.validate()
+    except DatafileSyntaxError as e:
+        try:
+            if add_context_to_datafile_syntax_errors:
+                e.get_context_from_file_contents(s)
+        finally:
+            raise e
+    # TODO(eclbg): all these exceptions that trigger a ClickException shouldn't be here, as this code will only run in
+    # the server soon
+    except ParseException as e:
+        raise click.ClickException(
+            FeedbackManager.error_parsing_file(filename=filename, lineno=e.lineno, error=e)
+        ) from None
+
+    return ParseResult(datafile=doc, warnings=warnings)

{tinybird-0.0.1.dev213/tinybird/tb/modules → tinybird-0.0.1.dev214/tinybird}/datafile/parse_datasource.py

@@ -3,15 +3,15 @@ from typing import Dict, Optional
 
 import click
 
-from tinybird.sql_template import render_template_with_secrets
-from tinybird.tb.modules.datafile.common import (
+from tinybird.datafile.common import (
     DatafileKind,
     DatafileSyntaxError,
     ParseResult,
     format_filename,
     parse,
 )
-from tinybird.tb.modules.datafile.exceptions import ParseException
+from tinybird.datafile.exceptions import ParseException
+from tinybird.sql_template import render_template_with_secrets
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 

{tinybird-0.0.1.dev213/tinybird/tb/modules → tinybird-0.0.1.dev214/tinybird}/datafile/parse_pipe.py

@@ -3,15 +3,15 @@ from typing import Dict, List, Optional
 
 import click
 
-from tinybird.sql_template import get_template_and_variables, render_sql_template, secret_template_key
-from tinybird.tb.modules.datafile.common import (
+from tinybird.datafile.common import (
     DatafileKind,
     DatafileSyntaxError,
     ParseResult,
     format_filename,
     parse,
 )
-from tinybird.tb.modules.datafile.exceptions import IncludeFileNotFoundException, ParseException
+from tinybird.datafile.exceptions import IncludeFileNotFoundException, ParseException
+from tinybird.sql_template import get_template_and_variables, render_sql_template, secret_template_key
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tornado_template import UnClosedIfError
 

{tinybird-0.0.1.dev213 → tinybird-0.0.1.dev214}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev213'
-__revision__ = '36a1e58'
+__version__ = '0.0.1.dev214'
+__revision__ = '9ddb3a4'

{tinybird-0.0.1.dev213 → tinybird-0.0.1.dev214}/tinybird/tb/cli.py

@@ -25,6 +25,7 @@ import tinybird.tb.modules.mock
 import tinybird.tb.modules.open
 import tinybird.tb.modules.pipe
 import tinybird.tb.modules.secret
+import tinybird.tb.modules.sink
 import tinybird.tb.modules.test
 import tinybird.tb.modules.token
 import tinybird.tb.modules.workspace

{tinybird-0.0.1.dev213 → tinybird-0.0.1.dev214}/tinybird/tb/client.py

@@ -650,13 +650,13 @@ class TinyB:
     async def pipe_remove_copy(self, pipe_name_or_id: str, node_id: str):
         return await self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/copy", method="DELETE")
 
-    async def pipe_run_copy(
-        self, pipe_name_or_id: str, params: Optional[Dict[str, str]] = None, mode: Optional[str] = None
+    async def pipe_run(
+        self, pipe_name_or_id: str, pipe_type: str, params: Optional[Dict[str, str]] = None, mode: Optional[str] = None
     ):
         params = {**params} if params else {}
         if mode:
             params["_mode"] = mode
-        return await self._req(f"/v0/pipes/{pipe_name_or_id}/copy?{urlencode(params)}", method="POST")
+        return await self._req(f"/v0/pipes/{pipe_name_or_id}/{pipe_type}?{urlencode(params)}", method="POST")
 
     async def pipe_resume_copy(self, pipe_name_or_id: str):
         return await self._req(f"/v0/pipes/{pipe_name_or_id}/copy/resume", method="POST")

{tinybird-0.0.1.dev213 → tinybird-0.0.1.dev214}/tinybird/tb/modules/build.py

@@ -13,14 +13,14 @@ import click
 import requests
 
 import tinybird.context as context
+from tinybird.datafile.exceptions import ParseException
+from tinybird.datafile.parse_datasource import parse_datasource
+from tinybird.datafile.parse_pipe import parse_pipe
 from tinybird.tb.client import TinyB
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import push_data, sys_exit
 from tinybird.tb.modules.config import CLIConfig
-from tinybird.tb.modules.datafile.exceptions import ParseException
 from tinybird.tb.modules.datafile.fixture import FixtureExtension, get_fixture_dir, persist_fixture
-from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
-from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
 from tinybird.tb.modules.datafile.playground import folder_playground
 from tinybird.tb.modules.dev_server import BuildStatus, start_server
 from tinybird.tb.modules.feedback_manager import FeedbackManager

{tinybird-0.0.1.dev213 → tinybird-0.0.1.dev214}/tinybird/tb/modules/common.py

@@ -1707,6 +1707,7 @@ async def run_aws_iamrole_connection_flow(
     service: str,
     environment: str,
     connection_name: str,
+    policy: str,
 ) -> Tuple[str, str, str]:
     if service == DataConnectorType.AMAZON_DYNAMODB:
         raise NotImplementedError("DynamoDB is not supported")
@@ -1721,7 +1722,7 @@ async def run_aws_iamrole_connection_flow(
     validate_string_connector_param("Region", region)
 
     access_policy, trust_policy, _ = await get_aws_iamrole_policies(
-        client, service=service, policy="read", bucket=bucket_name, external_id_seed=connection_name
+        client, service=service, policy=policy, bucket=bucket_name, external_id_seed=connection_name
     )
 
     click.echo(FeedbackManager.prompt_s3_iamrole_connection_login_aws())
@@ -1810,9 +1811,10 @@ async def production_aws_iamrole_only(
     bucket_name: str,
     environment: str,
     connection_name: str,
+    policy: str,
 ) -> Tuple[str, str, str]:
     _, trust_policy, external_id = await get_aws_iamrole_policies(
-        prod_client, service=service, policy="read", bucket=bucket_name, external_id_seed=connection_name
+        prod_client, service=service, policy=policy, bucket=bucket_name, external_id_seed=connection_name
     )
 
     trust_policy_copied = True

{tinybird-0.0.1.dev213 → tinybird-0.0.1.dev214}/tinybird/tb/modules/connection.py

@@ -178,12 +178,28 @@ async def connection_create_s3(ctx: Context) -> None:
 
     service = DataConnectorType.AMAZON_S3
     click.echo(FeedbackManager.prompt_s3_connection_header())
+
+    # Ask user for access type
+    access_type = click.prompt(
+        FeedbackManager.highlight(
+            message="What type of access do you need for this S3 connection?\n"
+            '  - "read" for S3 Data Source (reading from S3)\n'
+            '  - "write" for S3 Sink (writing to S3)\n'
+            "Access type",
+        ),
+        type=click.Choice(["read", "write"], case_sensitive=False),
+        default="read",
+        show_choices=True,
+        show_default=True,
+    )
+
     connection_name = get_s3_connection_name(project.folder)
     role_arn, region, bucket_name = await run_aws_iamrole_connection_flow(
         client,
         service=service,
         environment=obj["env"],
         connection_name=connection_name,
+        policy=access_type.lower(),
     )
     unique_suffix = uuid.uuid4().hex[:8]  # Use first 8 chars of a UUID for brevity
     secret_name = f"s3_role_arn_{connection_name}_{unique_suffix}"
@@ -211,6 +227,7 @@ async def connection_create_s3(ctx: Context) -> None:
             bucket_name=bucket_name,
             environment="cloud",
             connection_name=connection_name,
+            policy=access_type.lower(),
         )
         await prod_client.create_secret(name=secret_name, value=prod_role_arn)
 
@@ -222,12 +239,20 @@ async def connection_create_s3(ctx: Context) -> None:
         folder=project.folder,
     )
 
-    click.echo(
-        FeedbackManager.prompt_s3_iamrole_success(
-            connection_name=connection_name,
-            connection_path=str(connection_file_path),
+    if access_type.lower() == "write":
+        click.echo(
+            FeedbackManager.prompt_s3_iamrole_success_write(
+                connection_name=connection_name,
+                connection_path=str(connection_file_path),
+            )
+        )
+    else:
+        click.echo(
+            FeedbackManager.prompt_s3_iamrole_success_read(
+                connection_name=connection_name,
+                connection_path=str(connection_file_path),
+            )
         )
-    )
 
 
 @connection_create.command(name="gcs", short_help="Creates a Google Cloud Storage connection.")

{tinybird-0.0.1.dev213 → tinybird-0.0.1.dev214}/tinybird/tb/modules/copy.py

@@ -10,10 +10,10 @@ from typing import Optional, Tuple
 import click
 from click import Context
 
+from tinybird.datafile.common import get_name_version
 from tinybird.tb.client import AuthNoTokenException, TinyB
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import coro, echo_safe_humanfriendly_tables_format_smart_table, wait_job
-from tinybird.tb.modules.datafile.common import get_name_version
 from tinybird.tb.modules.exceptions import CLIPipeException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 
@@ -93,8 +93,7 @@ async def copy_run(ctx: click.Context, pipe_name_or_id: str, wait: bool, mode: s
     client: TinyB = ctx.ensure_object(dict)["client"]
 
     try:
-        response = await client.pipe_run_copy(pipe_name_or_id, params, mode)
-
+        response = await client.pipe_run(pipe_name_or_id, "copy", params, mode)
         job_id = response["job"]["job_id"]
         job_url = response["job"]["job_url"]
         target_datasource_id = response["tags"]["copy_target_datasource"]
@@ -112,43 +111,3 @@ async def copy_run(ctx: click.Context, pipe_name_or_id: str, wait: bool, mode: s
         raise
     except Exception as e:
         raise CLIPipeException(FeedbackManager.error_creating_copy_job(error=e))
-
-
-@copy.command(name="resume", short_help="Resume a paused copy pipe")
-@click.argument("pipe_name_or_id")
-@click.pass_context
-@coro
-async def copy_resume(ctx: click.Context, pipe_name_or_id: str):
-    """Resume a paused copy pipe"""
-
-    click.echo(FeedbackManager.info_copy_pipe_resuming(pipe=pipe_name_or_id))
-    client: TinyB = ctx.ensure_object(dict)["client"]
-
-    try:
-        await client.pipe_resume_copy(pipe_name_or_id)
-        click.echo(FeedbackManager.success_copy_pipe_resumed(pipe=pipe_name_or_id))
-
-    except AuthNoTokenException:
-        raise
-    except Exception as e:
-        raise CLIPipeException(FeedbackManager.error_resuming_copy_pipe(error=e))
-
-
-@copy.command(name="pause", short_help="Pause a running copy pipe")
-@click.argument("pipe_name_or_id")
-@click.pass_context
-@coro
-async def copy_pause(ctx: click.Context, pipe_name_or_id: str):
-    """Pause a running copy pipe"""
-
-    click.echo(FeedbackManager.info_copy_pipe_pausing(pipe=pipe_name_or_id))
-    client: TinyB = ctx.ensure_object(dict)["client"]
-
-    try:
-        await client.pipe_pause_copy(pipe_name_or_id)
-        click.echo(FeedbackManager.success_copy_pipe_paused(pipe=pipe_name_or_id))
-
-    except AuthNoTokenException:
-        raise
-    except Exception as e:
-        raise CLIPipeException(FeedbackManager.error_pausing_copy_pipe(error=e))

{tinybird-0.0.1.dev213 → tinybird-0.0.1.dev214}/tinybird/tb/modules/datafile/build.py

@@ -11,18 +11,7 @@ from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
 import click
 from toposort import toposort
 
-from tinybird.sql import parse_table_structure, schema_to_sql_columns
-from tinybird.sql_template import get_used_tables_in_template, render_sql_template
-from tinybird.tb.client import TinyB
-from tinybird.tb.modules.common import get_ca_pem_content
-from tinybird.tb.modules.datafile.build_datasource import is_datasource
-from tinybird.tb.modules.datafile.build_pipe import (
-    get_target_materialized_data_source_name,
-    is_endpoint,
-    is_endpoint_with_no_dependencies,
-    is_materialized,
-)
-from tinybird.tb.modules.datafile.common import (
+from tinybird.datafile.common import (
     DEFAULT_CRON_PERIOD,
     INTERNAL_TABLES,
     ON_DEMAND,
@@ -38,9 +27,20 @@ from tinybird.tb.modules.datafile.common import (
     get_project_filenames,
     pp,
 )
-from tinybird.tb.modules.datafile.exceptions import AlreadyExistsException, IncludeFileNotFoundException
-from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
-from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
+from tinybird.datafile.exceptions import AlreadyExistsException, IncludeFileNotFoundException
+from tinybird.datafile.parse_datasource import parse_datasource
+from tinybird.datafile.parse_pipe import parse_pipe
+from tinybird.sql import parse_table_structure, schema_to_sql_columns
+from tinybird.sql_template import get_used_tables_in_template, render_sql_template
+from tinybird.tb.client import TinyB
+from tinybird.tb.modules.common import get_ca_pem_content
+from tinybird.tb.modules.datafile.build_datasource import is_datasource
+from tinybird.tb.modules.datafile.build_pipe import (
+    get_target_materialized_data_source_name,
+    is_endpoint,
+    is_endpoint_with_no_dependencies,
+    is_materialized,
+)
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.project import Project
 

{tinybird-0.0.1.dev213 → tinybird-0.0.1.dev214}/tinybird/tb/modules/datafile/build_datasource.py

@@ -5,8 +5,8 @@ from typing import Any, Dict, List, Optional
 
 import click
 
+from tinybird.datafile.common import PREVIEW_CONNECTOR_SERVICES, ImportReplacements
 from tinybird.tb.client import DoesNotExistException, TinyB
-from tinybird.tb.modules.datafile.common import PREVIEW_CONNECTOR_SERVICES, ImportReplacements
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 

{tinybird-0.0.1.dev213 → tinybird-0.0.1.dev214}/tinybird/tb/modules/datafile/build_pipe.py

@@ -6,10 +6,10 @@ from urllib.parse import urlencode
 import click
 import requests
 
+from tinybird.datafile.common import PipeNodeTypes
 from tinybird.tb.client import TinyB
 from tinybird.tb.modules.common import requests_get
 from tinybird.tb.modules.config import CLIConfig
-from tinybird.tb.modules.datafile.common import PipeNodeTypes
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 

{tinybird-0.0.1.dev213 → tinybird-0.0.1.dev214}/tinybird/tb/modules/datafile/diff.py

@@ -22,9 +22,9 @@ import sys
 
 import click
 
+from tinybird.datafile.common import get_name_version, get_project_filenames, is_file_a_datasource, peek
 from tinybird.sql_template_fmt import DEFAULT_FMT_LINE_LENGTH
 from tinybird.tb.client import TinyB
-from tinybird.tb.modules.datafile.common import get_name_version, get_project_filenames, is_file_a_datasource, peek
 from tinybird.tb.modules.datafile.format_datasource import format_datasource
 from tinybird.tb.modules.datafile.format_pipe import format_pipe
 from tinybird.tb.modules.datafile.pull import folder_pull

{tinybird-0.0.1.dev213 → tinybird-0.0.1.dev214}/tinybird/tb/modules/datafile/format_common.py

@@ -1,6 +1,6 @@
 from typing import Any, List
 
-from tinybird.tb.modules.datafile.common import Datafile
+from tinybird.datafile.common import Datafile
 
 DATAFILE_NEW_LINE = "\n"
 DATAFILE_INDENT = " " * 4

{tinybird-0.0.1.dev213 → tinybird-0.0.1.dev214}/tinybird/tb/modules/datafile/format_datasource.py

@@ -1,8 +1,9 @@
 from typing import Any, Dict, List, Optional
 
+from tinybird.datafile.common import Datafile
+from tinybird.datafile.parse_datasource import parse_datasource
 from tinybird.sql import schema_to_sql_columns
 from tinybird.tb.client import TinyB
-from tinybird.tb.modules.datafile.common import Datafile
 from tinybird.tb.modules.datafile.format_common import (
     DATAFILE_INDENT,
     DATAFILE_NEW_LINE,
@@ -12,7 +13,6 @@ from tinybird.tb.modules.datafile.format_common import (
     format_tags,
     format_tokens,
 )
-from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
 
 
 async def format_datasource(

{tinybird-0.0.1.dev213 → tinybird-0.0.1.dev214}/tinybird/tb/modules/datafile/format_pipe.py

@@ -2,8 +2,7 @@ import os
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
-from tinybird.sql_template_fmt import DEFAULT_FMT_LINE_LENGTH, format_sql_template
-from tinybird.tb.modules.datafile.common import (
+from tinybird.datafile.common import (
     ON_DEMAND,
     CopyParameters,
     Datafile,
@@ -12,6 +11,8 @@ from tinybird.tb.modules.datafile.common import (
     _unquote,
     eval_var,
 )
+from tinybird.datafile.parse_pipe import parse_pipe
+from tinybird.sql_template_fmt import DEFAULT_FMT_LINE_LENGTH, format_sql_template
 from tinybird.tb.modules.datafile.format_common import (
     DATAFILE_INDENT,
     DATAFILE_NEW_LINE,
@@ -21,7 +22,6 @@ from tinybird.tb.modules.datafile.format_common import (
     format_tokens,
 )
 from tinybird.tb.modules.datafile.format_datasource import format_engine
-from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
 
 
 def format_node_sql(

{tinybird-0.0.1.dev213 → tinybird-0.0.1.dev214}/tinybird/tb/modules/datafile/pipe_checker.py

@@ -14,8 +14,8 @@ import requests
 from humanfriendly import format_size
 from requests import Response
 
+from tinybird.datafile.common import normalize_array
 from tinybird.tb.modules.common import getenv_bool
-from tinybird.tb.modules.datafile.common import normalize_array
 
 PIPE_CHECKER_RETRIES: int = 3