tinybird 0.0.1.dev188__py3-none-any.whl → 0.0.1.dev190__py3-none-any.whl

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev188'
-__revision__ = 'f13c1f2'
+__version__ = '0.0.1.dev190'
+__revision__ = '8a00bcf'

tinybird/tb/client.py

@@ -1024,11 +1024,7 @@ class TinyB:
         )
 
     async def kafka_list_topics(self, connection_id: str, timeout=5):
-        resp = await self._req(
-            f"/v0/connectors/{connection_id}/preview?preview_activity=false",
-            connect_timeout=timeout,
-            request_timeout=timeout,
-        )
+        resp = await self._req(f"/v0/connectors/{connection_id}/preview?preview_activity=false", timeout=timeout)
         return [x["topic"] for x in resp["preview"]]
 
     async def get_gcp_service_account_details(self) -> Dict[str, Any]:

tinybird/tb/modules/build.py

@@ -38,6 +38,15 @@ def build(ctx: click.Context, watch: bool) -> None:
     """
     project: Project = ctx.ensure_object(dict)["project"]
     tb_client: TinyB = ctx.ensure_object(dict)["client"]
+
+    if project.has_deeper_level():
+        click.echo(
+            FeedbackManager.warning(
+                message="Default max_depth is 2, but project has deeper level. "
+                "You can change max_depth by running `tb --max-depth <depth> <cmd>`"
+            )
+        )
+
     click.echo(FeedbackManager.highlight_building_project())
     process(project=project, tb_client=tb_client, watch=False)
     if watch:

tinybird/tb/modules/common.py

@@ -424,6 +424,14 @@ async def _analyze(filename: str, client: TinyB, format: str, connector: Optiona
     return meta, data
 
 
+async def analyze_file(filename: str, client: TinyB, format: str):
+    meta, data = await _analyze(filename, client, format)
+    schema = meta["analysis"]["schema"]
+    schema = schema.replace(", ", ",\n    ")
+    content = f"""DESCRIPTION >\n    Generated from {filename}\n\nSCHEMA >\n    {schema}"""
+    return content
+
+
 async def _generate_datafile(
     filename: str,
     client: TinyB,
@@ -971,7 +979,7 @@ async def push_data(
     cb.prev_done = 0  # type: ignore[attr-defined]
 
     if not silent:
-        click.echo(FeedbackManager.gray(message=f"\nImporting data to {datasource_name}..."))
+        click.echo(FeedbackManager.highlight(message=f"\n» Appending data to {datasource_name}..."))
 
     if isinstance(url, list):
         urls = url
@@ -1882,15 +1890,19 @@ def get_gcs_connection_name(project_folder) -> str:
     return get_connection_name(project_folder=project_folder, connection_type="GCS")
 
 
-def get_connection_name(project_folder: str, connection_type: str) -> str:
-    connection_name = None
+def get_kafka_connection_name(project_folder: str, connection_name: Optional[str] = None) -> str:
+    return get_connection_name(project_folder=project_folder, connection_type="KAFKA", connection_name=connection_name)
+
+
+def get_connection_name(project_folder: str, connection_type: str, connection_name: Optional[str] = None) -> str:
     valid_pattern = r"^[a-zA-Z][a-zA-Z0-9_]*$"
 
     while not connection_name:
+        short_id = str(uuid.uuid4())[:4]
         connection_name = click.prompt(
-            f"🔗 Enter a name for your new Tinybird {connection_type} connection (use alphanumeric characters, and underscores)",
-            prompt_suffix="\n> ",
+            "Enter a name (only alphanumeric characters and underscores)",
             show_default=True,
+            default=f"{connection_type.lower()}_{short_id}",
         )
         assert isinstance(connection_name, str)
 

tinybird/tb/modules/connection.py

@@ -18,6 +18,7 @@ from tinybird.tb.modules.common import (
     echo_safe_humanfriendly_tables_format_smart_table,
     get_gcs_connection_name,
     get_gcs_svc_account_creds,
+    get_kafka_connection_name,
     get_s3_connection_name,
     production_aws_iamrole_only,
     run_aws_iamrole_connection_flow,
@@ -26,6 +27,7 @@ from tinybird.tb.modules.common import (
 from tinybird.tb.modules.create import (
     generate_aws_iamrole_connection_file_with_secret,
     generate_gcs_connection_file_with_secrets,
+    generate_kafka_connection_with_secrets,
 )
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.project import Project
@@ -275,3 +277,21 @@ async def connection_create_gcs(ctx: Context) -> None:
             connection_path=connection_path,
         )
     )
+
+
+@connection_create.command(name="kafka", short_help="Creates a Kafka connection.")
+@click.option("--name", help="The name of the connection")
+@click.pass_context
+@coro
+async def connection_create_kafka(ctx: Context, name: Optional[str] = None) -> None:
+    """
+    Creates a Kafka connection.
+
+    \b
+    $ tb connection create kafka
+    """
+    click.echo(FeedbackManager.highlight(message="» Creating Kafka connection..."))
+    project: Project = ctx.ensure_object(dict)["project"]
+    name = get_kafka_connection_name(project.folder, name)
+    await generate_kafka_connection_with_secrets(name=name, folder=project.folder)
+    click.echo(FeedbackManager.success(message="✓ Done!"))

tinybird/tb/modules/create.py

@@ -409,10 +409,14 @@ def generate_connection_file(name: str, content: str, folder: str, skip_feedback
 
 
 async def generate_aws_iamrole_connection_file_with_secret(
-    name: str, service: str, role_arn_secret_name: str, region: str, folder: str
+    name: str, service: str, role_arn_secret_name: str, region: str, folder: str, with_default_secret: bool = False
 ) -> Path:
+    if with_default_secret:
+        default_secret = ', "arn:aws:iam::123456789012:role/my-role"'
+    else:
+        default_secret = ""
     content = f"""TYPE {service}
-S3_ARN {{{{ tb_secret("{role_arn_secret_name}") }}}}
+S3_ARN {{{{ tb_secret("{role_arn_secret_name}"{default_secret}) }}}}
 S3_REGION {region}
 """
     file_path = generate_connection_file(name, content, folder, skip_feedback=True)
@@ -483,6 +487,7 @@ async def create_resources_from_data(
     data: str,
     project: Project,
     config: Dict[str, Any],
+    skip_pipes: bool = False,
 ) -> List[Path]:
     local_client = await get_tinybird_local_client(config)
     folder_path = project.path
@@ -495,7 +500,7 @@ async def create_resources_from_data(
     result.append(ds_file)
     name = ds_file.stem
     no_pipes = len(project.get_pipe_files()) == 0
-    if no_pipes:
+    if not skip_pipes and no_pipes:
         pipe_file = generate_pipe_file(
             f"{name}_endpoint",
             f"""
@@ -510,7 +515,9 @@ TYPE ENDPOINT
     return result
 
 
-async def create_resources_from_url(url: str, project: Project, config: Dict[str, Any]) -> List[Path]:
+async def create_resources_from_url(
+    url: str, project: Project, config: Dict[str, Any], skip_pipes: bool = False
+) -> List[Path]:
     result: List[Path] = []
     local_client = await get_tinybird_local_client(config)
     format = url.split(".")[-1]
@@ -518,7 +525,7 @@ async def create_resources_from_url(url: str, project: Project, config: Dict[str
     result.append(ds_file)
     name = ds_file.stem
     no_pipes = len(project.get_pipe_files()) == 0
-    if no_pipes:
+    if not skip_pipes and no_pipes:
         pipe_file = generate_pipe_file(
             f"{name}_endpoint",
             f"""
@@ -531,3 +538,14 @@ TYPE ENDPOINT
         )
         result.append(pipe_file)
     return result
+
+
+async def generate_kafka_connection_with_secrets(name: str, folder: str) -> Path:
+    content = """TYPE kafka
+KAFKA_BOOTSTRAP_SERVERS {{ tb_secret("KAFKA_SERVERS", "localhost:9092") }}
+KAFKA_SECURITY_PROTOCOL SASL_SSL
+KAFKA_SASL_MECHANISM PLAIN
+KAFKA_KEY {{ tb_secret("KAFKA_USERNAME", "") }}
+KAFKA_SECRET {{ tb_secret("KAFKA_PASSWORD", "") }}
+"""
+    return generate_connection_file(name, content, folder, skip_feedback=True)

tinybird/tb/modules/datasource.py

@@ -7,22 +7,37 @@ import asyncio
 import json
 import os
 import re
+import uuid
+from datetime import datetime
+from pathlib import Path
 from typing import Optional
+from urllib.parse import urlparse
 
 import click
 import humanfriendly
+import requests
 from click import Context
 
+from tinybird.syncasync import sync_to_async
 from tinybird.tb.client import AuthNoTokenException, DoesNotExistException, TinyB
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import (
     _analyze,
+    analyze_file,
     coro,
     echo_safe_humanfriendly_tables_format_smart_table,
     get_format_from_filename_or_url,
     load_connector_config,
+    normalize_datasource_name,
     push_data,
 )
+from tinybird.tb.modules.config import CLIConfig
+from tinybird.tb.modules.create import (
+    create_resources_from_prompt,
+    generate_aws_iamrole_connection_file_with_secret,
+    generate_gcs_connection_file_with_secrets,
+    generate_kafka_connection_with_secrets,
+)
 from tinybird.tb.modules.datafile.common import get_name_version
 from tinybird.tb.modules.datafile.fixture import persist_fixture
 from tinybird.tb.modules.exceptions import CLIDatasourceException
@@ -107,33 +122,169 @@ async def datasource_ls(ctx: Context, match: Optional[str], format_: str):
 
 
 @datasource.command(name="append")
-@click.argument("datasource_name", required=True)
-@click.argument("url", nargs=-1, required=True)
+@click.argument("datasource_name", required=False)
+@click.argument("data", nargs=-1, required=False)
+@click.option("--url", type=str, help="URL to append data from")
+@click.option("--file", type=str, help="Local file to append data from")
+@click.option("--events", type=str, help="Events to append data from")
 @click.option("--concurrency", help="How many files to submit concurrently", default=1, hidden=True)
 @click.pass_context
 @coro
 async def datasource_append(
     ctx: Context,
     datasource_name: str,
-    url,
+    data: Optional[str],
+    url: str,
+    file: str,
+    events: str,
     concurrency: int,
 ):
     """
     Appends data to an existing data source from URL, local file  or a connector
 
-    - Load from URL `tb datasource append [datasource_name] https://url_to_csv`
+    - Events API: `tb datasource append [datasource_name] --events '{"a":"b, "c":"d"}'`\n
+    - Local File: `tb datasource append [datasource_name] --file /path/to/local/file`\n
+    - Remote URL: `tb datasource append [datasource_name] --url https://url_to_csv`\n
+    - Kafka, S3 and GCS: https://www.tinybird.co/docs/forward/get-data-in/connectors\n
 
-    - Load from local file `tb datasource append [datasource_name] /path/to/local/file`
+    More info: https://www.tinybird.co/docs/forward/get-data-in
     """
-
+    env: str = ctx.ensure_object(dict)["env"]
     client: TinyB = ctx.obj["client"]
-    await push_data(
-        client,
-        datasource_name,
-        url,
-        mode="append",
-        concurrency=concurrency,
-    )
+
+    # If data is passed as argument, we detect if it's a JSON object, a URL or a file
+    if data:
+        try:
+            json.loads(data)
+            events = data
+        except Exception:
+            pass
+        if urlparse(data).scheme in ("http", "https"):
+            url = data
+        if not events and not url:
+            file = data
+
+    # If data is not passed as argument, we use the data from the options
+    if not data:
+        data = file or url or events
+
+    if env == "local":
+        tip = "Did you build your project? Run `tb build` first."
+    else:
+        tip = "Did you deploy your project? Run `tb --cloud deploy` first."
+
+    datasources = await client.datasources()
+    if not datasources:
+        raise CLIDatasourceException(FeedbackManager.error(message=f"No data sources found. {tip}"))
+
+    if datasource_name and datasource_name not in [ds["name"] for ds in datasources]:
+        raise CLIDatasourceException(FeedbackManager.error(message=f"Datasource {datasource_name} not found. {tip}"))
+
+    if not datasource_name:
+        datasource_index = -1
+
+        click.echo(FeedbackManager.info(message="\n? Which data source do you want to ingest data into?"))
+        while datasource_index == -1:
+            for index, datasource in enumerate(datasources):
+                click.echo(f"  [{index + 1}] {datasource['name']}")
+            click.echo(
+                FeedbackManager.gray(message="Tip: Run tb datasource append [datasource_name] to skip this step.")
+            )
+
+            datasource_index = click.prompt("\nSelect option", default=1)
+
+            if datasource_index == 0:
+                click.echo(FeedbackManager.warning(message="Datasource type selection cancelled by user"))
+                return None
+
+            try:
+                datasource_name = datasources[int(datasource_index) - 1]["name"]
+            except Exception:
+                datasource_index = -1
+
+    if not datasource_name:
+        raise CLIDatasourceException(FeedbackManager.error_datasource_name())
+
+    if not data:
+        data_index = -1
+        options = (
+            "Events API",
+            "Local File",
+            "Remote URL",
+        )
+        click.echo(FeedbackManager.info(message="\n? How do you want to ingest data?"))
+        while data_index == -1:
+            for index, option in enumerate(options):
+                click.echo(f"  [{index + 1}] {option}")
+            click.echo(
+                FeedbackManager.gray(
+                    message="Tip: Run tb datasource append [datasource_name] --events | --file | --url to skip this step"
+                )
+            )
+
+            data_index = click.prompt("\nSelect option", default=1)
+
+            if data_index == 0:
+                click.echo(FeedbackManager.warning(message="Data selection cancelled by user"))
+                return None
+
+            try:
+                data_index = int(data_index)
+            except Exception:
+                data_index = -1
+
+        if data_index == 1:
+            events = click.prompt("Events data")
+        elif data_index == 2:
+            data = click.prompt("Path to local file")
+        elif data_index == 3:
+            data = click.prompt("URL to remote file")
+        else:
+            raise CLIDatasourceException(FeedbackManager.error(message="Invalid ingestion option"))
+
+    if events:
+        click.echo(FeedbackManager.highlight(message=f"\n» Sending events to {datasource_name}"))
+        try:
+            json_data = json.loads(events)
+        except Exception:
+            raise CLIDatasourceException(FeedbackManager.error(message="Invalid events data"))
+
+        response = await sync_to_async(requests.post)(
+            f"{client.host}/v0/events?name={datasource_name}",
+            headers={"Authorization": f"Bearer {client.token}"},
+            json=json_data,
+        )
+
+        try:
+            res = response.json()
+        except Exception:
+            raise CLIDatasourceException(FeedbackManager.error(message=response.text))
+
+        successful_rows = res["successful_rows"]
+        quarantined_rows = res["quarantined_rows"]
+        if successful_rows > 0:
+            click.echo(
+                FeedbackManager.success(
+                    message=f"✓ {successful_rows} row{'' if successful_rows == 1 else 's'} appended!"
+                )
+            )
+        if quarantined_rows > 0:
+            raise CLIDatasourceException(
+                FeedbackManager.error(
+                    message=f"{quarantined_rows} row{'' if quarantined_rows == 1 else 's'} went to quarantine"
+                )
+            )
+    else:
+        click.echo(FeedbackManager.highlight(message=f"\n» Appending data to {datasource_name}"))
+        await push_data(
+            client,
+            datasource_name,
+            data,
+            mode="append",
+            concurrency=concurrency,
+            silent=True,
+        )
+        click.echo(FeedbackManager.success(message="✓ Rows appended!"))
 
 
 @datasource.command(name="replace")
@@ -473,3 +624,281 @@ async def datasource_sync(ctx: Context, datasource_name: str, yes: bool):
         raise
     except Exception as e:
         raise CLIDatasourceException(FeedbackManager.error_syncing_datasource(datasource=datasource_name, error=str(e)))
+
+
+@datasource.command(name="create")
+@click.option("--name", type=str, help="Name of the data source")
+@click.option("--blank", is_flag=True, default=False, help="Create a blank data source")
+@click.option("--file", type=str, help="Create a data source from a local file")
+@click.option("--url", type=str, help="Create a data source from a remote URL")
+@click.option("--connection", type=str, help="Create a data source from a connection")
+@click.option("--prompt", type=str, help="Create a data source from a prompt")
+@click.option("--s3", is_flag=True, default=False, help="Create a data source from a S3 connection")
+@click.option("--gcs", is_flag=True, default=False, help="Create a data source from a GCS connection")
+@click.option("--kafka", is_flag=True, default=False, help="Create a data source from a Kafka connection")
+@click.pass_context
+@coro
+async def datasource_create(
+    ctx: Context,
+    name: str,
+    blank: bool,
+    file: str,
+    url: str,
+    connection: str,
+    prompt: str,
+    s3: bool,
+    gcs: bool,
+    kafka: bool,
+):
+    project: Project = ctx.ensure_object(dict)["project"]
+    client: TinyB = ctx.ensure_object(dict)["client"]
+    config = ctx.ensure_object(dict)["config"]
+    env: str = ctx.ensure_object(dict)["env"]
+
+    if env == "cloud":
+        raise CLIDatasourceException(
+            FeedbackManager.error(message="`tb datasource create` is not available against Tinybird Cloud.")
+        )
+
+    datasource_types = (
+        "Blank",
+        "Local file",
+        "Remote URL",
+        "Kafka",
+        "S3",
+        "GCS",
+    )
+    datasource_type: Optional[str] = None
+    connection_file: Optional[str] = None
+    ds_content = ""
+
+    if file:
+        datasource_type = "Local file"
+    elif url:
+        datasource_type = "Remote URL"
+    elif blank:
+        datasource_type = "Blank"
+    elif connection:
+        connection_files = project.get_connection_files()
+        connection_file = next((f for f in connection_files if f.endswith(f"{connection}.connection")), None)
+        if connection_file:
+            connection_content = Path(connection_file).read_text()
+            if project.is_kafka_connection(connection_content):
+                datasource_type = "Kafka"
+            elif project.is_s3_connection(connection_content):
+                datasource_type = "S3"
+            elif project.is_gcs_connection(connection_content):
+                datasource_type = "GCS"
+    elif s3:
+        datasource_type = "S3"
+    elif gcs:
+        datasource_type = "GCS"
+    elif kafka:
+        datasource_type = "Kafka"
+    elif prompt:
+        click.echo(FeedbackManager.gray(message="\n» Creating .datasource file..."))
+        user_token = config.get("user_token")
+        if not user_token:
+            raise Exception("This action requires authentication. Run 'tb login' first.")
+        project_config = CLIConfig.get_project_config()
+        tb_client: TinyB = project_config.get_client(token=config.get("token"), host=config.get("host"))
+        await create_resources_from_prompt(tb_client, user_token, prompt, project)
+        click.echo(FeedbackManager.success(message="✓ .datasource created!"))
+        return
+
+    if datasource_type is None:
+        click.echo(FeedbackManager.highlight(message="? Where do you want to create your .datasource from?"))
+        datasource_type_index = -1
+
+        while datasource_type_index == -1:
+            for index, datasource_type in enumerate(datasource_types):
+                click.echo(f"  [{index + 1}] {datasource_type}")
+            click.echo(
+                FeedbackManager.gray(
+                    message="Tip: Run `tb datasource create --file | --url | --connection` to skip this step."
+                )
+            )
+            datasource_type_index = click.prompt("\nSelect option", default=1)
+
+            if datasource_type_index == 0:
+                click.echo(FeedbackManager.warning(message="Datasource type selection cancelled by user"))
+                return None
+
+            try:
+                datasource_type = datasource_types[int(datasource_type_index) - 1]
+            except Exception:
+                datasource_type_index = -1
+
+    if not datasource_type:
+        click.echo(
+            FeedbackManager.error(
+                message=f"Invalid option: {datasource_type_index}. Please select a valid option from the list above."
+            )
+        )
+        return
+
+    connection_required = datasource_type in ("Kafka", "S3", "GCS")
+
+    if connection_required:
+        click.echo(FeedbackManager.gray(message="\n» Creating .datasource file..."))
+        connection_type = datasource_type.lower()
+
+        def get_connection_files():
+            connection_files = []
+            if connection_type == "kafka":
+                connection_files = project.get_kafka_connection_files()
+            elif connection_type == "s3":
+                connection_files = project.get_s3_connection_files()
+            elif connection_type == "gcs":
+                connection_files = project.get_gcs_connection_files()
+            return connection_files
+
+        connection_files = get_connection_files()
+        if len(connection_files) == 0:
+            click.echo(FeedbackManager.error(message=f"x No {datasource_type} connections found."))
+            if click.confirm(
+                FeedbackManager.highlight(message=f"\n? Do you want to create a {datasource_type} connection? [Y/n]"),
+                show_default=False,
+                default=True,
+            ):
+                click.echo(FeedbackManager.gray(message="\n» Creating .connection file..."))
+                default_connection_name = f"{datasource_type.lower()}_{generate_short_id()}"
+                connection_name = click.prompt(
+                    FeedbackManager.highlight(message=f"? Connection name [{default_connection_name}]"),
+                    show_default=False,
+                    default=default_connection_name,
+                )
+                if datasource_type == "Kafka":
+                    await generate_kafka_connection_with_secrets(connection_name, folder=project.folder)
+                elif datasource_type == "S3":
+                    await generate_aws_iamrole_connection_file_with_secret(
+                        connection_name,
+                        service="s3",
+                        role_arn_secret_name="S3_ARN",
+                        region="eu-west-1",
+                        folder=project.folder,
+                        with_default_secret=True,
+                    )
+                elif datasource_type == "GCS":
+                    await generate_gcs_connection_file_with_secrets(
+                        connection_name,
+                        service="gcs",
+                        svc_account_creds="GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON",
+                        folder=project.folder,
+                    )
+                click.echo(FeedbackManager.info(message=f"/connections/{connection_name}.connection"))
+                click.echo(FeedbackManager.success(message="✓ .connection created!"))
+                connection_files = get_connection_files()
+            else:
+                click.echo(
+                    FeedbackManager.info(message=f"→ Run `tb connection create {datasource_type.lower()}` to add one.")
+                )
+                return
+
+        if not connection_file:
+            connection_file = connection_files[0]
+            connection_path = Path(connection_file)
+            connection = connection_path.stem
+
+    ds_content = """SCHEMA >
+`timestamp` DateTime `json:$.timestamp`,
+`session_id` String `json:$.session_id`
+"""
+
+    if datasource_type == "Local file":
+        click.echo(FeedbackManager.gray(message="\n» Creating .datasource file..."))
+        if not file:
+            file = click.prompt(FeedbackManager.highlight(message="? Path"))
+            if file.startswith("~"):
+                file = os.path.expanduser(file)
+
+        folder_path = project.path
+        path = folder_path / file
+        if not path.exists():
+            path = Path(file)
+
+        data_format = path.suffix.lstrip(".")
+        ds_content = await analyze_file(str(path), client, format=data_format)
+        default_name = normalize_datasource_name(path.stem)
+        name = name or click.prompt(
+            FeedbackManager.highlight(message=f"? Data source name [{default_name}]"),
+            default=default_name,
+            show_default=False,
+        )
+
+    if datasource_type == "Remote URL":
+        click.echo(FeedbackManager.gray(message="\n» Creating .datasource file..."))
+        if not url:
+            url = click.prompt(FeedbackManager.highlight(message="? URL"))
+        format = url.split(".")[-1]
+        ds_content = await analyze_file(url, client, format)
+        default_name = normalize_datasource_name(Path(url).stem)
+        name = name or click.prompt(
+            FeedbackManager.highlight(message=f"? Data source name [{default_name}]"),
+            default=default_name,
+            show_default=False,
+        )
+
+    if datasource_type == "Blank":
+        click.echo(FeedbackManager.gray(message="\n» Creating .datasource file..."))
+
+    if datasource_type not in ("Remote URL", "Local file"):
+        default_name = f"ds_{generate_short_id()}"
+        name = name or click.prompt(
+            FeedbackManager.highlight(message=f"? Data source name [{default_name}]"),
+            default=default_name,
+            show_default=False,
+        )
+
+    if datasource_type == "Kafka":
+        connections = await client.connections("kafka")
+        connection_id = next((c["id"] for c in connections if c["name"] == connection), connection)
+        try:
+            topics = await client.kafka_list_topics(connection_id) if connection_id else []
+        except Exception:
+            topics = []
+        topic = topics[0] if len(topics) > 0 else "topic_0"
+        group_id = generate_kafka_group_id(topic)
+        ds_content += f"""
+KAFKA_CONNECTION_NAME {connection}
+KAFKA_TOPIC {{{{ tb_secret("KAFKA_TOPIC", "{topic}") }}}}
+KAFKA_GROUP_ID {{{{ tb_secret("KAFKA_GROUP_ID", "{group_id}") }}}}
+"""
+
+    if datasource_type == "S3":
+        if not connection:
+            connections = await client.connections("s3")
+            connection = next((c["name"] for c in connections if c["name"] == connection), connection)
+        ds_content += f"""
+IMPORT_CONNECTION_NAME "{connection}"
+IMPORT_BUCKET_URI "s3://my-bucket/*.csv"
+IMPORT_SCHEDULE "@auto"
+"""
+
+    if datasource_type == "GCS":
+        if not connection:
+            connections = await client.connections("gcs")
+            connection = next((c["name"] for c in connections if c["name"] == connection), connection)
+        ds_content += f"""
+IMPORT_CONNECTION_NAME "{connection}"
+IMPORT_BUCKET_URI "gs://my-bucket/*.csv"
+IMPORT_SCHEDULE "@auto"
+"""
+
+    click.echo(FeedbackManager.info(message=f"/datasources/{name}.datasource"))
+    datasources_path = project.path / "datasources"
+    if not datasources_path.exists():
+        datasources_path.mkdir()
+    ds_file = datasources_path / f"{name}.datasource"
+    if not ds_file.exists():
+        ds_file.touch()
+    ds_file.write_text(ds_content)
+    click.echo(FeedbackManager.success(message="✓ .datasource created!"))
+
+
+def generate_short_id():
+    return str(uuid.uuid4())[:4]
+
+
+def generate_kafka_group_id(topic: str):
+    return f"{topic}_{int(datetime.timestamp(datetime.now()))}"

tinybird/tb/modules/deployment.py

@@ -521,6 +521,14 @@ def create_deployment(
     TINYBIRD_API_URL = f"{client.host}/v1/deploy"
     TINYBIRD_API_KEY = client.token
 
+    if project.has_deeper_level():
+        click.echo(
+            FeedbackManager.warning(
+                message="\nDefault max_depth is 2, but project has deeper level. "
+                "You can change max_depth by running `tb --max-depth <depth> <cmd>`"
+            )
+        )
+
     files = [
         ("context://", ("cli-version", "1.0.0", "text/plain")),
     ]

tinybird/tb/modules/project.py

@@ -30,6 +30,17 @@ class Project:
             project_files.extend(glob.glob(f"{self.path}{'/*' * level}/*.{extension}", recursive=True))
         return project_files
 
+    def has_deeper_level(self) -> bool:
+        """Check if there are folders with depth greater than max_depth in project path.
+
+        Returns:
+            bool: True if there are folders deeper than max_depth, False otherwise
+        """
+        for obj in glob.glob(f"{self.path}{'/*' * (self.max_depth - 1)}/*", recursive=False):
+            if Path(obj).is_dir():
+                return True
+        return False
+
     def get_project_files(self) -> List[str]:
         project_files: List[str] = []
         for extension in self.extensions:
@@ -66,6 +77,15 @@ class Project:
     def get_connection_files(self) -> List[str]:
         return self.get_files("connection")
 
+    def get_kafka_connection_files(self) -> List[str]:
+        return [f for f in self.get_connection_files() if self.is_kafka_connection(Path(f).read_text())]
+
+    def get_s3_connection_files(self) -> List[str]:
+        return [f for f in self.get_connection_files() if self.is_s3_connection(Path(f).read_text())]
+
+    def get_gcs_connection_files(self) -> List[str]:
+        return [f for f in self.get_connection_files() if self.is_gcs_connection(Path(f).read_text())]
+
     def get_pipe_datafile(self, filename: str) -> Optional[Datafile]:
         try:
             return parse_pipe(filename).datafile
@@ -96,3 +116,27 @@ class Project:
     @staticmethod
     def is_endpoint(content: str) -> bool:
         return re.search(r"TYPE endpoint", content, re.IGNORECASE) is not None
+
+    @staticmethod
+    def is_kafka_connection(content: str) -> bool:
+        return re.search(r"TYPE kafka", content, re.IGNORECASE) is not None
+
+    @staticmethod
+    def is_s3_connection(content: str) -> bool:
+        return re.search(r"TYPE s3", content, re.IGNORECASE) is not None
+
+    @staticmethod
+    def is_gcs_connection(content: str) -> bool:
+        return re.search(r"TYPE gcs", content, re.IGNORECASE) is not None
+
+    @staticmethod
+    def is_kafka_datasource(content: str) -> bool:
+        return re.search(r"KAFKA_CONNECTION_NAME", content, re.IGNORECASE) is not None
+
+    @staticmethod
+    def is_s3_datasource(content: str) -> bool:
+        return re.search(r"IMPORT_CONNECTION_NAME", content, re.IGNORECASE) is not None
+
+    @staticmethod
+    def is_gcs_datasource(content: str) -> bool:
+        return re.search(r"IMPORT_CONNECTION_NAME", content, re.IGNORECASE) is not None

{tinybird-0.0.1.dev188.dist-info → tinybird-0.0.1.dev190.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev188
+Version: 0.0.1.dev190
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev188.dist-info → tinybird-0.0.1.dev190.dist-info}/RECORD

@@ -12,21 +12,21 @@ tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
 tinybird/tornado_template.py,sha256=jjNVDMnkYFWXflmT8KU_Ssbo5vR8KQq3EJMk5vYgXRw,41959
 tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj0,3890
 tinybird/ch_utils/engine.py,sha256=X4tE9OrfaUy6kO9cqVEzyI9cDcmOF3IAssRRzsTsfEQ,40781
-tinybird/tb/__cli__.py,sha256=0dMMoeV7WGBd5xykwKr1XNZzsV203grOWkx1PgeDVgo,247
+tinybird/tb/__cli__.py,sha256=Uk4h64toVqAfnRJ4rIo4K3vfl_FunHW228-sIHLBDt0,247
 tinybird/tb/check_pypi.py,sha256=rW4QmDRbtgKdUUwJCnBkVjmTjZSZGN-XgZhx7vMkC0w,1009
 tinybird/tb/cli.py,sha256=u3eGOhX0MHkuT6tiwaZ0_3twqLmqKXDAOxF7yV_Nn9Q,1075
-tinybird/tb/client.py,sha256=59GH0IoYSV_KUG0eEbDDYHSWH4OlkVnSRFXE3mYAM0s,56571
+tinybird/tb/client.py,sha256=CO-dQw8h28X6T6IO-Z79yPBKaJQT1Rwya5b6gexvw58,56491
 tinybird/tb/config.py,sha256=jT9xndpeCY_g0HdB5qE2EquC0TFRRnkPnQFWZWd04jo,3998
-tinybird/tb/modules/build.py,sha256=rRL5XKBdadMc9uVDEUt0GXm0h09Y6XXw199rdmRI1qo,19127
+tinybird/tb/modules/build.py,sha256=KvF0s8hGgY_rZs7jSqYiauCk3MAlCmW_gQtnsJDJWBk,19411
 tinybird/tb/modules/cicd.py,sha256=Njb6eZOHHbUkoJJx6KoixO9PsfA_T-3Ybkya9-50Ca8,7328
 tinybird/tb/modules/cli.py,sha256=dXZs-MuqYPvxStVj7aLg36LwXtEB8NzTobDmHV9nzZI,15508
-tinybird/tb/modules/common.py,sha256=DYCjpj0iBaCDZ8BJ0MNG_6m6NyFMCrpQShIajHKLIfM,83373
+tinybird/tb/modules/common.py,sha256=6SYrDH-GhWm8N8S2GmHdIbvf1mUYIzWYlz0WcjDIhGw,83947
 tinybird/tb/modules/config.py,sha256=ziqW_t_mRVvWOd85VoB4vKyvgMkEfpXDf9H4v38p2xc,11422
-tinybird/tb/modules/connection.py,sha256=7oOR7x4PhBcm1ETFFCH2YJ_3oeGXjAbmx1cnZX9_L70,9014
+tinybird/tb/modules/connection.py,sha256=z1xWP2gtjKEbjc4ZF1aD7QUgl8V--wf2IRNy-4sRFm8,9779
 tinybird/tb/modules/copy.py,sha256=2Mm4FWKehOG7CoOhiF1m9UZJgJn0W1_cMolqju8ONYg,5805
-tinybird/tb/modules/create.py,sha256=sfIOcN3tujt7O1r9RNWqhhI-gQTDnO6zEgMwZHH2D8s,20201
-tinybird/tb/modules/datasource.py,sha256=0_6Cn07p5GoNBBGdu88pSeLvTWojln1-k23FsS8jTDs,17801
-tinybird/tb/modules/deployment.py,sha256=t6DDLJ1YdY3SJiTPbEG7CRblSLkbuqwzauQ9y65FWtY,27147
+tinybird/tb/modules/create.py,sha256=2uW-4t7c7e4xkZ-GpK_8XaA-nuXwklq7rTks4k6qrtI,20917
+tinybird/tb/modules/datasource.py,sha256=gycHinGTDxZMDqwbIaRJkR2SUxdhbr1prCFQHkssgVs,35124
+tinybird/tb/modules/deployment.py,sha256=OGJdriqywhCtsG7rKLFtVSLjoEbbva1Nb29-jQBk3wM,27432
 tinybird/tb/modules/deprecations.py,sha256=rrszC1f_JJeJ8mUxGoCxckQTJFBCR8wREf4XXXN-PRc,4507
 tinybird/tb/modules/dev_server.py,sha256=57FCKuWpErwYUYgHspYDkLWEm9F4pbvVOtMrFXX1fVU,10129
 tinybird/tb/modules/endpoint.py,sha256=XySDt3pk66vxOZ0egUfz4bY8bEk3BjOXkv-L0OIJ3sc,12083
@@ -45,7 +45,7 @@ tinybird/tb/modules/materialization.py,sha256=QJX5kCPhhm6IXBO1JsalVfbQdypCe_eOUD
 tinybird/tb/modules/mock.py,sha256=IyHweMUM6bUH8IhyiX2tTMpdVpTFUeAJ41lZ5P42-HQ,5303
 tinybird/tb/modules/open.py,sha256=OuctINN77oexpSjth9uoIZPCelKO4Li-yyVxeSnk1io,1371
 tinybird/tb/modules/pipe.py,sha256=AQKEDagO6e3psPVjJkS_MDbn8aK-apAiLp26k7jgAV0,2432
-tinybird/tb/modules/project.py,sha256=_UWO79zhh2H0UA9F0wpKXYaylFPdjhGNmNAjIOSBvT8,3425
+tinybird/tb/modules/project.py,sha256=iIEaBQsdLXyzJ_45Paf1jwbSrtTwWv131VCuPsTQttA,5215
 tinybird/tb/modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb/modules/secret.py,sha256=WsqzxxLh9W_jkuHL2JofMXdIJy0lT5WEI-7bQSIDgAc,2921
 tinybird/tb/modules/shell.py,sha256=Zd_4Ak_5tKVX-cw6B4ag36xZeEGHeh-jZpAsIXkoMoE,14116
@@ -80,8 +80,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
-tinybird-0.0.1.dev188.dist-info/METADATA,sha256=psVJzC4sJRKjG8Re5Jjo0GNwSrW5JdCAxGrcrzcWgoY,1608
-tinybird-0.0.1.dev188.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-tinybird-0.0.1.dev188.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-0.0.1.dev188.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
-tinybird-0.0.1.dev188.dist-info/RECORD,,
+tinybird-0.0.1.dev190.dist-info/METADATA,sha256=MDRLrgiaBGisoaGthlSBKhjMSvdw8XSWPF5VEUsjrvM,1608
+tinybird-0.0.1.dev190.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+tinybird-0.0.1.dev190.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-0.0.1.dev190.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
+tinybird-0.0.1.dev190.dist-info/RECORD,,