tinybird 0.0.1.dev188__tar.gz → 0.0.1.dev190__tar.gz

{tinybird-0.0.1.dev188 → tinybird-0.0.1.dev190}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev188
+Version: 0.0.1.dev190
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev188 → tinybird-0.0.1.dev190}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev188'
-__revision__ = 'f13c1f2'
+__version__ = '0.0.1.dev190'
+__revision__ = '8a00bcf'

{tinybird-0.0.1.dev188 → tinybird-0.0.1.dev190}/tinybird/tb/client.py

@@ -1024,11 +1024,7 @@ class TinyB:
         )
 
     async def kafka_list_topics(self, connection_id: str, timeout=5):
-        resp = await self._req(
-            f"/v0/connectors/{connection_id}/preview?preview_activity=false",
-            connect_timeout=timeout,
-            request_timeout=timeout,
-        )
+        resp = await self._req(f"/v0/connectors/{connection_id}/preview?preview_activity=false", timeout=timeout)
         return [x["topic"] for x in resp["preview"]]
 
     async def get_gcp_service_account_details(self) -> Dict[str, Any]:

{tinybird-0.0.1.dev188 → tinybird-0.0.1.dev190}/tinybird/tb/modules/build.py

@@ -38,6 +38,15 @@ def build(ctx: click.Context, watch: bool) -> None:
     """
     project: Project = ctx.ensure_object(dict)["project"]
     tb_client: TinyB = ctx.ensure_object(dict)["client"]
+
+    if project.has_deeper_level():
+        click.echo(
+            FeedbackManager.warning(
+                message="Default max_depth is 2, but project has deeper level. "
+                "You can change max_depth by running `tb --max-depth <depth> <cmd>`"
+            )
+        )
+
     click.echo(FeedbackManager.highlight_building_project())
     process(project=project, tb_client=tb_client, watch=False)
     if watch:

{tinybird-0.0.1.dev188 → tinybird-0.0.1.dev190}/tinybird/tb/modules/common.py

@@ -424,6 +424,14 @@ async def _analyze(filename: str, client: TinyB, format: str, connector: Optiona
     return meta, data
 
 
+async def analyze_file(filename: str, client: TinyB, format: str):
+    meta, data = await _analyze(filename, client, format)
+    schema = meta["analysis"]["schema"]
+    schema = schema.replace(", ", ",\n    ")
+    content = f"""DESCRIPTION >\n    Generated from {filename}\n\nSCHEMA >\n    {schema}"""
+    return content
+
+
 async def _generate_datafile(
     filename: str,
     client: TinyB,
@@ -971,7 +979,7 @@ async def push_data(
     cb.prev_done = 0  # type: ignore[attr-defined]
 
     if not silent:
-        click.echo(FeedbackManager.gray(message=f"\nImporting data to {datasource_name}..."))
+        click.echo(FeedbackManager.highlight(message=f"\n» Appending data to {datasource_name}..."))
 
     if isinstance(url, list):
         urls = url
@@ -1882,15 +1890,19 @@ def get_gcs_connection_name(project_folder) -> str:
     return get_connection_name(project_folder=project_folder, connection_type="GCS")
 
 
-def get_connection_name(project_folder: str, connection_type: str) -> str:
-    connection_name = None
+def get_kafka_connection_name(project_folder: str, connection_name: Optional[str] = None) -> str:
+    return get_connection_name(project_folder=project_folder, connection_type="KAFKA", connection_name=connection_name)
+
+
+def get_connection_name(project_folder: str, connection_type: str, connection_name: Optional[str] = None) -> str:
     valid_pattern = r"^[a-zA-Z][a-zA-Z0-9_]*$"
 
     while not connection_name:
+        short_id = str(uuid.uuid4())[:4]
         connection_name = click.prompt(
-            f"🔗 Enter a name for your new Tinybird {connection_type} connection (use alphanumeric characters, and underscores)",
-            prompt_suffix="\n> ",
+            "Enter a name (only alphanumeric characters and underscores)",
             show_default=True,
+            default=f"{connection_type.lower()}_{short_id}",
         )
         assert isinstance(connection_name, str)
 

{tinybird-0.0.1.dev188 → tinybird-0.0.1.dev190}/tinybird/tb/modules/connection.py

@@ -18,6 +18,7 @@ from tinybird.tb.modules.common import (
     echo_safe_humanfriendly_tables_format_smart_table,
     get_gcs_connection_name,
     get_gcs_svc_account_creds,
+    get_kafka_connection_name,
     get_s3_connection_name,
     production_aws_iamrole_only,
     run_aws_iamrole_connection_flow,
@@ -26,6 +27,7 @@ from tinybird.tb.modules.common import (
 from tinybird.tb.modules.create import (
     generate_aws_iamrole_connection_file_with_secret,
     generate_gcs_connection_file_with_secrets,
+    generate_kafka_connection_with_secrets,
 )
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.project import Project
@@ -275,3 +277,21 @@ async def connection_create_gcs(ctx: Context) -> None:
             connection_path=connection_path,
         )
     )
+
+
+@connection_create.command(name="kafka", short_help="Creates a Kafka connection.")
+@click.option("--name", help="The name of the connection")
+@click.pass_context
+@coro
+async def connection_create_kafka(ctx: Context, name: Optional[str] = None) -> None:
+    """
+    Creates a Kafka connection.
+
+    \b
+    $ tb connection create kafka
+    """
+    click.echo(FeedbackManager.highlight(message="» Creating Kafka connection..."))
+    project: Project = ctx.ensure_object(dict)["project"]
+    name = get_kafka_connection_name(project.folder, name)
+    await generate_kafka_connection_with_secrets(name=name, folder=project.folder)
+    click.echo(FeedbackManager.success(message="✓ Done!"))

{tinybird-0.0.1.dev188 → tinybird-0.0.1.dev190}/tinybird/tb/modules/create.py

@@ -409,10 +409,14 @@ def generate_connection_file(name: str, content: str, folder: str, skip_feedback
 
 
 async def generate_aws_iamrole_connection_file_with_secret(
-    name: str, service: str, role_arn_secret_name: str, region: str, folder: str
+    name: str, service: str, role_arn_secret_name: str, region: str, folder: str, with_default_secret: bool = False
 ) -> Path:
+    if with_default_secret:
+        default_secret = ', "arn:aws:iam::123456789012:role/my-role"'
+    else:
+        default_secret = ""
     content = f"""TYPE {service}
-S3_ARN {{{{ tb_secret("{role_arn_secret_name}") }}}}
+S3_ARN {{{{ tb_secret("{role_arn_secret_name}"{default_secret}) }}}}
 S3_REGION {region}
 """
     file_path = generate_connection_file(name, content, folder, skip_feedback=True)
@@ -483,6 +487,7 @@ async def create_resources_from_data(
     data: str,
     project: Project,
     config: Dict[str, Any],
+    skip_pipes: bool = False,
 ) -> List[Path]:
     local_client = await get_tinybird_local_client(config)
     folder_path = project.path
@@ -495,7 +500,7 @@ async def create_resources_from_data(
     result.append(ds_file)
     name = ds_file.stem
     no_pipes = len(project.get_pipe_files()) == 0
-    if no_pipes:
+    if not skip_pipes and no_pipes:
         pipe_file = generate_pipe_file(
             f"{name}_endpoint",
             f"""
@@ -510,7 +515,9 @@ TYPE ENDPOINT
     return result
 
 
-async def create_resources_from_url(url: str, project: Project, config: Dict[str, Any]) -> List[Path]:
+async def create_resources_from_url(
+    url: str, project: Project, config: Dict[str, Any], skip_pipes: bool = False
+) -> List[Path]:
     result: List[Path] = []
     local_client = await get_tinybird_local_client(config)
     format = url.split(".")[-1]
@@ -518,7 +525,7 @@ async def create_resources_from_url(url: str, project: Project, config: Dict[str
     result.append(ds_file)
     name = ds_file.stem
     no_pipes = len(project.get_pipe_files()) == 0
-    if no_pipes:
+    if not skip_pipes and no_pipes:
         pipe_file = generate_pipe_file(
             f"{name}_endpoint",
             f"""
@@ -531,3 +538,14 @@ TYPE ENDPOINT
         )
         result.append(pipe_file)
     return result
+
+
+async def generate_kafka_connection_with_secrets(name: str, folder: str) -> Path:
+    content = """TYPE kafka
+KAFKA_BOOTSTRAP_SERVERS {{ tb_secret("KAFKA_SERVERS", "localhost:9092") }}
+KAFKA_SECURITY_PROTOCOL SASL_SSL
+KAFKA_SASL_MECHANISM PLAIN
+KAFKA_KEY {{ tb_secret("KAFKA_USERNAME", "") }}
+KAFKA_SECRET {{ tb_secret("KAFKA_PASSWORD", "") }}
+"""
+    return generate_connection_file(name, content, folder, skip_feedback=True)