tinybird 0.0.1.dev111__tar.gz → 0.0.1.dev113__tar.gz

{tinybird-0.0.1.dev111 → tinybird-0.0.1.dev113}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev111
+Version: 0.0.1.dev113
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev111 → tinybird-0.0.1.dev113}/tinybird/prompts.py

@@ -954,3 +954,35 @@ FORWARD_QUERY >
     </forward_query_example>
 </deployment_instruction>
 """
+
+
+def readme_prompt(readme: str, host: str, token: str, resources_xml: str) -> str:
+    return f"""
+You are an expert in SQL and Tinybird. Follow these instructions to generate a new README.md file for a tinybird project:
+Current README.md file:
+<current_resources_xml>
+{resources_xml}
+</current_resources_xml>
+<readme>{readme}</readme>
+<readme_instructions>
+- If it is not present in the current readme, generate a new ## Tinybird section with the following content:
+    - ### Overview section:
+        - Explaining the purpose of the project.
+    - ### Data sources section: 
+        - Explaining the purpose of each datasource.
+        - Add a snippet of how to ingest data into each datasource like the following (where the payload example matches the datasource schema respecting non-nullable types):
+        curl -X POST "{host}/v0/events?name=events" \
+    -H "Authorization: Bearer {token}" \
+    -d '{{"date":"2025-01-31","id":"123","user_id":"abc"}}'
+    - ### Endpoints section:
+        - Explaining the purpose of each endpoint.
+        - Add a snippet of how to use each endpoint like the following:
+        curl -X GET "{host}/v0/pipes/pipe_name.json?token={token}"
+        - DateTime parameters must be formatted as YYYY-MM-DD HH:MM:SS, or else will fail.
+- Do not include any other extra info related to Tinybird just maintain the existing one in the <readme> tag.
+- Make sure the API host is correct, {host}.
+- It is mandatory to return a <resource> tag with type "readme" in the response.
+- The response must follow the following format:
+<readme>[readme content here]</readme>
+</readme_instructions>
+"""

{tinybird-0.0.1.dev111 → tinybird-0.0.1.dev113}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev111'
-__revision__ = '621864a'
+__version__ = '0.0.1.dev113'
+__revision__ = '49946d6'

{tinybird-0.0.1.dev111 → tinybird-0.0.1.dev113}/tinybird/tb/modules/cicd.py

@@ -120,7 +120,9 @@ tinybird_ci_workflow:
   services:
     - name: tinybirdco/tinybird-local:beta
       alias: tinybird-local
+"""
 
+GITLAB_CD_YML = """
 tinybird_cd_workflow:
   image: ubuntu:latest
   stage: deploy
@@ -217,6 +219,11 @@ class GitLabCICDGenerator(CICDGeneratorBase):
             file_name="tinybird-ci.yml",
             dir_path=".gitlab/tinybird",
         ),
+        CICDFile(
+            template=GITLAB_CD_YML,
+            file_name="tinybird-cd.yml",
+            dir_path=".gitlab/tinybird",
+        ),
     ]
 
 

{tinybird-0.0.1.dev111 → tinybird-0.0.1.dev113}/tinybird/tb/modules/cli.py

@@ -6,6 +6,7 @@
 import json
 import logging
 import os
+import shutil
 import sys
 from os import getcwd
 from pathlib import Path
@@ -48,7 +49,13 @@ DEFAULT_PATTERNS: List[Tuple[str, Union[str, Callable[[str], str]]]] = [
 VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
 
 
-@click.group(cls=CatchAuthExceptions, context_settings={"help_option_names": ["-h", "--help"]})
+@click.group(
+    cls=CatchAuthExceptions,
+    context_settings={
+        "help_option_names": ["-h", "--help"],
+        "max_content_width": shutil.get_terminal_size().columns - 10,
+    },
+)
 @click.option(
     "--debug/--no-debug",
     default=False,

{tinybird-0.0.1.dev111 → tinybird-0.0.1.dev113}/tinybird/tb/modules/common.py

@@ -1751,90 +1751,69 @@ async def remove_release(
             click.echo(FeedbackManager.info_no_release_deleted())
 
 
-async def validate_aws_iamrole_integration(
+async def run_aws_iamrole_connection_flow(
     client: TinyB,
     service: str,
-    role_arn: Optional[str],
-    region: Optional[str],
-    policy: str = "write",
-    no_validate: Optional[bool] = False,
+    policy: str = "read",
 ):
-    if no_validate is False:
-        access_policy, trust_policy, external_id = await get_aws_iamrole_policies(
-            client, service=service, policy=policy
-        )
+    if service == DataConnectorType.AMAZON_DYNAMODB:
+        raise NotImplementedError("DynamoDB is not supported yet")
 
-        if not role_arn:
-            if not click.confirm(
-                FeedbackManager.prompt_s3_iamrole_connection_login_aws(),
-                show_default=False,
-                prompt_suffix="Press y to continue:",
-            ):
-                sys.exit(1)
+    resource_type = "table" if service == DataConnectorType.AMAZON_DYNAMODB else "bucket"
+    resource_name = click.prompt(f"Enter the name of the {resource_type}")
+    validate_string_connector_param(resource_type.capitalize(), resource_name)
 
-            access_policy_copied = True
-            try:
-                pyperclip.copy(access_policy)
-            except Exception:
-                access_policy_copied = False
+    resource_region = click.prompt(f"Enter the region where the {resource_type} is located")
+    validate_string_connector_param("Region", resource_region)
 
-            replacements_dict = {
-                "<bucket>": "<bucket> with your bucket name",
-                "<table_name>": "<table_name> with your DynamoDB table name",
-            }
+    access_policy, trust_policy, external_id = await get_aws_iamrole_policies(client, service=service, policy=policy)
+    access_policy = access_policy.replace("<bucket>", resource_name)
 
-            replacements = [
-                replacements_dict.get(replacement, "")
-                for replacement in replacements_dict.keys()
-                if replacement in access_policy
-            ]
+    if not click.confirm(
+        FeedbackManager.prompt_s3_iamrole_connection_login_aws(),
+        show_default=False,
+        prompt_suffix="Press y to continue:",
+    ):
+        sys.exit(1)
 
-            if not click.confirm(
-                (
-                    FeedbackManager.prompt_s3_iamrole_connection_policy(
-                        access_policy=access_policy, replacements=", ".join(replacements)
-                    )
-                    if access_policy_copied
-                    else FeedbackManager.prompt_s3_iamrole_connection_policy_not_copied(access_policy=access_policy)
-                ),
-                show_default=False,
-                prompt_suffix="Press y to continue:",
-            ):
-                sys.exit(1)
-
-            trust_policy_copied = True
-            try:
-                pyperclip.copy(trust_policy)
-            except Exception:
-                trust_policy_copied = False
-
-            if not click.confirm(
-                (
-                    FeedbackManager.prompt_s3_iamrole_connection_role(trust_policy=trust_policy)
-                    if trust_policy_copied
-                    else FeedbackManager.prompt_s3_iamrole_connection_role_not_copied(trust_policy=trust_policy)
-                ),
-                show_default=False,
-                prompt_suffix="Press y to continue:",
-            ):
-                sys.exit(1)
-    else:
-        try:
-            trust_policy = await client.get_trust_policy(service)
-            external_id = trust_policy["Statement"][0]["Condition"]["StringEquals"]["sts:ExternalId"]
-        except Exception:
-            external_id = ""
+    access_policy_copied = True
+    try:
+        pyperclip.copy(access_policy)
+    except Exception:
+        access_policy_copied = False
 
-    if not role_arn:
-        role_arn = click.prompt("Enter the ARN of the role you just created")
-        validate_string_connector_param("Role ARN", role_arn)
+    if not click.confirm(
+        (
+            FeedbackManager.prompt_s3_iamrole_connection_policy(access_policy=access_policy)
+            if access_policy_copied
+            else FeedbackManager.prompt_s3_iamrole_connection_policy_not_copied(access_policy=access_policy)
+        ),
+        show_default=False,
+        prompt_suffix="Press y to continue:",
+    ):
+        sys.exit(1)
 
-    if not region:
-        region_resource = "table" if service == DataConnectorType.AMAZON_DYNAMODB else "bucket"
-        region = click.prompt(f"Enter the region where the {region_resource} is located")
-        validate_string_connector_param("Region", region)
+    trust_policy_copied = True
+    try:
+        pyperclip.copy(trust_policy)
+    except Exception:
+        trust_policy_copied = False
 
-    return role_arn, region, external_id
+    if not click.confirm(
+        (
+            FeedbackManager.prompt_s3_iamrole_connection_role(trust_policy=trust_policy)
+            if trust_policy_copied
+            else FeedbackManager.prompt_s3_iamrole_connection_role_not_copied(trust_policy=trust_policy)
+        ),
+        show_default=False,
+        prompt_suffix="Press y to continue:",
+    ):
+        sys.exit(1)
+
+    role_arn = click.prompt("Enter the ARN of the role you just created")
+    validate_string_connector_param("Role ARN", role_arn)
+
+    return role_arn, resource_region, external_id
 
 
 async def get_aws_iamrole_policies(client: TinyB, service: str, policy: str = "write"):
@@ -1867,20 +1846,15 @@ async def get_aws_iamrole_policies(client: TinyB, service: str, policy: str = "w
     return json.dumps(access_policy, indent=4), json.dumps(trust_policy, indent=4), external_id
 
 
-async def validate_aws_iamrole_connection_name(
-    client: TinyB, connection_name: Optional[str], no_validate: Optional[bool] = False
-) -> str:
-    if connection_name and no_validate is False:
-        if await client.get_connector(connection_name, skip_bigquery=True) is not None:
-            raise CLIConnectionException(FeedbackManager.info_connection_already_exists(name=connection_name))
-    else:
-        while not connection_name:
-            connection_name = click.prompt("Enter the name for this connection", default=None, show_default=False)
-            assert isinstance(connection_name, str)
+async def validate_aws_iamrole_connection_name(client: TinyB) -> str:
+    connection_name = None
+    while not connection_name:
+        connection_name = click.prompt("Enter the name for this connection", default=None, show_default=False)
+        assert isinstance(connection_name, str)
 
-            if no_validate is False and await client.get_connector(connection_name) is not None:
-                click.echo(FeedbackManager.info_connection_already_exists(name=connection_name))
-                connection_name = None
+        if await client.get_connector(connection_name) is not None:
+            click.echo(FeedbackManager.info_connection_already_exists(name=connection_name))
+            connection_name = None
     assert isinstance(connection_name, str)
     return connection_name
 

{tinybird-0.0.1.dev111 → tinybird-0.0.1.dev113}/tinybird/tb/modules/connection.py

@@ -15,8 +15,12 @@ from tinybird.tb.modules.common import (
     _get_setting_value,
     coro,
     echo_safe_humanfriendly_tables_format_smart_table,
+    run_aws_iamrole_connection_flow,
+    validate_aws_iamrole_connection_name,
 )
+from tinybird.tb.modules.create import generate_aws_iamrole_connection_file
 from tinybird.tb.modules.feedback_manager import FeedbackManager
+from tinybird.tb.modules.project import Project
 
 DATA_CONNECTOR_SETTINGS: Dict[DataConnectorType, List[str]] = {
     DataConnectorType.KAFKA: [
@@ -123,3 +127,41 @@ async def connection_ls(ctx: Context, service: Optional[DataConnectorType] = Non
     column_names = [c.replace("kafka_", "") for c in columns]
     echo_safe_humanfriendly_tables_format_smart_table(table, column_names=column_names)
     click.echo("\n")
+
+
+@connection.group(name="create")
+@click.pass_context
+def connection_create(ctx: Context) -> None:
+    """Create a connection."""
+
+
+@connection_create.command(name="s3", short_help="Creates a AWS S3 connection.")
+@click.pass_context
+@coro
+async def connection_create_s3(ctx: Context) -> None:
+    """
+    Creates a AWS S3 connection.
+
+    \b
+    $ tb connection create s3
+    """
+    project: Project = ctx.ensure_object(dict)["project"]
+    obj: Dict[str, Any] = ctx.ensure_object(dict)
+    client: TinyB = obj["client"]
+    service = DataConnectorType.AMAZON_S3
+    connection_name = await validate_aws_iamrole_connection_name(client)
+    role_arn, region, external_id = await run_aws_iamrole_connection_flow(
+        client,
+        service=service,
+        policy="read",  # For now only read since we only support import from S3
+    )
+
+    await generate_aws_iamrole_connection_file(
+        name=connection_name, service=service, role_arn=role_arn, region=region, folder=project.folder
+    )
+    if external_id:
+        click.echo(
+            FeedbackManager.success_s3_iam_connection_created(
+                connection_name=connection_name, external_id=external_id, role_arn=role_arn
+            )
+        )

{tinybird-0.0.1.dev111 → tinybird-0.0.1.dev113}/tinybird/tb/modules/create.py

@@ -6,7 +6,7 @@ from typing import Optional, Tuple
 import click
 
 from tinybird.client import TinyB
-from tinybird.prompts import create_prompt, mock_prompt, rules_prompt
+from tinybird.prompts import create_prompt, mock_prompt, readme_prompt, rules_prompt
 from tinybird.tb.modules.cicd import init_cicd
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import _generate_datafile, coro, generate_datafile
@@ -88,12 +88,32 @@ async def create(
             click.echo(FeedbackManager.success(message="✓ Scaffolding completed!\n"))
             created_something = True
         result = ""
-        if data or prompt:
+        if (data or prompt) and user_token:
             click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
             result, created_something = await create_resources(
                 local_client, tb_client, user_token, data, prompt, folder
             )
             click.echo(FeedbackManager.success(message="✓ Done!\n"))
+            if prompt:
+                readme_path = Path(root_folder) / "README.md"
+                if readme_path.exists():
+                    click.echo(FeedbackManager.highlight(message="\n» Updating README.md..."))
+                else:
+                    click.echo(FeedbackManager.highlight(message="\n» Creating README.md..."))
+                    readme_path.touch()
+                llm = LLM(user_token=str(user_token), host=tb_client.host)
+                readme_user_prompt = prompt or ""
+                all_resources_xml = get_resources_xml(project)
+                readme_response = llm.ask(
+                    system_prompt=readme_prompt(
+                        readme_path.read_text(), tb_client.host, tb_client.token, all_resources_xml
+                    ),
+                    prompt=readme_user_prompt,
+                )
+                readme_result = extract_xml(readme_response, "readme")
+                readme_path.write_text(readme_result)
+                click.echo(FeedbackManager.success(message="✓ Done!\n"))
+                created_something = True
 
         if not already_has_cicd(root_folder):
             click.echo(FeedbackManager.highlight(message="\n» Creating CI/CD files for GitHub and GitLab..."))
@@ -353,6 +373,17 @@ def generate_connection_file(name: str, content: str, folder: str) -> Path:
     return f.relative_to(folder)
 
 
+async def generate_aws_iamrole_connection_file(
+    name: str, service: str, role_arn: str, region: str, folder: str
+) -> None:
+    content = f"""TYPE {service}
+
+S3_ARN {role_arn}
+S3_REGION {region}
+"""
+    generate_connection_file(name, content, folder)
+
+
 def create_rules(folder: str, source: str, agent: str):
     if agent == "cursor":
         extension = ".cursorrules"
@@ -379,3 +410,30 @@ def get_context() -> str:
 def save_context(prompt: str, feedback: str):
     context_file = get_context_file()
     context_file.write_text(f"- {prompt}\n{feedback}")
+
+
+def get_resources_xml(project: Project) -> str:
+    folder = project.folder
+    datasource_paths = [
+        Path(folder) / "datasources" / f for f in os.listdir(Path(folder) / "datasources") if f.endswith(".datasource")
+    ]
+    pipes_paths = [
+        Path(folder) / "endpoints" / f for f in os.listdir(Path(folder) / "endpoints") if f.endswith(".pipe")
+    ]
+    resources_xml = "\n".join(
+        [
+            f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
+            for resource_type, resource_name, resource_content in [
+                ("datasource", ds.stem, ds.read_text()) for ds in datasource_paths
+            ]
+            + [
+                (
+                    "pipe",
+                    pipe.stem,
+                    pipe.read_text(),
+                )
+                for pipe in pipes_paths
+            ]
+        ]
+    )
+    return resources_xml

{tinybird-0.0.1.dev111 → tinybird-0.0.1.dev113}/tinybird/tb/modules/datafile/fixture.py

@@ -5,7 +5,9 @@ from tinybird.tb.modules.common import format_data_to_ndjson
 
 
 def get_fixture_dir(folder: str) -> Path:
-    return Path(folder) / "fixtures"
+    fixture_dir = Path(folder) / "fixtures"
+    fixture_dir.mkdir(exist_ok=True)
+    return fixture_dir
 
 
 def persist_fixture_sql(fixture_name: str, sql: str, folder: str) -> Path:

{tinybird-0.0.1.dev111 → tinybird-0.0.1.dev113}/tinybird/tb/modules/feedback_manager.py

@@ -498,7 +498,7 @@ Ready? """
 
     prompt_s3_iamrole_connection_login_aws = prompt_message("""[1] Log into your AWS Console\n\n""")
     prompt_s3_iamrole_connection_policy = prompt_message(
-        """\n[2] Go to IAM > Policies. Create a new policy with the following permissions. Please, replace {replacements}:\n\n{access_policy}\n\n(The policy has been copied to your clipboard)\n\n"""
+        """\n[2] Go to IAM > Policies. Create a new policy with the following permissions:\n\n{access_policy}\n\n(The policy has been copied to your clipboard)\n\n"""
     )
     prompt_s3_iamrole_connection_policy_not_copied = prompt_message(
         """\n[2] Go to IAM > Policies. Create a new policy with the following permissions. Please, copy this policy and replace <bucket> with your bucket name:\n\n{access_policy}\n\n"""

{tinybird-0.0.1.dev111 → tinybird-0.0.1.dev113}/tinybird/tb/modules/infra.py

@@ -371,7 +371,7 @@ async def infra_init(
 
     # Create infra directory if it doesn't exist
     infra_dir = Path(f"infra/{cloud_provider}")
-    infra_dir.mkdir(exist_ok=True)
+    infra_dir.mkdir(parents=True, exist_ok=True)
     yaml_path = infra_dir / "k8s.yaml"
     tf_path = infra_dir / "main.tf"
     config_path = infra_dir / "config.json"

{tinybird-0.0.1.dev111 → tinybird-0.0.1.dev113}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev111
+Version: 0.0.1.dev113
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird