tinybird 3.1.0.dev0__tar.gz → 3.2.0__tar.gz

{tinybird-3.1.0.dev0 → tinybird-3.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 3.1.0.dev0
+Version: 3.2.0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -57,6 +57,11 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+3.0.2
+*******
+
+- `Changed` `IMPORT_FORMAT` is now optional for S3/GCS data sources and will be automatically inferred from the file extension if not provided
+
 3.0.1
 *******
 

{tinybird-3.1.0.dev0 → tinybird-3.2.0}/tinybird/datafile/common.py

@@ -270,7 +270,7 @@ REQUIRED_BLOB_STORAGE_PARAMS = {
     "import_schedule",
     "import_bucket_uri",
 }
-BLOB_STORAGE_PARAMS = REQUIRED_BLOB_STORAGE_PARAMS.union({"import_from_timestamp"})
+BLOB_STORAGE_PARAMS = REQUIRED_BLOB_STORAGE_PARAMS.union({"import_from_timestamp", "import_format"})
 VALID_BLOB_STORAGE_CRON_VALUES = {
     "@once",
     "@on-demand",
@@ -2017,6 +2017,7 @@ def parse(
             "import_schedule": assign_var("import_schedule"),
             "import_strategy": import_strategy_deprecated,  # Deprecated, always append
             "import_bucket_uri": assign_var("import_bucket_uri"),
+            "import_format": assign_var("import_format"),
             "import_from_timestamp": assign_var("import_from_timestamp"),
             "import_service": import_service_deprecated,  # Deprecated
             "import_external_datasource": import_external_datasource_deprecated,  # Deprecated, BQ and SFK
@@ -2196,6 +2197,7 @@ class ImportReplacements:
         ("import_connector", "connector", None),
         ("import_external_datasource", "external_data_source", None),
         ("import_bucket_uri", "bucket_uri", None),
+        ("import_format", "format", None),
         ("import_from_timestamp", "from_time", None),
         ("import_table_arn", "dynamodb_table_arn", None),
         ("import_export_bucket", "dynamodb_export_bucket", None),

{tinybird-3.1.0.dev0 → tinybird-3.2.0}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '3.1.0.dev0'
-__revision__ = 'a987e5a'
+__version__ = '3.2.0'
+__revision__ = '5429eb1'

{tinybird-3.1.0.dev0 → tinybird-3.2.0}/tinybird/tb/modules/connection_s3.py

@@ -236,6 +236,9 @@ def meta_to_schema(meta: list[dict[str, Any]], file_format: str) -> str:
     Returns:
         The schema string
     """
+    # Normalize format to lowercase for case-insensitive comparison
+    file_format = file_format.lower()
+
     if file_format == "csv":
         # CSV files don't use JSONPath
         return ",\n    ".join([f"`{col['name']}` {col['type']}" for col in meta])
@@ -249,6 +252,7 @@ def meta_to_s3_datasource_datafile(
     connection_name: str,
     bucket_uri: str,
     import_schedule: str = "@auto",
+    import_format: Optional[str] = None,
 ) -> str:
     """Generate S3 datasource datafile content from preview metadata.
 
@@ -257,11 +261,20 @@ def meta_to_s3_datasource_datafile(
         connection_name: The S3 connection name
         bucket_uri: The S3 bucket URI
         import_schedule: The import schedule (default: @auto)
+        import_format: The import format (default: auto-detected from bucket_uri)
 
     Returns:
         The datasource datafile content as a string
     """
-    file_format = get_format_from_uri(bucket_uri)
+    # Use provided format or auto-detect from bucket URI
+    # Treat "auto" the same as None to trigger auto-detection
+    # Normalize import_format to lowercase if provided
+    if import_format is not None:
+        import_format = import_format.lower()
+
+    file_format = (
+        import_format if import_format is not None and import_format != "auto" else get_format_from_uri(bucket_uri)
+    )
     schema: str = meta_to_schema(meta, file_format)
     ds_content = f"""SCHEMA >
     {schema}
@@ -273,6 +286,7 @@ ENGINE "MergeTree"
 IMPORT_CONNECTION_NAME {connection_name}
 IMPORT_BUCKET_URI "{bucket_uri}"
 IMPORT_SCHEDULE "{import_schedule}"
+IMPORT_FORMAT "{file_format}"
 # Learn more at https://www.tinybird.co/docs/forward/get-data-in/connectors/s3#datasource-settings
 """
     return ds_content

{tinybird-3.1.0.dev0 → tinybird-3.2.0}/tinybird/tb/modules/datafile/build_datasource.py

@@ -68,13 +68,24 @@ def new_ds(
                 and params.get("bucket_uri")
             ):
                 bucket_uri = params.get("bucket_uri")
+                file_format = params.get("format", "").lower()
                 extension = bucket_uri.split(".")[-1]
                 if extension == "gz":
                     extension = bucket_uri.split(".")[-2]
                 valid_formats = ["csv", "json", "jsonl", "ndjson", "parquet"]
-                if extension not in valid_formats:
-                    raise Exception(FeedbackManager.error_format(extension=extension, valid_formats=valid_formats))
-                params["format"] = extension
+
+                # If user provided a format, validate it specifically
+                if file_format:
+                    if file_format not in valid_formats:
+                        raise Exception(
+                            FeedbackManager.error_format(extension=file_format, valid_formats=valid_formats)
+                        )
+                    params["format"] = file_format
+                else:
+                    # Fall back to extension validation
+                    if extension not in valid_formats:
+                        raise Exception(FeedbackManager.error_format(extension=extension, valid_formats=valid_formats))
+                    params["format"] = extension
             datasource_response = client.datasource_create_from_definition(params)
             datasource = datasource_response.get("datasource", {})
 

{tinybird-3.1.0.dev0 → tinybird-3.2.0}/tinybird/tb/modules/datasource.py

@@ -672,6 +672,12 @@ def datasource_sync(ctx: Context, datasource_name: str, yes: bool):
     type=click.Choice(["@auto", "@once"], case_sensitive=False),
     help="S3 import schedule (@auto for automatic ingestion, @once for on-demand)",
 )
+@click.option(
+    "--s3-format",
+    "s3_format_param",
+    type=click.Choice(["csv", "ndjson", "parquet"], case_sensitive=False),
+    help="S3 import format (default: auto-detected from file extension)",
+)
 @click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
 @click.pass_context
 def datasource_create(
@@ -691,6 +697,7 @@ def datasource_create(
     s3_bucket_uri_param: Optional[str],
     s3_sample_file_param: Optional[str],
     s3_schedule_param: Optional[str],
+    s3_format_param: Optional[str],
     yes: bool,
 ):
     wizard_data: dict[str, str | bool | float] = {
@@ -1174,6 +1181,7 @@ ENGINE "MergeTree"
                 s3_bucket_uri = select_bucket_uri(None)
 
             s3_sample_file = select_sample_file_uri(s3_sample_file_param, s3_bucket_uri, s3_connection_id, client)
+            s3_format: str = s3_format_param.lower() if s3_format_param else select_format_file(None)
 
             s3_schedule: Optional[str] = None
             if s3_schedule_param:
@@ -1194,10 +1202,13 @@ ENGINE "MergeTree"
                 assert s3_bucket_uri is not None
                 assert s3_sample_file is not None
                 assert s3_schedule is not None
+                assert s3_format is not None
                 click.echo(FeedbackManager.gray(message="\n» Generating schema..."))
                 response = client.preview_s3(s3_connection_id, s3_bucket_uri, s3_sample_file, None)
                 meta = response.get("preview", {}).get("meta", [])
-                ds_content = meta_to_s3_datasource_datafile(meta, connection_name, s3_bucket_uri, s3_schedule)
+                ds_content = meta_to_s3_datasource_datafile(
+                    meta, connection_name, s3_bucket_uri, s3_schedule, s3_format
+                )
 
             while not confirmed:
                 # Select connection if not set or if user wants to change it
@@ -1235,7 +1246,9 @@ ENGINE "MergeTree"
                 preview_result = echo_s3_data(s3_connection_id, connection_name, s3_bucket_uri, s3_sample_file, client)
                 click.echo(FeedbackManager.highlight(message=f"\n» Previewing {name}.datasource"))
                 meta = preview_result["meta"]
-                ds_content = meta_to_s3_datasource_datafile(meta, connection_name, s3_bucket_uri, s3_schedule)
+                ds_content = meta_to_s3_datasource_datafile(
+                    meta, connection_name, s3_bucket_uri, s3_schedule, s3_format
+                )
                 click.echo(create_terminal_box(ds_content, title=f"{name}.datasource"))
 
                 # Confirmation step
@@ -1361,3 +1374,21 @@ def select_auto_offset_reset(current_value: Optional[str] = None) -> str:
         default=current_value or "latest",
         show_default=True,
     )
+
+
+def select_format_file(current_value: Optional[str] = None) -> str:
+    """Select file format for import operations.
+
+    Args:
+        current_value: Current format value to use as default
+
+    Returns:
+        Selected format string ("auto", "csv", "ndjson", or "parquet")
+    """
+    result = click.prompt(
+        FeedbackManager.highlight(message="? Import format"),
+        type=click.Choice(["Auto", "CSV", "NDJSON", "Parquet"], case_sensitive=False),
+        default=current_value or "Auto",
+        show_default=True,
+    )
+    return result.lower()

{tinybird-3.1.0.dev0 → tinybird-3.2.0}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 3.1.0.dev0
+Version: 3.2.0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -57,6 +57,11 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+3.0.2
+*******
+
+- `Changed` `IMPORT_FORMAT` is now optional for S3/GCS data sources and will be automatically inferred from the file extension if not provided
+
 3.0.1
 *******