tinybird 0.0.1.dev19__py3-none-any.whl → 0.0.1.dev21__py3-none-any.whl

tinybird/tb/modules/datafile/build_common.py

@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional, Tuple
 import click
 
 from tinybird.client import DoesNotExistException, TinyB
-from tinybird.feedback_manager import FeedbackManager
+from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 
 async def update_tags(resource_id: str, resource_name: str, resource_type: str, tags: List[str], tb_client: TinyB):

tinybird/tb/modules/datafile/build_datasource.py

@@ -6,8 +6,8 @@ from typing import Any, Dict, List, Optional
 import click
 
 from tinybird.client import DoesNotExistException, TinyB
-from tinybird.feedback_manager import FeedbackManager
 from tinybird.tb.modules.datafile.common import PREVIEW_CONNECTOR_SERVICES, ImportReplacements
+from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 
 async def new_ds(

tinybird/tb/modules/datafile/build_pipe.py

@@ -9,11 +9,11 @@ import requests
 from croniter import croniter
 
 from tinybird.client import DoesNotExistException, TinyB
-from tinybird.feedback_manager import FeedbackManager
 from tinybird.tb.modules.common import getenv_bool, requests_delete, requests_get, wait_job
 from tinybird.tb.modules.datafile.common import ON_DEMAND, CopyModes, CopyParameters, PipeNodeTypes, PipeTypes
 from tinybird.tb.modules.datafile.pipe_checker import PipeCheckerRunner
 from tinybird.tb.modules.exceptions import CLIPipeException
+from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.table import format_pretty_table
 
 

tinybird/tb/modules/datafile/common.py

@@ -20,9 +20,9 @@ import click
 from mypy_extensions import KwArg, VarArg
 
 from tinybird.ch_utils.engine import ENABLED_ENGINES
-from tinybird.feedback_manager import FeedbackManager
 from tinybird.tb.modules.datafile.exceptions import IncludeFileNotFoundException, ParseException, ValidationException
 from tinybird.tb.modules.exceptions import CLIPipeException
+from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 # Code from sql.py has been duplicated so I can change it without breaking absolutely everything in the app
 # I'll try not to make logic changes, just error reporting changes
@@ -33,7 +33,7 @@ class DatafileSyntaxError(Exception):
     def __init__(self, message: str, lineno: int, pos: int, hint: Optional[str] = None):
         super().__init__(message)
         self.message = message
-        self.context = None
+        self.context: Optional[str] = None
         self.hint = hint
         self.lineno = lineno
         self.pos = pos
@@ -331,12 +331,12 @@ def parse_indexes_structure(indexes: Optional[List[str]]) -> List[TableIndex]:
     return parsed_indices
 
 
-def clean_comments_rstrip_keep_empty_lines(schema_to_clean: Optional[str]) -> Tuple[Optional[str], bool]:
+def clean_comments_rstrip_keep_empty_lines(schema_to_clean: Optional[str]) -> str:
     """Remove the comments from the schema
     If the comments are between backticks, they will not be removed.
     Lines that are empty after removing comments are also removed. Lines are only rstripped of whitespaces
-    >>> clean_comments_rstrip_keep_empty_lines(None) is None
-    True
+    >>> clean_comments_rstrip_keep_empty_lines(None)
+    ''
     >>> clean_comments_rstrip_keep_empty_lines('')
     ''
     >>> clean_comments_rstrip_keep_empty_lines('    ')
@@ -394,7 +394,7 @@ def clean_comments_rstrip_keep_empty_lines(schema_to_clean: Optional[str]) -> Tu
         return line
 
     if schema_to_clean is None:
-        return schema_to_clean
+        return ""
 
     cleaned_schema = ""
     for line in schema_to_clean.splitlines():
@@ -1421,7 +1421,7 @@ def parse(
         # When the error is in a multiline block, add the start lineno to the error lineno so the error location is in
         # respect to the whole file
         if parser_state.multiline:
-            e.lineno += parser_state.start_lineno
+            e.lineno += parser_state.start_lineno or 0
         raise e
     except ParseException as e:
         raise ParseException(str(e), lineno=lineno)
@@ -1650,7 +1650,7 @@ def find_file_by_name(
     name: str,
     verbose: bool = False,
     is_raw: bool = False,
-    workspace_lib_paths: Optional[List[Tuple[str, str]]] = None,
+    vendor_paths: Optional[List[Tuple[str, str]]] = None,
     resource: Optional[Dict] = None,
 ):
     f = Path(folder)
@@ -1675,9 +1675,9 @@ def find_file_by_name(
         return token, None
 
     # look for the file in subdirectories if it's not found in datasources folder
-    if workspace_lib_paths:
+    if vendor_paths:
         _resource = None
-        for wk_name, wk_path in workspace_lib_paths:
+        for wk_name, wk_path in vendor_paths:
             file = None
             if name.startswith(f"{wk_name}."):
                 file, _resource = find_file_by_name(
@@ -1692,7 +1692,7 @@ def find_file_by_name(
             name,
             verbose=verbose,
             is_raw=True,
-            workspace_lib_paths=workspace_lib_paths,
+            vendor_paths=vendor_paths,
             resource=resource,
         )
         return f, raw

tinybird/tb/modules/datafile/diff.py

@@ -23,12 +23,12 @@ import sys
 import click
 
 from tinybird.client import TinyB
-from tinybird.feedback_manager import FeedbackManager
 from tinybird.sql_template_fmt import DEFAULT_FMT_LINE_LENGTH
 from tinybird.tb.modules.datafile.common import get_name_version, get_project_filenames, is_file_a_datasource, peek
 from tinybird.tb.modules.datafile.format_datasource import format_datasource
 from tinybird.tb.modules.datafile.format_pipe import format_pipe
 from tinybird.tb.modules.datafile.pull import folder_pull
+from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 
 async def diff_files(

tinybird/tb/modules/datafile/fixture.py

@@ -18,7 +18,7 @@ def build_fixture_name(filename: str, datasource_name: str, datasource_content:
         str: A unique fixture name combining a hash of the inputs with the datasource name
     """
 
-    doc = parse_datasource(filename, datasource_content)
+    doc = parse_datasource(filename, content=datasource_content)
     schema = doc.nodes[0].get("schema", "").strip()
     # Combine all inputs into a single string
     combined = f"{datasource_name}{schema}"

tinybird/tb/modules/datafile/format_common.py

@@ -20,13 +20,6 @@ def format_description(file_parts: List[str], doc: Any) -> List[str]:
     return file_parts
 
 
-def format_sources(file_parts: List[str], doc: Datafile) -> List[str]:
-    for source in doc.sources:
-        file_parts.append(f"SOURCE {source}")
-        file_parts.append(DATAFILE_NEW_LINE)
-    return file_parts
-
-
 def format_maintainer(file_parts: List[str], doc: Datafile) -> List[str]:
     maintainer = doc.maintainer if doc.maintainer is not None else ""
     if maintainer:

tinybird/tb/modules/datafile/format_datasource.py

@@ -9,7 +9,6 @@ from tinybird.tb.modules.datafile.format_common import (
     format_description,
     format_include,
     format_maintainer,
-    format_sources,
     format_tags,
     format_tokens,
 )
@@ -61,7 +60,6 @@ async def format_datasource(
             format_import_settings(file_parts, doc.nodes[0])
         format_shared_with(file_parts, doc)
     else:
-        format_sources(file_parts, doc)
         format_maintainer(file_parts, doc)
         format_description(file_parts, doc)
         format_tokens(file_parts, doc)

tinybird/tb/modules/datafile/format_pipe.py

@@ -17,7 +17,6 @@ from tinybird.tb.modules.datafile.format_common import (
     DATAFILE_NEW_LINE,
     format_description,
     format_maintainer,
-    format_sources,
     format_tags,
     format_tokens,
 )
@@ -140,7 +139,6 @@ async def format_pipe(
         doc = parse_pipe(filename, replace_includes=replace_includes, skip_eval=skip_eval, content=content)
 
     file_parts: List[str] = []
-    format_sources(file_parts, doc)
     format_maintainer(file_parts, doc)
     format_description(file_parts, doc)
     format_tokens(file_parts, doc)

tinybird/tb/modules/datafile/parse_datasource.py

@@ -3,7 +3,6 @@ from typing import Optional
 
 import click
 
-from tinybird.feedback_manager import FeedbackManager
 from tinybird.tb.modules.datafile.common import (
     Datafile,
     DatafileSyntaxError,
@@ -11,6 +10,7 @@ from tinybird.tb.modules.datafile.common import (
     parse,
 )
 from tinybird.tb.modules.datafile.exceptions import ParseException
+from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 
 def parse_datasource(

tinybird/tb/modules/datafile/parse_pipe.py

@@ -3,7 +3,6 @@ from typing import Optional
 
 import click
 
-from tinybird.feedback_manager import FeedbackManager
 from tinybird.sql_template import get_template_and_variables, render_sql_template
 from tinybird.tb.modules.datafile.common import (
     Datafile,
@@ -12,6 +11,7 @@ from tinybird.tb.modules.datafile.common import (
     parse,
 )
 from tinybird.tb.modules.datafile.exceptions import IncludeFileNotFoundException, ParseException
+from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tornado_template import UnClosedIfError
 
 

tinybird/tb/modules/datafile/pull.py

@@ -7,10 +7,10 @@ import aiofiles
 import click
 
 from tinybird.client import AuthNoTokenException, TinyB
-from tinybird.feedback_manager import FeedbackManager
 from tinybird.tb.modules.datafile.common import get_name_version
 from tinybird.tb.modules.datafile.format_datasource import format_datasource
 from tinybird.tb.modules.datafile.format_pipe import format_pipe
+from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 
 async def folder_pull(

tinybird/tb/modules/datasource.py

@@ -20,7 +20,6 @@ from tinybird.tb.modules.config import CLIConfig
 if TYPE_CHECKING:
     from tinybird.connectors import Connector
 
-from tinybird.feedback_manager import FeedbackManager
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import (
     _analyze,
@@ -42,6 +41,7 @@ from tinybird.tb.modules.common import (
 )
 from tinybird.tb.modules.datafile.common import get_name_version
 from tinybird.tb.modules.exceptions import CLIDatasourceException
+from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 
 @cli.group()
@@ -125,26 +125,6 @@ async def datasource_ls(ctx: Context, match: Optional[str], format_: str):
 @datasource.command(name="append")
 @click.argument("datasource_name")
 @click.argument("url", nargs=-1)
-@click.option(
-    "--connector",
-    type=click.Choice(["bigquery", "snowflake"], case_sensitive=True),
-    help="Import from one of the selected connectors",
-    hidden=True,
-)
-@click.option("--sql", default=None, help="Query to extract data from one of the SQL connectors", hidden=True)
-@click.option(
-    "--incremental",
-    default=None,
-    help="It does an incremental append, taking the max value for the date column name provided as a parameter. It only works when the `connector` parameter is passed.",
-    hidden=True,
-)
-@click.option(
-    "--ignore-empty",
-    help="Wheter or not to ignore empty results from the connector",
-    is_flag=True,
-    default=False,
-    hidden=True,
-)
 @click.option("--concurrency", help="How many files to submit concurrently", default=1, hidden=True)
 @click.pass_context
 @coro
@@ -152,10 +132,6 @@ async def datasource_append(
     ctx: Context,
     datasource_name: str,
     url,
-    connector: Optional[str],
-    sql: Optional[str],
-    incremental: Optional[str],
-    ignore_empty: bool,
     concurrency: int,
 ):
     """
@@ -164,37 +140,14 @@ async def datasource_append(
     - Load from URL `tb datasource append [datasource_name] https://url_to_csv`
 
     - Load from local file `tb datasource append [datasource_name] /path/to/local/file`
-
-    - Load from connector `tb datasource append [datasource_name] --connector [connector_name] --sql [the_sql_to_extract_from]`
     """
 
-    if not url and not connector:
-        raise CLIDatasourceException(FeedbackManager.error_missing_url_or_connector(datasource=datasource_name))
-
-    if incremental and not connector:
-        raise CLIDatasourceException(FeedbackManager.error_incremental_not_supported())
-
     client: TinyB = ctx.obj["client"]
-    if incremental:
-        date = None
-        source_column = incremental.split(":")[0]
-        dest_column = incremental.split(":")[-1]
-        result = await client.query(f"SELECT max({dest_column}) as inc from {datasource_name} FORMAT JSON")
-        try:
-            date = result["data"][0]["inc"]
-        except Exception as e:
-            raise CLIDatasourceException(f"{str(e)}")
-        if date:
-            sql = f"{sql} WHERE {source_column} > '{date}'"
     await push_data(
-        ctx,
         client,
         datasource_name,
         url,
-        connector,
-        sql,
         mode="append",
-        ignore_empty=ignore_empty,
         concurrency=concurrency,
     )
 
@@ -202,33 +155,16 @@ async def datasource_append(
 @datasource.command(name="replace")
 @click.argument("datasource_name")
 @click.argument("url", nargs=-1)
-@click.option(
-    "--connector",
-    type=click.Choice(["bigquery", "snowflake"], case_sensitive=True),
-    help="Import from one of the selected connectors",
-    hidden=True,
-)
-@click.option("--sql", default=None, help="Query to extract data from one of the SQL connectors", hidden=True)
 @click.option("--sql-condition", default=None, help="SQL WHERE condition to replace data", hidden=True)
 @click.option("--skip-incompatible-partition-key", is_flag=True, default=False, hidden=True)
-@click.option(
-    "--ignore-empty",
-    help="Wheter or not to ignore empty results from the connector",
-    is_flag=True,
-    default=False,
-    hidden=True,
-)
 @click.pass_context
 @coro
 async def datasource_replace(
-    ctx,
+    ctx: Context,
     datasource_name,
     url,
-    connector,
-    sql,
     sql_condition,
     skip_incompatible_partition_key,
-    ignore_empty: bool,
 ):
     """
     Replaces the data in a data source from a URL, local file or a connector
@@ -236,26 +172,19 @@ async def datasource_replace(
     - Replace from URL `tb datasource replace [datasource_name] https://url_to_csv --sql-condition "country='ES'"`
 
     - Replace from local file `tb datasource replace [datasource_name] /path/to/local/file --sql-condition "country='ES'"`
-
-    - Replace from connector `tb datasource replace [datasource_name] --connector [connector_name] --sql [the_sql_to_extract_from] --sql-condition "country='ES'"`
     """
 
-    if not url and not connector:
-        raise CLIDatasourceException(FeedbackManager.error_missing_url_or_connector(datasource=datasource_name))
-
     replace_options = set()
     if skip_incompatible_partition_key:
         replace_options.add("skip_incompatible_partition_key")
+    client: TinyB = ctx.obj["client"]
     await push_data(
-        ctx,
+        client,
         datasource_name,
         url,
-        connector,
-        sql,
         mode="replace",
         sql_condition=sql_condition,
         replace_options=replace_options,
-        ignore_empty=ignore_empty,
     )