tinybird 0.0.1.dev180__tar.gz → 0.0.1.dev182__tar.gz

{tinybird-0.0.1.dev180 → tinybird-0.0.1.dev182}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev180
+Version: 0.0.1.dev182
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev180 → tinybird-0.0.1.dev182}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev180'
-__revision__ = '4ea1a7e'
+__version__ = '0.0.1.dev182'
+__revision__ = 'b1eeede'

{tinybird-0.0.1.dev180 → tinybird-0.0.1.dev182}/tinybird/tb/client.py

@@ -4,7 +4,7 @@ import logging
 import os
 import ssl
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Union
+from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Tuple, Union
 from urllib.parse import quote, urlencode
 
 import aiofiles
@@ -402,6 +402,9 @@ class TinyB:
     async def pipe_file(self, pipe: str):
         return await self._req(f"/v1/pipes/{pipe}.pipe")
 
+    async def connection_file(self, connection: str):
+        return await self._req(f"/v0/connectors/{connection}.connection")
+
     async def datasource_file(self, datasource: str):
         try:
             return await self._req(f"/v0/datasources/{datasource}.datasource")
@@ -693,12 +696,30 @@ class TinyB:
         else:
             return await self._req(f"/v0/sql?q={quote(sql, safe='')}&{urlencode(params)}")
 
-    async def jobs(self, status=None):
-        jobs = (await self._req("/v0/jobs"))["jobs"]
-        if status:
-            status = [status] if isinstance(status, str) else status
-            jobs = [j for j in jobs if j["status"] in status]
-        return jobs
+    async def jobs(
+        self, status: Optional[Tuple[str, ...]] = None, kind: Optional[Tuple[str, ...]] = None
+    ) -> List[Dict[str, Any]]:
+        async def fetch_jobs(params: Dict[str, str]) -> List[Dict[str, Any]]:
+            query_string = urlencode(params) if params else ""
+            endpoint = f"/v0/jobs?{query_string}" if query_string else "/v0/jobs"
+            response = await self._req(endpoint)
+            return response["jobs"]
+
+        if not status and not kind:
+            return await fetch_jobs({})
+        result: List[Dict[str, Any]] = []
+        if status and kind:
+            for s in status:
+                for k in kind:
+                    result.extend(await fetch_jobs({"status": s, "kind": k}))
+        elif status:
+            for s in status:
+                result.extend(await fetch_jobs({"status": s}))
+        elif kind:
+            for k in kind:
+                result.extend(await fetch_jobs({"kind": k}))
+
+        return result
 
     async def job(self, job_id: str):
         return await self._req(f"/v0/jobs/{job_id}")

{tinybird-0.0.1.dev180 → tinybird-0.0.1.dev182}/tinybird/tb/modules/datafile/build_datasource.py

@@ -87,7 +87,7 @@ async def new_ds(
             if datasource.get("service") == "dynamodb":
                 job_id = datasource_response.get("import_id", None)
                 if job_id:
-                    jobs = await client.jobs(status=["waiting", "working"])
+                    jobs = await client.jobs(status=("waiting", "working"))
                     job_url = next((job["job_url"] for job in jobs if job["id"] == job_id), None)
                     if job_url:
                         click.echo(FeedbackManager.success_dynamodb_initial_load(job_url=job_url))

{tinybird-0.0.1.dev180 → tinybird-0.0.1.dev182}/tinybird/tb/modules/datafile/common.py

@@ -1261,22 +1261,12 @@ class ParseResult(NamedTuple):
 
 def parse(
     s: str,
+    kind: DatafileKind,
     default_node: Optional[str] = None,
     basepath: str = ".",
     replace_includes: bool = True,
     skip_eval: bool = False,
-    kind: Optional[DatafileKind] = None,
 ) -> ParseResult:
-    """
-    Parses `s` string into a document
-    >>> d, w = parse("NODE \\"test_01\\"\\n    DESCRIPTION this is a node that does whatever\\nSQL >\\n\\n        SELECT * from test_00\\n\\n\\nNODE \\"test_02\\"\\n    DESCRIPTION this is a node that does whatever\\nSQL >\\n\\n    SELECT * from test_01\\n    WHERE a > 1\\n    GROUP by a\\n")
-    >>> len(d.nodes)
-    2
-    >>> d.nodes[0]
-    {'name': 'test_01', 'description': 'this is a node that does whatever', 'sql': 'SELECT * from test_00'}
-    >>> d.nodes[1]
-    {'name': 'test_02', 'description': 'this is a node that does whatever', 'sql': 'SELECT * from test_01\\nWHERE a > 1\\nGROUP by a'}
-    """
     lines = list(StringIO(s, newline=None))
 
     doc = Datafile()
@@ -1403,21 +1393,28 @@ def parse(
 
         return _f
 
-    def kafka_key_avro_deserialization(*args: str, **kwargs: Any):
+    def kafka_key_avro_deserialization_deprecated(*args: str, **kwargs: Any):
         raise DatafileSyntaxError(
             f'{kwargs["cmd"].upper()} has been deprecated. Use "KAFKA_KEY_FORMAT avro" in the corresponding .datasource file instead',
             lineno=kwargs["lineno"],
             pos=1,
         )
 
-    def kafka_target_partitions(*args: str, **kwargs: Any):
+    def kafka_target_partitions_deprecated(*args: str, **kwargs: Any):
         warnings.append(
             DatafileParseWarning(
                 message=f"{kwargs['cmd'].upper()} has been deprecated and will be ignored.",
             )
         )
 
-    def import_service(*args: str, **kwargs: Any):
+    def import_external_datasource_deprecated(*args: str, **kwargs: Any):
+        warnings.append(
+            DatafileParseWarning(
+                message=f"{kwargs['cmd'].upper()} has been deprecated and will be ignored.",
+            )
+        )
+
+    def import_service_deprecated(*args: str, **kwargs: Any):
         warnings.append(
             DatafileParseWarning(
                 message=(
@@ -1427,14 +1424,14 @@ def parse(
             )
         )
 
-    def import_strategy(*args: str, **kwargs: Any):
+    def import_strategy_deprecated(*args: str, **kwargs: Any):
         raise DatafileSyntaxError(
             f"{kwargs['cmd'].upper()} has been deprecated. It is now fixed to 'append'",
             lineno=kwargs["lineno"],
             pos=1,
         )
 
-    def kafka_store_binary_headers(*args: str, **kwargs: Any):
+    def kafka_store_binary_headers_deprecated(*args: str, **kwargs: Any):
         raise DatafileSyntaxError(
             f"{kwargs['cmd'].upper()} has been deprecated. When KAFKA_STORE_HEADERS is True, __headers is always of type Map(String, String)",
             lineno=kwargs["lineno"],
@@ -1652,17 +1649,17 @@ def parse(
             "kafka_auto_offset_reset": assign_var("kafka_auto_offset_reset"),
             "kafka_store_raw_value": assign_var("kafka_store_raw_value"),
             "kafka_store_headers": assign_var("kafka_store_headers"),
-            "kafka_store_binary_headers": kafka_store_binary_headers,
+            "kafka_store_binary_headers": kafka_store_binary_headers_deprecated,
             "kafka_key_format": assign_var("kafka_key_format"),
             "kafka_value_format": assign_var("kafka_value_format"),
-            "kafka_target_partitions": kafka_target_partitions,  # Deprecated
+            "kafka_target_partitions": kafka_target_partitions_deprecated,  # Deprecated
             "import_connection_name": assign_var("import_connection_name"),
             "import_schedule": assign_var("import_schedule"),
-            "import_strategy": import_strategy,  # Deprecated, always append
+            "import_strategy": import_strategy_deprecated,  # Deprecated, always append
             "import_bucket_uri": assign_var("import_bucket_uri"),
             "import_from_timestamp": assign_var("import_from_timestamp"),
-            "import_service": import_service,  # Deprecated
-            "import_external_datasource": assign_var("import_external_datasource"),  # Deprecated, BQ and SFK
+            "import_service": import_service_deprecated,  # Deprecated
+            "import_external_datasource": import_external_datasource_deprecated,  # Deprecated, BQ and SFK
             "import_query": assign_var("import_query"),  # Deprecated, BQ and SFK
             "import_table_arn": assign_var("import_table_arn"),  # Only for DynamoDB
             "import_export_bucket": assign_var("import_export_bucket"),  # For DynamoDB
@@ -1678,6 +1675,7 @@ def parse(
             "export_compression": assign_var("export_compression"),
             "export_write_strategy": assign_var("export_write_strategy"),
             "export_kafka_topic": assign_var("export_kafka_topic"),
+            # ENGINE_* commands are added dynamically after this dict's definition
         },
         DatafileKind.pipe: {
             "node": node,
@@ -1706,7 +1704,7 @@ def parse(
             "kafka_ssl_ca_pem": assign_var("kafka_ssl_ca_pem"),
             "kafka_security_protocol": assign_var("kafka_security_protocol"),
             "kafka_sasl_mechanism": assign_var("kafka_sasl_mechanism"),
-            "kafka_key_avro_deserialization": kafka_key_avro_deserialization,
+            "kafka_key_avro_deserialization": kafka_key_avro_deserialization_deprecated,
             "s3_region": assign_var("s3_region"),
             "s3_arn": assign_var("s3_arn"),
             "s3_access_key": assign_var("s3_access_key"),
@@ -1729,13 +1727,7 @@ def parse(
     if default_node:
         node(default_node)
 
-    def get_all_cmds() -> dict[str, Callable]:
-        all_cmds: dict[str, Callable] = {}
-        for _, cmds in cmds_per_datafile_kind.items():
-            all_cmds.update(cmds)
-        return all_cmds
-
-    cmds = cmds_per_datafile_kind[kind] if kind is not None else get_all_cmds()
+    cmds = cmds_per_datafile_kind[kind]
 
     lineno = 1
     try:

{tinybird-0.0.1.dev180 → tinybird-0.0.1.dev182}/tinybird/tb/modules/datafile/parse_datasource.py

@@ -36,7 +36,12 @@ def parse_datasource(
     filename = format_filename(filename, hide_folders)
     try:
         doc, warnings = parse(
-            s, "default", basepath, replace_includes=replace_includes, skip_eval=skip_eval, kind=DatafileKind.datasource
+            s,
+            default_node="default",
+            basepath=basepath,
+            replace_includes=replace_includes,
+            skip_eval=skip_eval,
+            kind=DatafileKind.datasource,
         )
         doc.validate()
     except DatafileSyntaxError as e:

{tinybird-0.0.1.dev180 → tinybird-0.0.1.dev182}/tinybird/tb/modules/datafile/pull.py

@@ -1,13 +1,11 @@
-import re
 from asyncio import Semaphore, gather
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Optional
 
 import aiofiles
 import click
 
 from tinybird.tb.client import AuthNoTokenException, TinyB
-from tinybird.tb.modules.datafile.common import get_name_version
 from tinybird.tb.modules.datafile.format_datasource import format_datasource
 from tinybird.tb.modules.datafile.format_pipe import format_pipe
 from tinybird.tb.modules.feedback_manager import FeedbackManager
@@ -21,24 +19,11 @@ async def folder_pull(
     progress_bar: bool = False,
     fmt: bool = False,
 ):
-    def _get_latest_versions(resources: List[Tuple[str, str]]):
-        versions: Dict[str, Any] = {}
-
-        for x, resource_type in resources:
-            t = get_name_version(x)
-            t["original_name"] = x
-            if t["version"] is None:
-                t["version"] = -1
-            name = t["name"]
-            t["type"] = resource_type
-
-            if name not in versions or name == x or versions[name]["version"] < t["version"]:
-                versions[name] = t
-        return versions
-
     def get_file_folder(extension: str, resource_type: Optional[str]):
         if extension == "datasource":
             return "datasources"
+        if extension == "connection":
+            return "connections"
         if resource_type == "endpoint":
             return "endpoints"
         if resource_type == "sink":
@@ -52,17 +37,16 @@ async def folder_pull(
         return None
 
     async def write_files(
-        versions: Dict[str, Any],
-        resources: List[str],
+        resources: list[dict[str, Any]],
         extension: str,
         get_resource_function: str,
         progress_bar: bool = False,
         fmt: bool = False,
     ):
-        async def write_resource(k: Dict[str, Any]):
+        async def write_resource(k: dict[str, Any]):
             name = f"{k['name']}.{extension}"
             try:
-                resource = await getattr(client, get_resource_function)(k["original_name"])
+                resource = await getattr(client, get_resource_function)(k["name"])
                 resource_to_write = resource
 
                 if fmt:
@@ -76,32 +60,19 @@ async def folder_pull(
                     dest_folder = Path(folder) / "vendor" / k["name"].split(".", 1)[0]
                     name = f"{k['name'].split('.', 1)[1]}.{extension}"
 
-                file_folder = get_file_folder(extension, k["type"])
+                file_folder = get_file_folder(extension, k.get("type"))
                 f = Path(dest_folder) / file_folder if file_folder is not None else Path(dest_folder)
 
                 if not f.exists():
                     f.mkdir(parents=True)
 
                 f = f / name
-                resource_names = [x.split(".")[-1] for x in resources]
 
                 if verbose:
                     click.echo(FeedbackManager.info_writing_resource(resource=f))
                 if not f.exists() or force:
                     async with aiofiles.open(f, "w") as fd:
-                        # versions are a client only thing so
-                        # datafiles from the server do not contains information about versions
-                        if k["version"] >= 0:
-                            if fmt:
-                                resource_to_write = "\n" + resource_to_write  # fmt strips the first line
-
-                            resource_to_write = f"VERSION {k['version']}\n" + resource_to_write
                         if resource_to_write:
-                            matches = re.findall(r"([^\s\.]*__v\d+)", resource_to_write)
-                            for match in set(matches):
-                                m = match.split("__v")[0]
-                                if m in resources or m in resource_names:
-                                    resource_to_write = resource_to_write.replace(match, m)
                             await fd.write(resource_to_write)
                 else:
                     if verbose:
@@ -109,32 +80,40 @@ async def folder_pull(
             except Exception as e:
                 raise click.ClickException(FeedbackManager.error_exception(error=e))
 
-        values = versions.values()
-
         if progress_bar:
-            with click.progressbar(values, label=f"Pulling {extension}s") as values:  # type: ignore
-                for k in values:
+            with click.progressbar(resources, label=f"Pulling {extension}s") as resources:  # type: ignore
+                for k in resources:
                     await write_resource(k)
         else:
-            tasks = [write_resource(k) for k in values]
+            tasks = [write_resource(k) for k in resources]
             await _gather_with_concurrency(5, *tasks)
 
     try:
         datasources = await client.datasources()
-        remote_datasources = sorted([(x["name"], x.get("type", "csv")) for x in datasources], key=lambda x: x[0])
-        datasources_versions = _get_latest_versions(remote_datasources)
-
         pipes = await client.pipes()
-        remote_pipes = sorted([(pipe["name"], pipe.get("type", "default")) for pipe in pipes], key=lambda x: x[0])
-        pipes_versions = _get_latest_versions(remote_pipes)
-
-        resources = list(datasources_versions.keys()) + list(pipes_versions.keys())
+        connections = await client.connections(skip_bigquery=True)
 
         await write_files(
-            datasources_versions, resources, "datasource", "datasource_file", progress_bar=progress_bar, fmt=fmt
+            resources=datasources,
+            extension="datasource",
+            get_resource_function="datasource_file",
+            progress_bar=progress_bar,
+            fmt=fmt,
+        )
+        await write_files(
+            resources=pipes,
+            extension="pipe",
+            get_resource_function="pipe_file",
+            progress_bar=progress_bar,
+            fmt=fmt,
+        )
+        await write_files(
+            resources=connections,
+            extension="connection",
+            get_resource_function="connection_file",
+            progress_bar=progress_bar,
+            fmt=fmt,
         )
-        await write_files(pipes_versions, resources, "pipe", "pipe_file", progress_bar=progress_bar, fmt=fmt)
-
         return
 
     except AuthNoTokenException:

{tinybird-0.0.1.dev180 → tinybird-0.0.1.dev182}/tinybird/tb/modules/job.py

@@ -3,6 +3,8 @@
 # - If it makes sense and only when strictly necessary, you can create utility functions in this file.
 # - But please, **do not** interleave utility functions and command definitions.
 
+from typing import Tuple
+
 import click
 from click import Context
 
@@ -24,16 +26,23 @@ def job(ctx: Context) -> None:
     "-s",
     "--status",
     help="Show only jobs with this status",
-    type=click.Choice(["waiting", "working", "done", "error"], case_sensitive=False),
+    type=click.Choice(["waiting", "working", "done", "error", "cancelling", "cancelled"], case_sensitive=False),
+    multiple=True,
+    default=None,
+)
+@click.option(
+    "-k",
+    "--kind",
+    help="Show only jobs of this kind",
     multiple=True,
     default=None,
 )
 @click.pass_context
 @coro
-async def jobs_ls(ctx: Context, status: str) -> None:
-    """List jobs, up to 100 in the last 48h"""
+async def jobs_ls(ctx: Context, status: Tuple[str, ...], kind: Tuple[str, ...]) -> None:
+    """List jobs, up to 100"""
     client: TinyB = ctx.ensure_object(dict)["client"]
-    jobs = await client.jobs(status=status)
+    jobs = await client.jobs(status=status, kind=kind)
     columns = ["id", "kind", "status", "created at", "updated at", "job url"]
     click.echo(FeedbackManager.info_jobs())
     table = []

{tinybird-0.0.1.dev180 → tinybird-0.0.1.dev182}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev180
+Version: 0.0.1.dev182
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird