tinybird 0.0.1.dev42__tar.gz → 0.0.1.dev43__tar.gz

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird
-Version: 0.0.1.dev42
+Version: 0.0.1.dev43
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev43}/tinybird/client.py

@@ -72,7 +72,7 @@ def parse_error_response(response: Response) -> str:
         if content.get("error", None):
             error = content["error"]
             if content.get("errors", None):
-                error += f' -> errors: {content.get("errors")}'
+                error += f" -> errors: {content.get('errors')}"
         else:
             error = json.dumps(response, indent=4)
         return error

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev43}/tinybird/connectors.py

@@ -246,7 +246,7 @@ class BigQuery(Connector):
                     uri='{self.gcs.gs_url()}{destination}*.csv',
                     format='CSV',
                     overwrite=true,
-                    header={ "true" if with_headers else "false" },
+                    header={"true" if with_headers else "false"},
                     field_delimiter=',') AS
                     {sql}
                 """
@@ -319,7 +319,7 @@ class Snowflake(Connector):
 
     def create_stage(self):
         sql = f"""
-                    create stage "{self.options['schema']}".{self.stage()}
+                    create stage "{self.options["schema"]}".{self.stage()}
                         url='{self.gcs.gcs_url()}'
                         storage_integration = {self.storage_integration()};
                 """
@@ -337,7 +337,7 @@ class Snowflake(Connector):
                     from ({sql})
                     overwrite = true
                     file_format = (TYPE=CSV COMPRESSION=NONE ESCAPE_UNENCLOSED_FIELD=NONE FIELD_DELIMITER='|' FIELD_OPTIONALLY_ENCLOSED_BY='"' null_if=())
-                    header = {"true" if with_headers else "false" }
+                    header = {"true" if with_headers else "false"}
                     max_file_size = 2500000000;
                 """
         self.execute(sql)

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev43}/tinybird/datafile.py

@@ -2175,8 +2175,8 @@ class PipeCheckerRunner:
                             AND extractURLParameter(assumeNotNull(url), 'debug') <> 'query'
                             AND error = 0
                             AND not mapContains(parameters, '__tb__semver')
-                            {" AND " + " AND ".join([f"mapContains(pipe_request_params, '{match}')" for match in matches]) if matches and len(matches) > 0 else ''}
-                            { extra_where_clause }
+                            {" AND " + " AND ".join([f"mapContains(pipe_request_params, '{match}')" for match in matches]) if matches and len(matches) > 0 else ""}
+                            {extra_where_clause}
                         Limit 5000000 -- Enough to bring data while not processing all requests from highly used pipes
                         )
                     group by request_param_names, http_method
@@ -2202,7 +2202,7 @@ class PipeCheckerRunner:
                                         AND extractURLParameter(assumeNotNull(url), 'debug') <> 'query'
                                         AND error = 0
                                         AND not mapContains(parameters, '__tb__semver')
-                                        {" AND " + " AND ".join([f"mapContains(pipe_request_params, '{match}')" for match in matches]) if matches and len(matches) > 0 else ''}
+                                        {" AND " + " AND ".join([f"mapContains(pipe_request_params, '{match}')" for match in matches]) if matches and len(matches) > 0 else ""}
                                         {extra_where_clause}
                                     LIMIT {limit}
                                 )
@@ -4425,7 +4425,7 @@ async def folder_push(
                                     name=(
                                         name
                                         if to_run[name]["version"] is None
-                                        else f'{name}__v{to_run[name]["version"]}'
+                                        else f"{name}__v{to_run[name]['version']}"
                                     )
                                 )
                             )
@@ -4442,7 +4442,7 @@ async def folder_push(
                     if raise_on_exists:
                         raise AlreadyExistsException(
                             FeedbackManager.warning_name_already_exists(
-                                name=name if to_run[name]["version"] is None else f'{name}__v{to_run[name]["version"]}'
+                                name=name if to_run[name]["version"] is None else f"{name}__v{to_run[name]['version']}"
                             )
                         )
                     else:
@@ -4457,7 +4457,7 @@ async def folder_push(
                                     name=(
                                         name
                                         if to_run[name]["version"] is None
-                                        else f'{name}__v{to_run[name]["version"]}'
+                                        else f"{name}__v{to_run[name]['version']}"
                                     )
                                 )
                             )
@@ -5013,7 +5013,7 @@ async def format_engine(
     else:
         if node.get("engine", None):
             empty = '""'
-            file_parts.append(f'ENGINE {node["engine"]["type"]}' if node.get("engine", {}).get("type") else empty)
+            file_parts.append(f"ENGINE {node['engine']['type']}" if node.get("engine", {}).get("type") else empty)
             file_parts.append(DATAFILE_NEW_LINE)
             for arg in sorted(node["engine"].get("args", [])):
                 elem = ", ".join([x.strip() for x in arg[1].split(",")])
@@ -5030,7 +5030,7 @@ async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[
     if node_type == PipeNodeTypes.MATERIALIZED:
         file_parts.append(node_type_upper)
         file_parts.append(DATAFILE_NEW_LINE)
-        file_parts.append(f'DATASOURCE {node["datasource"]}')
+        file_parts.append(f"DATASOURCE {node['datasource']}")
         file_parts.append(DATAFILE_NEW_LINE)
         await format_engine(file_parts, node)
 
@@ -5038,10 +5038,10 @@ async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[
     if node_type == PipeNodeTypes.COPY:
         file_parts.append(node_type_upper)
         file_parts.append(DATAFILE_NEW_LINE)
-        file_parts.append(f'TARGET_DATASOURCE {node["target_datasource"]}')
+        file_parts.append(f"TARGET_DATASOURCE {node['target_datasource']}")
         if node.get("mode"):
             file_parts.append(DATAFILE_NEW_LINE)
-            file_parts.append(f'COPY_MODE {node.get("mode")}')
+            file_parts.append(f"COPY_MODE {node.get('mode')}")
 
         if node.get(CopyParameters.COPY_SCHEDULE):
             is_ondemand = node[CopyParameters.COPY_SCHEDULE].lower() == ON_DEMAND
@@ -5095,7 +5095,7 @@ async def format_node(
     if item and not unroll_includes:
         return
 
-    file_parts.append(f'NODE {node["name"].strip()}')
+    file_parts.append(f"NODE {node['name'].strip()}")
     file_parts.append(DATAFILE_NEW_LINE)
 
     from collections import namedtuple

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev43}/tinybird/feedback_manager.py

@@ -951,7 +951,7 @@ Ready? """
     )
     success_datasource_alter = success_message("** The Data Source has been correctly updated.")
     success_datasource_kafka_connected = success_message(
-        "** Data Source '{id}' created\n" "** Kafka streaming connection configured successfully!"
+        "** Data Source '{id}' created\n** Kafka streaming connection configured successfully!"
     )
     success_datasource_shared = success_message(
         "** The Data Source {datasource} has been correctly shared with {workspace}"

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev43}/tinybird/prompts.py

@@ -373,6 +373,7 @@ You are a Tinybird expert. You will be given a pipe containing different nodes w
     - The parameter within Tinybird templating syntax looks like this one {{String(my_param_name, default_value)}}.
     - If there are no parameters, you can omit parameters and generate a single test.
     - The format of the parameters is the following: ?param1=value1&param2=value2&param3=value3
+    - If some parameters are provided by the user and you need to use them, preserve in the same format as they were provided, like case sensitive.
 </instructions>
 
 This is an example of a test with parameters:

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev43}/tinybird/sql.py

@@ -241,7 +241,7 @@ def format_parse_error(
         message += f" found at position {adjusted_position - len(keyword)}"
     else:
         message += (
-            f" found {repr(table_structure[i]) if len(table_structure)>i else 'EOF'} at position {adjusted_position}"
+            f" found {repr(table_structure[i]) if len(table_structure) > i else 'EOF'} at position {adjusted_position}"
         )
     return message
 

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev43}/tinybird/sql_template_fmt.py

@@ -100,7 +100,7 @@ def _format_jinja_node(self, node: Node, max_length: int) -> bool:
                 parts = tag.code.split("\n")
                 prefix = INDENT * (node.depth[0] + node.depth[1])
                 if len(parts) > 1:
-                    tag.code = "\n".join([f'{prefix if i != 0 else ""}{part}' for i, part in enumerate(parts)])
+                    tag.code = "\n".join([f"{prefix if i != 0 else ''}{part}" for i, part in enumerate(parts)])
 
         node.value = str(tag)
 

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev43}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev42'
-__revision__ = '14b255c'
+__version__ = '0.0.1.dev43'
+__revision__ = 'ecb1311'

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev43}/tinybird/tb/modules/build.py

@@ -31,17 +31,16 @@ def build(ctx: click.Context, watch: bool) -> None:
     project: Project = ctx.ensure_object(dict)["project"]
     tb_client = asyncio.run(get_tinybird_local_client(str(project.path)))
     click.echo(FeedbackManager.highlight(message="\n» Building project..."))
-
     time_start = time.time()
 
-    def process(file_changed: Optional[str] = None) -> None:
+    def process(file_changed: Optional[str] = None, diff: Optional[str] = None) -> None:
         if file_changed and file_changed.endswith(".ndjson"):
             rebuild_fixture(project, tb_client, file_changed)
         else:
             build_project(project, tb_client)
             try:
                 if file_changed:
-                    build_and_print_resource(tb_client, file_changed)
+                    build_and_print_resource(tb_client, file_changed, diff)
             except Exception:
                 pass
 
@@ -79,6 +78,9 @@ def build_project(project: Project, tb_client: TinyB) -> None:
         project_path = project.path
         project_files = project.get_project_files()
 
+        if not project_files:
+            return
+
         for file_path in project_files:
             relative_path = str(Path(file_path).relative_to(project_path))
             fd = open(file_path, "rb")
@@ -102,16 +104,19 @@ def build_project(project: Project, tb_client: TinyB) -> None:
             datasources = result.get("datasources", [])
             pipes = result.get("pipes", [])
             for ds in datasources:
-                ds_path = next((p for p in project_files if p.endswith(ds.get("name") + ".datasource")), None)
-                if ds_path:
-                    ds_path = ds_path.replace(f"{project.folder}/", "")
-                    click.echo(FeedbackManager.info(message=f"✓ {ds_path} created"))
+                ds_path_str: Optional[str] = next(
+                    (p for p in project_files if p.endswith(ds.get("name") + ".datasource")), None
+                )
+                if ds_path_str:
+                    ds_path = Path(ds_path_str)
+                    ds_path_str = ds_path_str.replace(f"{project.folder}/", "")
+                    click.echo(FeedbackManager.info(message=f"✓ {ds_path_str} created"))
             for pipe in pipes:
                 pipe_name = pipe.get("name")
-                pipe_path = next((p for p in project_files if p.endswith(pipe_name + ".pipe")), None)
-                if pipe_path:
-                    pipe_path = pipe_path.replace(f"{project.folder}/", "")
-                    click.echo(FeedbackManager.info(message=f"✓ {pipe_path} created"))
+                pipe_path_str: Optional[str] = next((p for p in project_files if p.endswith(pipe_name + ".pipe")), None)
+                if pipe_path_str:
+                    pipe_path_str = pipe_path_str.replace(f"{project.folder}/", "")
+                    click.echo(FeedbackManager.info(message=f"✓ {pipe_path_str} created"))
 
             try:
                 for filename in project_files:
@@ -185,9 +190,17 @@ def rebuild_fixture(project: Project, tb_client: TinyB, fixture: str) -> None:
         click.echo(FeedbackManager.error_exception(error=e))
 
 
-def build_and_print_resource(tb_client: TinyB, filename: str):
+def build_and_print_resource(tb_client: TinyB, filename: str, diff: Optional[str] = None):
+    table_name = diff
     resource_path = Path(filename)
-    name = resource_path.stem
-    pipeline = name if filename.endswith(".pipe") else None
-    res = asyncio.run(tb_client.query(f"SELECT * FROM {name} FORMAT JSON", pipeline=pipeline))
-    print_table_formatted(res, name)
+    resource_name = resource_path.stem
+
+    pipeline = resource_name if filename.endswith(".pipe") else None
+
+    if not table_name:
+        table_name = resource_name
+
+    sql = f"SELECT * FROM {table_name} FORMAT JSON"
+
+    res = asyncio.run(tb_client.query(sql, pipeline=pipeline))
+    print_table_formatted(res, table_name)

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev43}/tinybird/tb/modules/build_client.py

@@ -166,7 +166,7 @@ def build_client(
     build_ok = asyncio.run(build_once(filenames))
 
     if watch:
-        shell = Shell(project=project, client=tb_client)
+        shell = Shell(project=project, tb_client=tb_client)
         click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
         watcher_thread = threading.Thread(
             target=watch_files, args=(filenames, process, shell, project, build_ok), daemon=True

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev43}/tinybird/tb/modules/cicd.py

@@ -39,7 +39,7 @@ jobs:
         working-directory: '{{ data_project_dir }}'
     services:
       tinybird:
-        image: tinybirdco/tinybird-local:latest
+        image: tinybirdco/tinybird-local:beta
         ports:
           - 80:80
     steps:
@@ -83,7 +83,7 @@ tinybird_ci_workflow:
     - tb build
     - tb test run
   services:
-    - name: tinybirdco/tinybird-local:latest
+    - name: tinybirdco/tinybird-local:beta
       alias: tinybird-local
 """
 

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev43}/tinybird/tb/modules/cli.py

@@ -66,15 +66,17 @@ VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
 @click.option("--host", help="Use custom host, defaults to TB_HOST envvar, then to https://api.tinybird.co")
 @click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens")
 @click.option("--prod/--local", is_flag=True, default=False, help="Run against production or local")
-@click.option("--folder", type=str, default=os.getcwd(), help="Folder where files will be placed")
+@click.option("--folder", type=str, help="Folder where files will be placed")
 @click.version_option(version=VERSION)
 @click.pass_context
 @coro
-async def cli(ctx: Context, debug: bool, token: str, host: str, show_tokens: bool, prod: bool, folder: str) -> None:
+async def cli(
+    ctx: Context, debug: bool, token: str, host: str, show_tokens: bool, prod: bool, folder: Optional[str]
+) -> None:
     """
     Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
     """
-    project = Project(folder=folder)
+    project = Project(folder=folder or os.getcwd())
     # We need to unpatch for our tests not to break
     if show_tokens or not prod or ctx.invoked_subcommand == "build":
         __unpatch_click_output()
@@ -87,7 +89,7 @@ async def cli(ctx: Context, debug: bool, token: str, host: str, show_tokens: boo
     if debug:
         logging.basicConfig(level=logging.DEBUG)
 
-    config_temp = CLIConfig.get_project_config(project.path)
+    config_temp = CLIConfig.get_project_config(str(project.path))
     if token:
         config_temp.set_token(token)
     if host:

tinybird-0.0.1.dev43/tinybird/tb/modules/copy.py

@@ -0,0 +1,159 @@
+# This is a command file for our CLI. Please keep it clean.
+#
+# - If it makes sense and only when strictly necessary, you can create utility functions in this file.
+# - But please, **do not** interleave utility functions and command definitions.
+
+import json
+import re
+from typing import Optional, Tuple
+
+import click
+from click import Context
+
+from tinybird.client import AuthNoTokenException, TinyB
+from tinybird.tb.modules.cli import cli
+from tinybird.tb.modules.common import coro, echo_safe_humanfriendly_tables_format_smart_table, wait_job
+from tinybird.tb.modules.datafile.common import get_name_version
+from tinybird.tb.modules.exceptions import CLIPipeException
+from tinybird.tb.modules.feedback_manager import FeedbackManager
+
+
+@cli.group(hidden=True)
+@click.pass_context
+def copy(ctx):
+    """Copy pipe commands"""
+
+
+@copy.command(name="ls")
+@click.option("--match", default=None, help="Retrieve any resourcing matching the pattern. eg --match _test")
+@click.option(
+    "--format",
+    "format_",
+    type=click.Choice(["json"], case_sensitive=False),
+    default=None,
+    help="Force a type of the output",
+)
+@click.pass_context
+@coro
+async def copy_ls(ctx: Context, match: str, format_: str):
+    """List copy pipes"""
+
+    client: TinyB = ctx.ensure_object(dict)["client"]
+    pipes = await client.pipes(dependencies=False, node_attrs="name", attrs="name,updated_at,type")
+    copies = [p for p in pipes if p.get("type") == "copy"]
+    copies = sorted(copies, key=lambda p: p["updated_at"])
+    columns = ["name", "updated at", "nodes"]
+    table_human_readable = []
+    table_machine_readable = []
+    pattern = re.compile(match) if match else None
+    for t in copies:
+        tk = get_name_version(t["name"])
+        if pattern and not pattern.search(tk["name"]):
+            continue
+        table_human_readable.append((tk["name"], t["updated_at"][:-7], len(t["nodes"])))
+        table_machine_readable.append(
+            {
+                "name": tk["name"],
+                "updated at": t["updated_at"][:-7],
+                "nodes": len(t["nodes"]),
+            }
+        )
+
+    if not format_:
+        click.echo(FeedbackManager.info_pipes())
+        echo_safe_humanfriendly_tables_format_smart_table(table_human_readable, column_names=columns)
+        click.echo("\n")
+    elif format_ == "json":
+        click.echo(json.dumps({"pipes": table_machine_readable}, indent=2))
+    else:
+        raise CLIPipeException(FeedbackManager.error_pipe_ls_type())
+
+
+@copy.command(name="run", short_help="Run an on-demand copy job")
+@click.argument("pipe_name_or_id")
+@click.option("--wait", is_flag=True, default=False, help="Wait for the copy job to finish")
+@click.option(
+    "--mode", type=click.Choice(["append", "replace"], case_sensitive=True), default=None, help="Copy strategy"
+)
+@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
+@click.option(
+    "--param",
+    nargs=1,
+    type=str,
+    multiple=True,
+    default=None,
+    help="Key and value of the params you want the Copy pipe to be called with. For example: tb pipe copy run <my_copy_pipe> --param foo=bar",
+)
+@click.pass_context
+@coro
+async def copy_run(
+    ctx: click.Context, pipe_name_or_id: str, wait: bool, mode: str, yes: bool, param: Optional[Tuple[str]]
+):
+    """Run an on-demand copy job"""
+
+    params = dict(key_value.split("=") for key_value in param) if param else {}
+
+    if yes or click.confirm(FeedbackManager.warning_confirm_copy_pipe(pipe=pipe_name_or_id)):
+        click.echo(FeedbackManager.info_copy_job_running(pipe=pipe_name_or_id))
+        client: TinyB = ctx.ensure_object(dict)["client"]
+
+        try:
+            response = await client.pipe_run_copy(pipe_name_or_id, params, mode)
+
+            job_id = response["job"]["job_id"]
+            job_url = response["job"]["job_url"]
+            target_datasource_id = response["tags"]["copy_target_datasource"]
+            target_datasource = await client.get_datasource(target_datasource_id)
+            target_datasource_name = target_datasource["name"]
+            click.echo(
+                FeedbackManager.success_copy_job_created(target_datasource=target_datasource_name, job_url=job_url)
+            )
+
+            if wait:
+                await wait_job(client, job_id, job_url, "** Copying data")
+                click.echo(FeedbackManager.success_data_copied_to_ds(target_datasource=target_datasource_name))
+
+        except AuthNoTokenException:
+            raise
+        except Exception as e:
+            raise CLIPipeException(FeedbackManager.error_creating_copy_job(error=e))
+
+
+@copy.command(name="resume", short_help="Resume a paused copy pipe")
+@click.argument("pipe_name_or_id")
+@click.pass_context
+@coro
+async def copy_resume(ctx: click.Context, pipe_name_or_id: str):
+    """Resume a paused copy pipe"""
+
+    click.echo(FeedbackManager.info_copy_pipe_resuming(pipe=pipe_name_or_id))
+    client: TinyB = ctx.ensure_object(dict)["client"]
+
+    try:
+        await client.pipe_resume_copy(pipe_name_or_id)
+        click.echo(FeedbackManager.success_copy_pipe_resumed(pipe=pipe_name_or_id))
+
+    except AuthNoTokenException:
+        raise
+    except Exception as e:
+        raise CLIPipeException(FeedbackManager.error_resuming_copy_pipe(error=e))
+
+
+@copy.command(name="pause", short_help="Pause a running copy pipe")
+@click.argument("pipe_name_or_id")
+@click.pass_context
+@coro
+async def copy_pause(ctx: click.Context, pipe_name_or_id: str):
+    """Pause a running copy pipe"""
+
+    click.echo(FeedbackManager.info_copy_pipe_pausing(pipe=pipe_name_or_id))
+    client: TinyB = ctx.ensure_object(dict)["client"]
+
+    try:
+        await client.pipe_pause_copy(pipe_name_or_id)
+        click.echo(FeedbackManager.success_copy_pipe_paused(pipe=pipe_name_or_id))
+
+    except AuthNoTokenException:
+        raise
+    except Exception as e:
+        raise CLIPipeException(FeedbackManager.error_pausing_copy_pipe(error=e))

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev43}/tinybird/tb/modules/datafile/build.py

@@ -707,6 +707,7 @@ async def process(
 ):
     name, kind = filename.rsplit(".", 1)
     warnings = []
+    embedded_datasources = {} if embedded_datasources is None else embedded_datasources
 
     try:
         res = await process_file(
@@ -806,31 +807,45 @@ async def get_processed(
     to_run: Optional[Dict[str, Any]] = None,
     vendor_paths: Optional[List[Tuple[str, str]]] = None,
     processed: Optional[Set[str]] = None,
-    tb_client: TinyB = None,
+    tb_client: Optional[TinyB] = None,
     skip_connectors: bool = False,
     current_ws: Optional[Dict[str, Any]] = None,
     fork_downstream: Optional[bool] = False,
     is_internal: Optional[bool] = False,
     dir_path: Optional[str] = None,
-    embedded_datasources: Optional[Dict[str, Any]] = None,
+    embedded_datasources: Optional[Dict[str, Dict[str, Any]]] = None,
 ):
-    if deps is None:
-        deps = []
-    if dep_map is None:
-        dep_map = {}
-    if to_run is None:
-        to_run = {}
-    if processed is None:
-        processed = set()
+    # Initialize with proper type annotations
+    deps_list: List[str] = [] if deps is None else deps
+    dep_map_dict: Dict[str, Any] = {} if dep_map is None else dep_map
+    to_run_dict: Dict[str, Any] = {} if to_run is None else to_run
+    processed_set: Set[str] = set() if processed is None else processed
+    embedded_ds: Dict[str, Dict[str, Any]] = {} if embedded_datasources is None else embedded_datasources
 
     for filename in filenames:
         # just process changed filenames (tb deploy and --only-changes)
-        if changed:
+        if changed is not None:
             resource = Path(filename).resolve().stem
             if resource in changed and (not changed[resource] or changed[resource] in ["shared", "remote"]):
                 continue
         if os.path.isdir(filename):
-            await get_processed(filenames=get_project_filenames(filename))
+            await get_processed(
+                filenames=get_project_filenames(filename),
+                changed=changed,
+                verbose=verbose,
+                deps=deps_list,
+                dep_map=dep_map_dict,
+                to_run=to_run_dict,
+                vendor_paths=vendor_paths,
+                processed=processed_set,
+                tb_client=tb_client,
+                skip_connectors=skip_connectors,
+                current_ws=current_ws,
+                fork_downstream=fork_downstream,
+                is_internal=is_internal,
+                dir_path=dir_path,
+                embedded_datasources=embedded_ds,
+            )
         else:
             if verbose:
                 click.echo(FeedbackManager.info_processing_file(filename=filename))
@@ -838,12 +853,15 @@ async def get_processed(
             if ".incl" in filename:
                 click.echo(FeedbackManager.warning_skipping_include_file(file=filename))
 
+            if tb_client is None:
+                raise ValueError("tb_client cannot be None")
+
             name, warnings = await process(
                 filename=filename,
                 tb_client=tb_client,
-                deps=deps,
-                dep_map=dep_map,
-                to_run=to_run,
+                deps=deps_list,
+                dep_map=dep_map_dict,
+                to_run=to_run_dict,
                 vendor_paths=vendor_paths,
                 skip_connectors=skip_connectors,
                 current_ws=current_ws,
@@ -852,9 +870,9 @@ async def get_processed(
                 is_internal=is_internal,
                 dir_path=dir_path,
                 verbose=verbose,
-                embedded_datasources=embedded_datasources,
+                embedded_datasources=embedded_ds,
             )
-            processed.add(name)
+            processed_set.add(name)
 
             if verbose:
                 if len(warnings) == 1:
@@ -890,7 +908,7 @@ async def build_graph(
     to_run: Dict[str, Any] = {}
     deps: List[str] = []
     dep_map: Dict[str, Any] = {}
-    embedded_datasources = {}
+    embedded_datasources: Dict[str, Dict[str, Any]] = {}
 
     # These dictionaries are used to store all the resources and there dependencies for the whole project
     # This is used for the downstream dependency graph
@@ -919,17 +937,18 @@ async def build_graph(
         all_dep_map = all_dependencies_graph.dep_map
         all_resources = all_dependencies_graph.to_run
 
-    processed = set()
+    processed: Set[str] = set()
 
     await get_processed(
         filenames=filenames,
-        tb_client=tb_client,
         changed=changed,
+        verbose=verbose,
         deps=deps,
         dep_map=dep_map,
         to_run=to_run,
         vendor_paths=vendor_paths,
         processed=processed,
+        tb_client=tb_client,
         skip_connectors=skip_connectors,
         current_ws=current_ws,
         fork_downstream=fork_downstream,
@@ -1198,18 +1217,22 @@ async def process_file(
                     raise Exception(f"Invalid import schedule: '{cron}'. Valid values are: {valid_values}")
 
                 if cron == ON_DEMAND_CRON:
+                    if import_params is None:
+                        import_params = {}
                     import_params["import_schedule"] = ON_DEMAND_CRON_EXPECTED_BY_THE_API
+
                 if cron == AUTO_CRON:
                     period: int = DEFAULT_CRON_PERIOD
 
-                    if current_ws:
+                    if current_ws is not None:
                         workspaces = (await tb_client.user_workspaces()).get("workspaces", [])
                         workspace_rate_limits: Dict[str, Dict[str, int]] = next(
                             (w.get("rate_limits", {}) for w in workspaces if w["id"] == current_ws["id"]), {}
                         )
-                        period = workspace_rate_limits.get("api_datasources_create_append_replace", {}).get(
-                            "period", DEFAULT_CRON_PERIOD
-                        )
+                        if workspace_rate_limits:
+                            rate_limit_config = workspace_rate_limits.get("api_datasources_create_append_replace", {})
+                            if rate_limit_config:
+                                period = rate_limit_config.get("period", DEFAULT_CRON_PERIOD)
 
                     def seconds_to_cron_expression(seconds: int) -> str:
                         minutes = seconds // 60
@@ -1223,10 +1246,13 @@ async def process_file(
                             return f"*/{minutes} * * * *"
                         return f"*/{seconds} * * * *"
 
+                    if import_params is None:
+                        import_params = {}
                     import_params["import_schedule"] = seconds_to_cron_expression(period)
 
-            # Include all import_ parameters in the datasource params
-            params.update(import_params)
+                # Include all import_ parameters in the datasource params
+                if import_params is not None:
+                    params.update(import_params)
 
             # Substitute the import parameters with the ones used by the
             # import API: