tinybird 0.0.1.dev6__py3-none-any.whl → 0.0.1.dev8__py3-none-any.whl

tinybird/tb/modules/datafile/diff.py

@@ -0,0 +1,197 @@
+import difflib
+import os
+from os import getcwd
+from pathlib import Path
+from typing import Any, Dict, Generator, Iterable, List, Optional
+
+try:
+    from colorama import Back, Fore, Style, init
+
+    init()
+except ImportError:  # fallback so that the imported classes always exist
+
+    class ColorFallback:
+        def __getattr__(self, name):
+            return ""
+
+    Fore = Back = Style = ColorFallback()
+
+
+import shutil
+import sys
+
+import click
+
+from tinybird.client import TinyB
+from tinybird.feedback_manager import FeedbackManager
+from tinybird.sql_template_fmt import DEFAULT_FMT_LINE_LENGTH
+from tinybird.tb.modules.datafile.common import get_name_version, get_project_filenames, is_file_a_datasource, peek
+from tinybird.tb.modules.datafile.format_datasource import format_datasource
+from tinybird.tb.modules.datafile.format_pipe import format_pipe
+from tinybird.tb.modules.datafile.pull import folder_pull
+
+
+async def diff_files(
+    from_file: str,
+    to_file: str,
+    from_file_suffix: str = "[remote]",
+    to_file_suffix: str = "[local]",
+    with_format: bool = True,
+    with_color: bool = False,
+    client: Optional[TinyB] = None,
+    for_deploy: bool = False,
+):
+    def file_lines(filename):
+        with open(filename) as file:
+            return file.readlines()
+
+    async def parse(filename, with_format=True, unroll_includes=False):
+        extensions = Path(filename).suffixes
+        lines = None
+        if is_file_a_datasource(filename):
+            lines = (
+                await format_datasource(
+                    filename,
+                    unroll_includes=unroll_includes,
+                    for_diff=True,
+                    client=client,
+                    replace_includes=True,
+                    for_deploy_diff=for_deploy,
+                )
+                if with_format
+                else file_lines(filename)
+            )
+        elif (".pipe" in extensions) or (".incl" in extensions):
+            lines = (
+                await format_pipe(
+                    filename,
+                    DEFAULT_FMT_LINE_LENGTH,
+                    unroll_includes=unroll_includes,
+                    replace_includes=True,
+                    for_deploy_diff=for_deploy,
+                )
+                if with_format
+                else file_lines(filename)
+            )
+        else:
+            click.echo(f"Unsupported file type: {filename}")
+        if lines:
+            return [f"{l}\n" for l in lines.split("\n")] if with_format else lines  # noqa: E741
+
+    try:
+        lines1 = await parse(from_file, with_format)
+        lines2 = await parse(to_file, with_format, unroll_includes=True)
+    except FileNotFoundError as e:
+        filename = os.path.basename(str(e)).strip("'")
+        raise click.ClickException(FeedbackManager.error_diff_file(filename=filename))
+
+    if not lines1 or not lines2:
+        return
+
+    diff = difflib.unified_diff(
+        lines1, lines2, fromfile=f"{Path(from_file).name} {from_file_suffix}", tofile=f"{to_file} {to_file_suffix}"
+    )
+
+    if with_color:
+        diff = color_diff(diff)
+
+    return diff
+
+
+async def diff_command(
+    filenames: Optional[List[str]],
+    fmt: bool,
+    client: TinyB,
+    no_color: Optional[bool] = False,
+    with_print: Optional[bool] = True,
+    verbose: Optional[bool] = None,
+    clean_up: Optional[bool] = False,
+    progress_bar: bool = False,
+    for_deploy: bool = False,
+):
+    def is_shared_datasource(name):
+        return "." in name
+
+    with_explicit_filenames = filenames
+    verbose = True if verbose is None else verbose
+
+    target_dir = getcwd() + os.path.sep + ".diff_tmp"
+    Path(target_dir).mkdir(parents=True, exist_ok=True)
+
+    if filenames:
+        if len(filenames) == 1:
+            filenames = [filenames[0], *get_project_filenames(filenames[0])]
+        await folder_pull(client, target_dir, False, None, True, verbose=False)
+    else:
+        filenames = get_project_filenames(".")
+        if verbose:
+            click.echo("Saving remote resources in .diff_tmp folder.\n")
+        await folder_pull(client, target_dir, False, None, True, verbose=verbose, progress_bar=progress_bar)
+
+    remote_datasources: List[Dict[str, Any]] = await client.datasources()
+    remote_pipes: List[Dict[str, Any]] = await client.pipes()
+
+    local_resources = {
+        Path(file).resolve().stem: file
+        for file in filenames
+        if (".datasource" in file or ".pipe" in file) and ".incl" not in file
+    }
+
+    changed = {}
+    for resource in remote_datasources + remote_pipes:
+        properties: Dict[str, Any] = get_name_version(resource["name"])
+        name = properties.get("name", None)
+        if name:
+            (rfilename, file) = next(
+                ((rfilename, file) for (rfilename, file) in local_resources.items() if name == rfilename),
+                ("", None),
+            )
+            if not file:
+                if not with_explicit_filenames:
+                    if with_print:
+                        click.echo(f"{resource['name']} only exists remotely\n")
+                    if is_shared_datasource(resource["name"]):
+                        changed[resource["name"]] = "shared"
+                    else:
+                        changed[resource["name"]] = "remote"
+                continue
+
+            suffix = ".datasource" if ".datasource" in file else ".pipe"
+            target = target_dir + os.path.sep + rfilename + suffix
+
+            diff_lines = await diff_files(
+                target, file, with_format=fmt, with_color=(not no_color), client=client, for_deploy=for_deploy
+            )
+            not_empty, diff_lines = peek(diff_lines)
+            changed[rfilename] = not_empty
+            if not_empty and with_print:
+                sys.stdout.writelines(diff_lines)
+                click.echo("")
+
+    for rfilename, _ in local_resources.items():
+        if rfilename not in changed:
+            for resource in remote_datasources + remote_pipes:
+                properties = get_name_version(resource["name"])
+                name = properties.get("name", None)
+                if name and name == rfilename:
+                    break
+
+                if with_print and rfilename not in changed:
+                    click.echo(f"{rfilename} only exists locally\n")
+                changed[rfilename] = "local"
+    if clean_up:
+        shutil.rmtree(target_dir)
+
+    return changed
+
+
+def color_diff(diff: Iterable[str]) -> Generator[str, Any, None]:
+    for line in diff:
+        if line.startswith("+"):
+            yield Fore.GREEN + line + Fore.RESET
+        elif line.startswith("-"):
+            yield Fore.RED + line + Fore.RESET
+        elif line.startswith("^"):
+            yield Fore.BLUE + line + Fore.RESET
+        else:
+            yield line

tinybird/tb/modules/datafile/exceptions.py

@@ -0,0 +1,23 @@
+import click
+
+
+class AlreadyExistsException(click.ClickException):
+    pass
+
+
+class ParseException(Exception):
+    def __init__(self, err: str, lineno: int = -1):
+        self.lineno: int = lineno
+        super().__init__(err)
+
+
+class IncludeFileNotFoundException(Exception):
+    def __init__(self, err: str, lineno: int = -1):
+        self.lineno: int = lineno
+        super().__init__(err)
+
+
+class ValidationException(Exception):
+    def __init__(self, err: str, lineno: int = -1) -> None:
+        self.lineno: int = lineno
+        super().__init__(err)

tinybird/tb/modules/datafile/format_common.py

@@ -0,0 +1,66 @@
+from typing import Any, List
+
+from tinybird.tb.modules.datafile.common import Datafile
+
+DATAFILE_NEW_LINE = "\n"
+DATAFILE_INDENT = " " * 4
+
+
+def format_description(file_parts: List[str], doc: Any) -> List[str]:
+    description = doc.description if doc.description is not None else ""
+    if description:
+        file_parts.append("DESCRIPTION >")
+        file_parts.append(DATAFILE_NEW_LINE)
+        [
+            file_parts.append(f"{DATAFILE_INDENT}{d.strip()}\n")  # type: ignore
+            for d in description.split(DATAFILE_NEW_LINE)
+            if d.strip()
+        ]
+        file_parts.append(DATAFILE_NEW_LINE)
+    return file_parts
+
+
+def format_sources(file_parts: List[str], doc: Datafile) -> List[str]:
+    for source in doc.sources:
+        file_parts.append(f"SOURCE {source}")
+        file_parts.append(DATAFILE_NEW_LINE)
+    return file_parts
+
+
+def format_maintainer(file_parts: List[str], doc: Datafile) -> List[str]:
+    maintainer = doc.maintainer if doc.maintainer is not None else ""
+    if maintainer:
+        file_parts.append(f"MAINTAINER {maintainer}")
+        file_parts.append(DATAFILE_NEW_LINE)
+        file_parts.append(DATAFILE_NEW_LINE)
+    return file_parts
+
+
+def format_tokens(file_parts: List[str], doc: Datafile) -> List[str]:
+    for token in doc.tokens:
+        file_parts.append(f'TOKEN "{token["token_name"]}" {token["permissions"]}')
+        file_parts.append(DATAFILE_NEW_LINE)
+    if len(doc.tokens):
+        file_parts.append(DATAFILE_NEW_LINE)
+    return file_parts
+
+
+def format_tags(file_parts: List[str], doc: Datafile) -> List[str]:
+    if doc.filtering_tags:
+        file_parts.append(f'TAGS "{", ".join(doc.filtering_tags)}"')
+        file_parts.append(DATAFILE_NEW_LINE)
+        file_parts.append(DATAFILE_NEW_LINE)
+    return file_parts
+
+
+def format_include(file_parts: List[str], doc: Datafile, unroll_includes: bool = False) -> List[str]:
+    if unroll_includes:
+        return file_parts
+
+    assert doc.raw is not None
+
+    include = [line for line in doc.raw if "INCLUDE" in line and ".incl" in line]
+    if len(include):
+        file_parts.append(include[0])
+        file_parts.append(DATAFILE_NEW_LINE)
+    return file_parts

tinybird/tb/modules/datafile/format_datasource.py

@@ -0,0 +1,160 @@
+from typing import Any, Dict, List, Optional
+
+from tinybird.client import TinyB
+from tinybird.sql import schema_to_sql_columns
+from tinybird.tb.modules.datafile.common import Datafile
+from tinybird.tb.modules.datafile.format_common import (
+    DATAFILE_INDENT,
+    DATAFILE_NEW_LINE,
+    format_description,
+    format_include,
+    format_maintainer,
+    format_sources,
+    format_tags,
+    format_tokens,
+)
+from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
+
+
+async def format_datasource(
+    filename: str,
+    unroll_includes: bool = False,
+    for_diff: bool = False,
+    client: Optional[TinyB] = None,
+    replace_includes: bool = False,
+    datafile: Optional[Datafile] = None,
+    for_deploy_diff: bool = False,
+    skip_eval: bool = False,
+    content: Optional[str] = None,
+) -> str:
+    if datafile:
+        doc = datafile
+    else:
+        doc = parse_datasource(filename, replace_includes=replace_includes, skip_eval=skip_eval, content=content)
+
+    file_parts: List[str] = []
+    if for_diff:
+        is_kafka = "kafka_connection_name" in doc.nodes[0]
+        if is_kafka:
+            kafka_metadata_columns = [
+                "__value",
+                "__headers",
+                "__topic",
+                "__partition",
+                "__offset",
+                "__timestamp",
+                "__key",
+            ]
+            columns = [c for c in doc.nodes[0]["columns"] if c["name"] not in kafka_metadata_columns]
+            doc.nodes[0].update(
+                {
+                    "columns": columns,
+                }
+            )
+        if for_deploy_diff:
+            format_description(file_parts, doc)
+        format_tags(file_parts, doc)
+        format_schema(file_parts, doc.nodes[0])
+        format_indices(file_parts, doc.nodes[0])
+        await format_engine(file_parts, doc.nodes[0], only_ttl=bool(not for_deploy_diff), client=client)
+        if for_deploy_diff:
+            format_import_settings(file_parts, doc.nodes[0])
+        format_shared_with(file_parts, doc)
+    else:
+        format_sources(file_parts, doc)
+        format_maintainer(file_parts, doc)
+        format_description(file_parts, doc)
+        format_tokens(file_parts, doc)
+        format_tags(file_parts, doc)
+        format_schema(file_parts, doc.nodes[0])
+        format_indices(file_parts, doc.nodes[0])
+        await format_engine(file_parts, doc.nodes[0])
+        format_include(file_parts, doc, unroll_includes=unroll_includes)
+        format_data_connector(file_parts, doc.nodes[0])
+        format_import_settings(file_parts, doc.nodes[0])
+        format_shared_with(file_parts, doc)
+    result = "".join(file_parts)
+    result = result.rstrip("\n") + "\n"
+    return result
+
+
+def format_schema(file_parts: List[str], node: Dict[str, Any]) -> List[str]:
+    if node.get("schema"):
+        file_parts.append("SCHEMA >")
+        file_parts.append(DATAFILE_NEW_LINE)
+        columns = schema_to_sql_columns(node["columns"])
+        file_parts.append(f",{DATAFILE_NEW_LINE}".join(map(lambda x: f"    {x}", columns)))
+        file_parts.append(DATAFILE_NEW_LINE)
+        file_parts.append(DATAFILE_NEW_LINE)
+
+    return file_parts
+
+
+def format_indices(file_parts: List[str], node: Dict[str, Any]) -> List[str]:
+    if node.get("indexes"):
+        indexes = node["indexes"]
+        file_parts.append("INDEXES >")
+        file_parts.append(DATAFILE_NEW_LINE)
+        file_parts.append(f"{DATAFILE_NEW_LINE}".join(map(lambda index: f"    {index.to_datafile()}", indexes)))
+        file_parts.append(DATAFILE_NEW_LINE)
+        file_parts.append(DATAFILE_NEW_LINE)
+
+    return file_parts
+
+
+def format_data_connector(file_parts: List[str], node: Dict[str, Any]) -> List[str]:
+    ll = len(file_parts)
+    quotes = "''"
+    [file_parts.append(f"{k.upper()} {v or quotes}{DATAFILE_NEW_LINE}") for k, v in node.items() if "kafka" in k]  # type: ignore
+    if ll < len(file_parts):
+        file_parts.append(DATAFILE_NEW_LINE)
+    return file_parts
+
+
+def format_import_settings(file_parts: List[str], node: Dict[str, Any]) -> List[str]:
+    ll = len(file_parts)
+    [file_parts.append(f"{k.upper()} {v}{DATAFILE_NEW_LINE}") for k, v in node.items() if "import_" in k]  # type: ignore
+    if ll < len(file_parts):
+        file_parts.append(DATAFILE_NEW_LINE)
+    return file_parts
+
+
+def format_shared_with(file_parts: List[str], doc: Datafile) -> List[str]:
+    if doc.shared_with:
+        file_parts.append("SHARED_WITH >")
+        file_parts.append(DATAFILE_NEW_LINE)
+        file_parts.append("\n".join([f"{DATAFILE_INDENT}{workspace_name}" for workspace_name in doc.shared_with]))
+    return file_parts
+
+
+async def format_engine(
+    file_parts: List[str], node: Dict[str, Any], only_ttl: bool = False, client: Optional[TinyB] = None
+) -> List[str]:
+    if only_ttl:
+        if node.get("engine", None):
+            for arg in sorted(node["engine"].get("args", [])):
+                if arg[0].upper() == "TTL":
+                    elem = ", ".join([x.strip() for x in arg[1].split(",")])
+                    try:
+                        if client:
+                            ttl_sql = await client.sql_get_format(f"select {elem}", with_clickhouse_format=True)
+                            formatted_ttl = ttl_sql[7:]
+                        else:
+                            formatted_ttl = elem
+                    except Exception:
+                        formatted_ttl = elem
+                    file_parts.append(f"ENGINE_{arg[0].upper()} {formatted_ttl}")
+                    file_parts.append(DATAFILE_NEW_LINE)
+            file_parts.append(DATAFILE_NEW_LINE)
+        return file_parts
+    else:
+        if node.get("engine", None):
+            empty = '""'
+            file_parts.append(f'ENGINE {node["engine"]["type"]}' if node.get("engine", {}).get("type") else empty)
+            file_parts.append(DATAFILE_NEW_LINE)
+            for arg in sorted(node["engine"].get("args", [])):
+                elem = ", ".join([x.strip() for x in arg[1].split(",")])
+                file_parts.append(f"ENGINE_{arg[0].upper()} {elem if elem else empty}")
+                file_parts.append(DATAFILE_NEW_LINE)
+            file_parts.append(DATAFILE_NEW_LINE)
+        return file_parts

tinybird/tb/modules/datafile/format_pipe.py

@@ -0,0 +1,195 @@
+import os
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from tinybird.sql_template_fmt import DEFAULT_FMT_LINE_LENGTH, format_sql_template
+from tinybird.tb.modules.datafile.common import (
+    ON_DEMAND,
+    CopyParameters,
+    Datafile,
+    ExportReplacements,
+    PipeNodeTypes,
+    _unquote,
+    eval_var,
+)
+from tinybird.tb.modules.datafile.format_common import (
+    DATAFILE_INDENT,
+    DATAFILE_NEW_LINE,
+    format_description,
+    format_maintainer,
+    format_sources,
+    format_tags,
+    format_tokens,
+)
+from tinybird.tb.modules.datafile.format_datasource import format_engine
+from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
+
+
+def format_node_sql(
+    file_parts: List[str], node: Dict[str, Any], line_length: Optional[int] = None, lower_keywords: bool = False
+) -> List[str]:
+    file_parts.append("SQL >")
+    file_parts.append(DATAFILE_NEW_LINE)
+    file_parts.append(format_sql(node["sql"], DATAFILE_INDENT, line_length=line_length, lower_keywords=lower_keywords))
+    file_parts.append(DATAFILE_NEW_LINE)
+    file_parts.append(DATAFILE_NEW_LINE)
+    return file_parts
+
+
+async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[str]:
+    node_type = node.get("type", "").lower()
+    node_type_upper = f"TYPE {node_type.upper()}"
+    # Materialized pipe
+    if node_type == PipeNodeTypes.MATERIALIZED:
+        file_parts.append(node_type_upper)
+        file_parts.append(DATAFILE_NEW_LINE)
+        file_parts.append(f'DATASOURCE {node["datasource"]}')
+        file_parts.append(DATAFILE_NEW_LINE)
+        await format_engine(file_parts, node)
+
+    # Copy pipe
+    if node_type == PipeNodeTypes.COPY:
+        file_parts.append(node_type_upper)
+        file_parts.append(DATAFILE_NEW_LINE)
+        file_parts.append(f'TARGET_DATASOURCE {node["target_datasource"]}')
+        if node.get("mode"):
+            file_parts.append(DATAFILE_NEW_LINE)
+            file_parts.append(f'COPY_MODE {node.get("mode")}')
+
+        if node.get(CopyParameters.COPY_SCHEDULE):
+            is_ondemand = node[CopyParameters.COPY_SCHEDULE].lower() == ON_DEMAND
+            file_parts.append(DATAFILE_NEW_LINE)
+            file_parts.append(
+                f"{CopyParameters.COPY_SCHEDULE.upper()} {ON_DEMAND if is_ondemand else node[CopyParameters.COPY_SCHEDULE]}"
+            )
+        else:
+            file_parts.append(DATAFILE_NEW_LINE)
+            file_parts.append(f"{CopyParameters.COPY_SCHEDULE.upper()} {ON_DEMAND}")
+        file_parts.append(DATAFILE_NEW_LINE)
+
+    # Sink or Stream pipe
+    if ExportReplacements.is_export_node(node):
+        file_parts.append(node_type_upper)
+        export_params = ExportReplacements.get_params_from_datafile(node)
+        file_parts.append(DATAFILE_NEW_LINE)
+        for param, value in export_params.items():
+            if param == "schedule_cron" and not value:
+                value = ON_DEMAND
+            datafile_key = ExportReplacements.get_datafile_key(param, node)
+            if datafile_key and value:
+                file_parts.append(f"{datafile_key} {value}")
+                file_parts.append(DATAFILE_NEW_LINE)
+
+    return file_parts
+
+
+def format_pipe_include(file_parts: List[str], node: Dict[str, Any], includes: Dict[str, Any]) -> List[str]:
+    if includes:
+        for k, v in includes.copy().items():
+            if node["name"] in v:
+                file_parts.append(f"INCLUDE {k}")
+                file_parts.append(DATAFILE_NEW_LINE)
+                file_parts.append(DATAFILE_NEW_LINE)
+                del includes[k]
+    return file_parts
+
+
+def format_sql(sql: str, DATAFILE_INDENT: str, line_length: Optional[int] = None, lower_keywords: bool = False) -> str:
+    sql = format_sql_template(sql.strip(), line_length=line_length, lower_keywords=lower_keywords)
+    return "\n".join([f"{DATAFILE_INDENT}{part}" for part in sql.split("\n") if len(part.strip())])
+
+
+async def format_node(
+    file_parts: List[str],
+    node: Dict[str, Any],
+    includes: Dict[str, Any],
+    line_length: Optional[int] = None,
+    unroll_includes: bool = False,
+    lower_keywords: bool = False,
+) -> None:
+    if not unroll_includes:
+        format_pipe_include(file_parts, node, includes)
+    item = [k for k, _ in includes.items() if node["name"].strip() in k]
+    if item and not unroll_includes:
+        return
+
+    file_parts.append(f'NODE {node["name"].strip()}')
+    file_parts.append(DATAFILE_NEW_LINE)
+
+    from collections import namedtuple
+
+    Doc = namedtuple("Doc", ["description"])
+    format_description(file_parts, Doc(node.get("description", "")))
+    format_node_sql(file_parts, node, line_length=line_length, lower_keywords=lower_keywords)
+    await format_node_type(file_parts, node)
+
+
+async def format_pipe(
+    filename: str,
+    line_length: Optional[int] = DEFAULT_FMT_LINE_LENGTH,
+    unroll_includes: bool = False,
+    replace_includes: bool = False,
+    datafile: Optional[Datafile] = None,
+    for_deploy_diff: bool = False,
+    skip_eval: bool = False,
+    content: Optional[str] = None,
+) -> str:
+    if datafile:
+        doc = datafile
+    else:
+        doc = parse_pipe(filename, replace_includes=replace_includes, skip_eval=skip_eval, content=content)
+
+    file_parts: List[str] = []
+    format_sources(file_parts, doc)
+    format_maintainer(file_parts, doc)
+    format_description(file_parts, doc)
+    format_tokens(file_parts, doc)
+    format_tags(file_parts, doc)
+    if doc.includes and not unroll_includes:
+        for k in doc.includes:
+            # We filter only the include files as we currently have 2 items for each include
+            # { 'include_file.incl': 'First node of the include" }
+            # { 'first node of the pipe after the include': }
+            if ".incl" not in k:
+                continue
+
+            # We get all the nodes inside the include and remove them from the unrolled pipe as we want things unrolled
+            include_parameters = _unquote(k)
+
+            # If they use an include with parameters like `INCLUDE "xxx.incl" "GROUP_COL=path" "MATERIALIZED_VIEW=speed_insights_path_daily_mv"``
+            # We just want the file name to take nodes
+            include_file = include_parameters.split('"')[0]
+            include_file = (
+                Path(os.path.dirname(filename)) / eval_var(include_file)
+                if "." in include_file
+                else eval_var(include_file)
+            )
+            included_pipe = parse_pipe(include_file, skip_eval=skip_eval)
+            pipe_nodes = doc.nodes.copy()
+            for included_node in included_pipe.nodes.copy():
+                unrolled_included_node = next(
+                    (node for node in pipe_nodes if node["name"] == included_node["name"]), None
+                )
+                if unrolled_included_node:
+                    doc.nodes.remove(unrolled_included_node)
+    for node in doc.nodes:
+        await format_node(
+            file_parts,
+            node,
+            doc.includes,
+            line_length=line_length,
+            unroll_includes=unroll_includes,
+            lower_keywords=bool(for_deploy_diff),
+        )
+
+    if not unroll_includes:
+        for k, _ in doc.includes.items():
+            if ".incl" not in k:
+                continue
+            file_parts.append(f"INCLUDE {k}")
+            file_parts.append(DATAFILE_NEW_LINE)
+            file_parts.append(DATAFILE_NEW_LINE)
+
+    result = "".join(file_parts)
+    result = result.rstrip("\n") + "\n"
+    return result

tinybird/tb/modules/datafile/parse_datasource.py

@@ -0,0 +1,41 @@
+import os
+from typing import Optional
+
+import click
+
+from tinybird.feedback_manager import FeedbackManager
+from tinybird.tb.modules.datafile.common import (
+    Datafile,
+    format_filename,
+    parse,
+)
+from tinybird.tb.modules.datafile.exceptions import ParseException
+
+
+def parse_datasource(
+    filename: str,
+    replace_includes: bool = True,
+    content: Optional[str] = None,
+    skip_eval: bool = False,
+    hide_folders: bool = False,
+) -> Datafile:
+    basepath = ""
+    if not content:
+        with open(filename) as file:
+            s = file.read()
+        basepath = os.path.dirname(filename)
+    else:
+        s = content
+
+    filename = format_filename(filename, hide_folders)
+    try:
+        doc = parse(s, "default", basepath, replace_includes=replace_includes, skip_eval=skip_eval)
+    except ParseException as e:
+        raise click.ClickException(
+            FeedbackManager.error_parsing_file(filename=filename, lineno=e.lineno, error=e)
+        ) from None
+
+    if len(doc.nodes) > 1:
+        raise ValueError(f"{filename}: datasources can't have more than one node")
+
+    return doc