fal 0.9.2__py3-none-any.whl → 0.9.4__py3-none-any.whl

dbt/fal/adapters/teleport/impl.py

@@ -0,0 +1,103 @@
+import abc
+from typing import List, Union
+
+from dbt.exceptions import NotImplementedError
+from dbt.adapters.base.relation import BaseRelation
+from dbt.adapters.base.impl import BaseAdapter
+
+from dbt.fal.adapters.teleport.info import TeleportInfo
+
+
+class TeleportAdapter:
+    """The TeleportAdapter provides an abstract class for adapters usable with Teleport.
+
+    Adapters must implement the following methods and macros. Some of the
+    methods can be safely overridden as a noop, where it makes sense. Those
+    methods are marked with a (passable) in their docstrings. Check docstrings
+    for type information, etc.
+
+    To implement a macro, implement "${adapter_type}__${macro_name}" in the
+    adapter's internal project.
+
+    To invoke a method in an adapter macro, call it on the 'adapter' Jinja
+    object using dot syntax.
+
+    To invoke a method in model code, add the @available decorator atop a method
+    declaration. Methods are invoked as macros.
+
+    Methods:
+        - storage_formats
+        # - teleport_backends
+        - teleport_from_external_storage
+        - teleport_to_external_storage
+
+    Macros:
+        -
+
+    """
+
+    @classmethod
+    @abc.abstractmethod
+    def storage_formats(cls) -> List[str]:
+        """
+        List of formats this adapter handles. e.g. `['csv', 'parquet']`
+        """
+        raise NotImplementedError(
+            "`storage_formats` is not implemented for this adapter!"
+        )
+
+    @abc.abstractmethod
+    def teleport_from_external_storage(
+        self, relation: BaseRelation, data_path: str, teleport_info: TeleportInfo
+    ) -> None:
+        """
+        Localize data found in `data_path` into the a local table with the name given by `relation`.
+        Handle each possible format defined in `storage_formats` method.
+        """
+        raise NotImplementedError(
+            "`teleport_from_external_storage` is not implemented for this adapter!"
+        )
+
+    @abc.abstractmethod
+    def teleport_to_external_storage(
+        self, relation: BaseRelation, teleport_info: TeleportInfo
+    ) -> str:
+        """
+        Take local table `relation` and upload the data with Teleport. Return the `data_path` the data is uploaded to.
+        Handle each possible format defined in `storage_formats` method.
+        """
+        raise NotImplementedError(
+            "`teleport_to_external_storage` is not implemented for this adapter!"
+        )
+
+    @classmethod
+    def is_teleport_adapter(
+        cls, adapter: Union[BaseAdapter, "TeleportAdapter"]
+    ) -> bool:
+        methods = [
+            "storage_formats",
+            # "teleport_backends",
+            "teleport_from_external_storage",
+            "teleport_to_external_storage",
+        ]
+        return isinstance(adapter, TeleportAdapter) or all(
+            map(lambda m: hasattr(adapter, m), methods)
+        )
+
+    @classmethod
+    def find_format(
+        cls, target_adapter: "TeleportAdapter", ref_adapter: "TeleportAdapter"
+    ):
+        """
+        Find common format between target and ref adapter, giving priority to target list ordering.
+        """
+        target_formats = target_adapter.storage_formats()
+        ref_formats = ref_adapter.storage_formats()
+        for format in target_formats:
+            if format in ref_formats:
+                return format
+
+        raise RuntimeError(
+            f"No common format between {target_adapter.type()} and {ref_adapter.type()} "
+            f"—  {target_formats} | {ref_formats}"
+        )

dbt/fal/adapters/teleport/info.py

@@ -0,0 +1,73 @@
+from pathlib import Path
+from dataclasses import dataclass
+from typing import Any, Union
+
+from dbt.adapters.base.relation import BaseRelation
+from dbt.contracts.relation import ComponentName
+
+@dataclass
+class TeleportInfo:
+    format: str
+    credentials: Any
+
+    @classmethod
+    def relation_name(cls, relation: Union[str, BaseRelation]):
+        if isinstance(relation, str):
+            # TODO: should we check for quoting?
+            return relation
+        else:
+            path = relation.path
+            db = path.get_lowered_part(ComponentName.Database)
+            sc = path.get_lowered_part(ComponentName.Schema)
+            tb = path.get_lowered_part(ComponentName.Identifier)
+            return f"{db}.{sc}.{tb}"
+
+    def build_relation_path(self, relation: Union[str, BaseRelation]):
+        rel_name = TeleportInfo.relation_name(relation)
+        return rel_name + "." + self.format
+
+    def build_url(self, path: str) -> str:
+        raise NotImplemented
+
+@dataclass
+class LocalTeleportInfo(TeleportInfo):
+    base_dir: Path
+
+    def __init__(self, format: str, credentials: Any, base_dir: Union[str, Path]):
+        super().__init__(format, credentials)
+
+        self.base_dir = Path(base_dir)
+        self.base_dir.mkdir(exist_ok=True)
+        assert self.base_dir.is_dir(), "Local Teleport base_dir should be a directory"
+
+    def build_url(self, path: str):
+        return self.base_dir / path
+
+# TODO: How to do Teleport Configuration
+# Will each adapter have to implement a specific case for each teleport backend.
+# And how will we generalize these configurations?
+#
+# Each adapter will have a specific `teleport` property.
+#       my_db:
+#         type: duckdb
+#         path: ./local_file.db
+#         teleport:
+#             type: s3
+#             bucket: my_bucket
+#             access_key_id: ...
+#             secret_access_key: ...
+#       my_py:
+#         type: fal
+#         teleport:
+#             type: s3
+#             s3_bucket: my_bucket
+#             s3_access_key_id: ...
+#             s3_access_key: ...
+# Since all adapters could use different properties and methods to achieve this
+@dataclass
+class S3TeleportInfo(TeleportInfo):
+    bucket: str
+    inner_path: str
+
+    def build_url(self, path: str):
+        return f's3://{self.bucket}/{self.inner_path}/{path}'

dbt/include/fal/__init__.py

@@ -0,0 +1,3 @@
+import os
+
+PACKAGE_PATH = os.path.dirname(__file__)

dbt/include/fal/dbt_project.yml

@@ -0,0 +1,5 @@
+config-version: 2
+name: dbt_fal
+version: 1.0
+
+macro-paths: ["macros"]

dbt/include/fal/macros/materializations/table.sql

@@ -0,0 +1,46 @@
+{% materialization table, adapter='fal', supported_languages=['python', 'sql'] -%}
+
+{%- set language = model['language'] -%}
+
+{%- if language == 'python' -%}
+
+  {%- if adapter.is_teleport() -%}
+    {%- for _ref in model.refs -%}
+      {% set _ref_args = [_ref.get('package'), _ref['name']] if _ref.get('package') else [_ref['name'],] %}
+      {%- set resolved = ref(*_ref_args, v=_ref.get('version')) -%}
+      {%- do adapter.sync_teleport_relation(resolved) -%}
+    {%- endfor -%}
+  {%- endif -%}
+
+  {%- set relation = this.incorporate(type='table') -%}
+
+  {%- call statement('main', language=language) -%}
+
+    {{- py_write(compiled_code, relation) }}
+
+  {%- endcall %}
+
+{{- return({'relations': [relation]}) }}
+
+{%- elif language == 'sql' -%}
+
+  {# HACK: proxy to the db adapter table materialization #}
+  {{- return(adapter.db_materialization(context, "table")) }}
+
+{%- endif -%}
+
+{% endmaterialization %}
+
+{% macro py_write(code, relation) -%}
+{{- compiled_code -}}
+
+# Generated by dbt-fal
+
+def main(read_df, write_df, fal_context=None):
+  dbt_context = dbtObj(read_df)
+  df = model(dbt_context, fal_context)
+  return write_df(
+      '{{ relation.quote(False, False, False) }}',
+      df
+  )
+{%- endmacro %}

dbt/include/fal/macros/teleport_duckdb.sql

@@ -0,0 +1,8 @@
+{% macro duckdb__copy_to(relation, url) -%}
+    COPY {{ relation }} TO '{{ url }}'
+{%- endmacro %}
+
+{% macro duckdb__copy_from_parquet(relation, url) -%}
+    CREATE OR REPLACE TABLE {{ relation }} AS
+        SELECT * FROM read_parquet('{{ url }}');
+{%- endmacro %}

dbt/include/fal/macros/teleport_snowflake.sql

@@ -0,0 +1,31 @@
+{% macro snowflake__create_table_from_parquet(relation, location) -%}
+    CREATE OR REPLACE TABLE {{ relation }} USING template (
+      SELECT array_agg(object_construct(*))
+	  	FROM table(
+  			infer_schema(
+	  			LOCATION=>'{{ location }}',
+		  		FILE_FORMAT=>'falparquet',
+		  		IGNORE_CASE=>TRUE
+			  )
+  		)
+    );
+{%- endmacro %}
+
+{% macro snowflake__copy_from_parquet(relation, location) -%}
+    COPY INTO {{ relation }} FROM (
+      SELECT
+        {% for col in adapter.get_columns_in_relation(relation) %}
+          $1:{{col.column}}
+          {%- if not loop.last -%},{%- endif -%}
+        {% endfor %}
+      FROM {{ location }})
+    FILE_FORMAT = (FORMAT_NAME = 'falparquet');
+{%- endmacro %}
+
+{% macro snowflake__copy_to_parquet(relation, location) -%}
+    COPY INTO {{ location }} FROM {{ relation }}
+    FILE_FORMAT = (TYPE=parquet)
+    OVERWRITE = TRUE
+    SINGLE = TRUE
+    HEADER = TRUE;
+{%- endmacro %}

dbt/include/fal_experimental/__init__.py

@@ -0,0 +1,3 @@
+import os
+
+PACKAGE_PATH = os.path.dirname(__file__)

dbt/include/fal_experimental/dbt_project.yml

@@ -0,0 +1,5 @@
+config-version: 2
+name: dbt_fal_experimental
+version: 1.0
+
+macro-paths: ["macros"]

dbt/include/fal_experimental/macros/materializations/table.sql

@@ -0,0 +1,36 @@
+-- This file may be outdated because we actually use the `fal` materialization
+{% materialization table, adapter='fal_experimental', supported_languages=['python'] -%}
+
+  -- TODO: change to sync_teleport() when available in dbt-core
+  {%- if adapter.is_teleport() -%}
+    {%- for _ref in model.refs -%}
+        {%- set resolved = ref(*_ref) -%}
+        {%- do adapter.sync_teleport_relation(ref(*_ref)) -%}
+    {%- endfor -%}
+  {%- endif -%}
+
+  {%- set relation = this.incorporate(type='table') -%}
+
+  {%- call statement('main', language='python') -%}
+
+    {{- py_write(compiled_code, relation) }}
+
+  {%- endcall %}
+
+  {{- return({'relations': [relation]}) }}
+
+{% endmaterialization %}
+
+{% macro py_write(code, relation) -%}
+{{- compiled_code -}}
+
+# Generated by dbt-fal
+
+def main(read_df, write_df, fal_context=None):
+  dbt_context = dbtObj(read_df)
+  df = model(dbt_context, fal_context)
+  return write_df(
+      '{{ relation.quote(False, False, False) }}',
+      df
+  )
+{%- endmacro %}

fal/__init__.py

@@ -1,11 +1,61 @@
-from dbt.contracts.results import NodeStatus
-
-from faldbt.project import (
-    FalDbt,
-    DbtModel,
-    DbtSource,
-    DbtTest,
-    DbtGenericTest,
-    DbtSingularTest,
-)
-from fal.fal_script import Context, CurrentModel
+# TODO: remove once `fal` is no longer a supported package
+# The whole file
+DBT_FAL_IMPORT_NOTICE = \
+"""The dbt tool `fal` and `dbt-fal` adapter have been merged into a single tool.
+Please import from the `fal.dbt` module instead.
+Running `pip install dbt-fal` will install the new tool and the adapter alongside.
+Then import from the `fal.dbt` module like
+
+    from fal.dbt import {name}
+"""
+
+global DBT_FAL_IMPORT_NOTICE_SHOWN
+DBT_FAL_IMPORT_NOTICE_SHOWN = {}
+
+def _warn(name: str):
+    # Print once per deprecated import
+    global DBT_FAL_IMPORT_NOTICE_SHOWN
+    if not DBT_FAL_IMPORT_NOTICE_SHOWN.get(name, False):
+        print(DBT_FAL_IMPORT_NOTICE.format(name=name))
+        DBT_FAL_IMPORT_NOTICE_SHOWN[name] = True
+
+# Avoid printing on non-direct imports
+def __getattr__(name: str):
+    if name == "NodeStatus":
+        _warn(name)
+        from fal.dbt import NodeStatus
+        return NodeStatus
+    elif name == "FalDbt":
+        _warn(name)
+        from fal.dbt import FalDbt
+        return FalDbt
+    elif name == "DbtModel":
+        _warn(name)
+        from fal.dbt import DbtModel
+        return DbtModel
+    elif name == "DbtSource":
+        _warn(name)
+        from fal.dbt import DbtSource
+        return DbtSource
+    elif name == "DbtTest":
+        _warn(name)
+        from fal.dbt import DbtTest
+        return DbtTest
+    elif name == "DbtGenericTest":
+        _warn(name)
+        from fal.dbt import DbtGenericTest
+        return DbtGenericTest
+    elif name == "DbtSingularTest":
+        _warn(name)
+        from fal.dbt import DbtSingularTest
+        return DbtSingularTest
+    elif name == "Context":
+        _warn(name)
+        from fal.dbt import Context
+        return Context
+    elif name == "CurrentModel":
+        _warn(name)
+        from fal.dbt import CurrentModel
+        return CurrentModel
+
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

fal/dbt/__init__.py

@@ -0,0 +1,11 @@
+from dbt.contracts.results import NodeStatus
+
+from fal.dbt.integration.project import (
+    FalDbt,
+    DbtModel,
+    DbtSource,
+    DbtTest,
+    DbtGenericTest,
+    DbtSingularTest,
+)
+from fal.dbt.fal_script import Context, CurrentModel

fal/dbt/cli/__init__.py

@@ -0,0 +1 @@
+from .cli import cli, fal_cli

fal/{cli → dbt/cli}/args.py

@@ -204,11 +204,16 @@ def _build_flow_parser(sub: argparse.ArgumentParser):
 
 def _build_cli_parser():
     parser = argparse.ArgumentParser(
-        prog="fal",
+        prog="dbt-fal",
         description="Run Python scripts on dbt models",
     )
 
-    version = pkg_resources.get_distribution("fal").version
+    try:
+        version = pkg_resources.get_distribution("dbt-fal").version
+    except pkg_resources.DistributionNotFound:
+        # TODO: remove once `fal` is no longer a supported package
+        version = pkg_resources.get_distribution("fal").version
+
     parser.add_argument(
         "-v",
         "--version",

fal/{cli → dbt/cli}/cli.py

@@ -3,14 +3,29 @@ import sys
 
 from click.exceptions import ClickException
 
-from fal.cli.flow_runner import fal_flow_run
+from fal.dbt.cli.flow_runner import fal_flow_run
 from .args import parse_args
 from .fal_runner import fal_run
-from fal.telemetry import telemetry
+from fal.dbt.telemetry import telemetry
 
-from faldbt.logger import log_manager
+from fal.dbt.integration.logger import log_manager
 
 
+# TODO: remove once `fal` is no longer a supported package
+DBT_FAL_COMMAND_NOTICE = \
+"""The dbt tool `fal` and `dbt-fal` adapter have been merged into a single tool.
+Please use the new `dbt-fal` command line tool instead.
+Running `pip install dbt-fal` will install the new tool and the adapter alongside.
+Then run your command like
+
+    dbt-fal <command>
+"""
+
+# TODO: remove once `fal` is no longer a supported package
+def fal_cli(argv: List[str] = sys.argv):
+    print(DBT_FAL_COMMAND_NOTICE)
+    cli(argv)
+
 def cli(argv: List[str] = sys.argv):
     # Wrapper to be able to shutdown telemetry afterwards
     try:

fal/{cli → dbt/cli}/dbt_runner.py

@@ -3,7 +3,7 @@ from multiprocessing.connection import Connection
 from typing import Any, Dict, Optional, List
 import warnings
 import json
-from faldbt.logger import LOGGER
+from fal.dbt.integration.logger import LOGGER
 import os
 import argparse
 

fal/{cli → dbt/cli}/fal_runner.py

@@ -2,12 +2,12 @@ import argparse
 from pathlib import Path
 from typing import Any, Dict, List
 
-from fal.planner.executor import parallel_executor
-from fal.planner.schedule import Scheduler
-from fal.planner.tasks import FalLocalHookTask, Status, TaskGroup
+from fal.dbt.planner.executor import parallel_executor
+from fal.dbt.planner.schedule import Scheduler
+from fal.dbt.planner.tasks import FalLocalHookTask, Status, TaskGroup
 
-from fal.fal_script import FalScript, TimingType
-from faldbt.project import FAL, DbtModel, FalDbt, FalGeneralException
+from fal.dbt.fal_script import FalScript, TimingType
+from fal.dbt.integration.project import FAL, DbtModel, FalDbt, FalGeneralException
 
 
 def create_fal_dbt(
@@ -151,7 +151,7 @@ def _get_filtered_models(faldbt: FalDbt, all, selected, before) -> List[DbtModel
         and not before
         and faldbt._run_results.native_run_result is None
     ):
-        from faldbt.parse import FalParseError
+        from fal.dbt.integration.parse import FalParseError
 
         raise FalParseError(
             "Cannot define models to run without selection flags or dbt run_results artifact or --before flag"

fal/{cli → dbt/cli}/flow_runner.py

@@ -3,12 +3,12 @@ import copy
 from pathlib import Path
 from typing import Any, Dict, Optional, cast, Union
 
-from fal.cli.fal_runner import create_fal_dbt
-from fal.cli.selectors import ExecutionPlan
-from fal.cli.model_generator import generate_python_dbt_models
-from fal.fal_script import FalScript
-from fal.node_graph import DbtModelNode, FalFlowNode, NodeGraph, ScriptNode
-from faldbt.project import FalDbt, NodeStatus
+from fal.dbt.cli.fal_runner import create_fal_dbt
+from fal.dbt.cli.selectors import ExecutionPlan
+from fal.dbt.cli.model_generator import generate_python_dbt_models
+from fal.dbt.fal_script import FalScript
+from fal.dbt.node_graph import DbtModelNode, FalFlowNode, NodeGraph, ScriptNode
+from fal.dbt.integration.project import FalDbt, NodeStatus
 import argparse
 
 
@@ -23,14 +23,14 @@ def run_threaded(
     parsed: argparse.Namespace,
     node_graph: NodeGraph,
 ) -> int:
-    from fal.planner.plan import (
+    from fal.dbt.planner.plan import (
         OriginGraph,
         FilteredGraph,
         PlannedGraph,
         ScriptConnectedGraph,
     )
-    from fal.planner.schedule import schedule_graph
-    from fal.planner.executor import parallel_executor
+    from fal.dbt.planner.schedule import schedule_graph
+    from fal.dbt.planner.executor import parallel_executor
 
     execution_plan = ExecutionPlan.create_plan_from_graph(parsed, node_graph, fal_dbt)
 

fal/{cli → dbt/cli}/model_generator/model_generator.py

@@ -3,17 +3,17 @@ from functools import partial
 import re
 from typing import Callable, Iterable, List, TypeVar
 from pathlib import Path
-from fal.fal_script import python_from_file
+from fal.dbt.fal_script import python_from_file
 
-from faldbt.parse import get_fal_models_dirs, load_dbt_project_contract
-from fal.cli.model_generator.module_check import (
+from fal.dbt.integration.parse import get_fal_models_dirs, load_dbt_project_contract
+from fal.dbt.cli.model_generator.module_check import (
     generate_dbt_dependencies,
     write_to_model_check,
 )
 
-from faldbt.logger import LOGGER
+from fal.dbt.integration.logger import LOGGER
 
-from fal.telemetry import telemetry
+from fal.dbt.telemetry import telemetry
 
 SQL_MODEL_TEMPLATE = """
 {{ config(materialized='ephemeral') }}

fal/{cli → dbt/cli}/selectors.py

@@ -2,8 +2,8 @@ import itertools
 import re
 from dataclasses import dataclass
 from typing import List, Optional, Union, Iterator
-from fal.node_graph import NodeGraph
-from faldbt.project import CompileArgs, FalDbt, FalGeneralException
+from fal.dbt.node_graph import NodeGraph
+from fal.dbt.integration.project import CompileArgs, FalDbt, FalGeneralException
 from dbt.task.compile import CompileTask
 from enum import Enum
 from functools import reduce

fal/{fal_script.py → dbt/fal_script.py}

@@ -9,14 +9,14 @@ from deprecation import deprecated
 
 import hashlib
 
-from faldbt.parse import normalize_path
-from faldbt.project import DbtModel, FalDbt, FAL
+from fal.dbt.integration.parse import normalize_path
+from fal.dbt.integration.project import DbtModel, FalDbt, FAL
 
 from dbt.contracts.results import RunStatus
 from dbt.config.runtime import RuntimeConfig
-from faldbt.logger import LOGGER
+from fal.dbt.integration.logger import LOGGER
 
-from fal.telemetry import telemetry
+from fal.dbt.telemetry import telemetry
 
 from dbt.contracts.graph.nodes import ColumnInfo
 

{faldbt → fal/dbt/integration}/lib.py

@@ -29,8 +29,8 @@ from sqlalchemy.sql import Insert
 from dbt.contracts.sql import RemoteRunResult
 
 
-from faldbt import parse
-from faldbt.logger import LOGGER
+from fal.dbt.integration import parse
+from fal.dbt.integration.logger import LOGGER
 
 
 class WriteModeEnum(Enum):

{faldbt → fal/dbt/integration}/magics.py

@@ -1,6 +1,6 @@
 from IPython.core.magic import register_line_magic, needs_local_scope
 from functools import partial
-from fal import FalDbt
+from fal.dbt import FalDbt
 
 
 @register_line_magic
@@ -11,7 +11,7 @@ def init_fal(line="", local_ns={}):
 
     Example:
     """
-    from faldbt.magics import init_fal
+    from fal.dbt.integration.magics import init_fal
 
     %init_fal project_dir=/my_project_dir profiles_dir=/my_profiles_dir default_model_name=my_model
     """

{faldbt → fal/dbt/integration}/parse.py

@@ -14,14 +14,14 @@ from dbt.config.profile import read_user_config
 
 from dbt.exceptions import IncompatibleSchemaError, DbtRuntimeError
 
-from fal.utils import cache_static
+from fal.dbt.utils import cache_static
 
-from faldbt.logger import LOGGER
-from faldbt.utils.yaml_helper import load_yaml
-from fal.telemetry import telemetry
+from fal.dbt.integration.logger import LOGGER
+from fal.dbt.integration.utils.yaml_helper import load_yaml
+from fal.dbt.telemetry import telemetry
 
 if TYPE_CHECKING:
-    from fal.packages.environments import BaseEnvironment
+    from fal.dbt.packages.environments import BaseEnvironment
 
 FAL_SCRIPTS_PATH = "fal-scripts-path"
 FAL_MODELS_PATHS = "fal-models-paths"
@@ -225,8 +225,8 @@ def _get_required_key(data: Dict[str, Any], name: str) -> Any:
 
 
 def load_environments(base_dir: str) -> Dict[str, "BaseEnvironment"]:
-    from fal.packages.environments import create_environment
-    from fal.fal_script import _is_local_environment
+    from fal.dbt.packages.environments import create_environment
+    from fal.dbt.fal_script import _is_local_environment
 
     try:
         fal_project_path = os.path.join(base_dir, "fal_project.yml")