fal 0.9.2__py3-none-any.whl → 0.9.4__py3-none-any.whl

{faldbt → fal/dbt/integration}/project.py

@@ -14,7 +14,7 @@ from typing import (
 from pathlib import Path
 from deprecation import deprecated
 
-import faldbt.version as version
+import fal.dbt.integration.version as version
 
 from dbt.cli.resolvers import default_profiles_dir
 from dbt.cli.main import dbtRunner, dbtRunnerResult
@@ -49,16 +49,16 @@ from . import parse
 from . import lib
 from . import version
 
-from fal.feature_store.feature import Feature
+from fal.dbt.feature_store.feature import Feature
 
 import pandas as pd
 
-from fal.telemetry import telemetry
-from fal.utils import has_side_effects
+from fal.dbt.telemetry import telemetry
+from fal.dbt.utils import has_side_effects
 
 if TYPE_CHECKING:
-    from fal.fal_script import Hook, TimingType
-    from fal.packages.environments import BaseEnvironment
+    from fal.dbt.fal_script import Hook, TimingType
+    from fal.dbt.packages.environments import BaseEnvironment
 
 
 class FalGeneralException(Exception):
@@ -252,7 +252,7 @@ class DbtModel(_DbtTestableNode):
         self,
         hook_type: "TimingType",
     ) -> List["Hook"]:
-        from fal.fal_script import create_hook, TimingType
+        from fal.dbt.fal_script import create_hook, TimingType
 
         meta = self.meta or {}
 

fal/dbt/integration/utils/yaml_helper.py

@@ -0,0 +1,80 @@
+from typing import Any, Dict, Optional
+import yaml
+import yaml.scanner
+
+# the C version is faster, but it doesn't always exist
+try:
+    from yaml import CLoader as Loader, CSafeLoader as SafeLoader, CDumper as Dumper
+except ImportError:
+    from yaml import Loader, SafeLoader, Dumper  # type: ignore  # noqa: F401
+
+
+YAML_ERROR_MESSAGE = """
+Syntax error near line {line_number}
+------------------------------
+{nice_error}
+
+Raw Error:
+------------------------------
+{raw_error}
+""".strip()
+
+
+def line_no(i, line, width=3):
+    line_number = str(i).ljust(width)
+    return "{}| {}".format(line_number, line)
+
+
+def prefix_with_line_numbers(string, no_start, no_end):
+    line_list = string.split("\n")
+
+    numbers = range(no_start, no_end)
+    relevant_lines = line_list[no_start:no_end]
+
+    return "\n".join(
+        [line_no(i + 1, line) for (i, line) in zip(numbers, relevant_lines)]
+    )
+
+
+def contextualized_yaml_error(raw_contents, error):
+    mark = error.problem_mark
+
+    min_line = max(mark.line - 3, 0)
+    max_line = mark.line + 4
+
+    nice_error = prefix_with_line_numbers(raw_contents, min_line, max_line)
+
+    return YAML_ERROR_MESSAGE.format(
+        line_number=mark.line + 1, nice_error=nice_error, raw_error=error
+    )
+
+
+def safe_load(contents) -> Dict[str, Any]:
+    return yaml.load(contents, Loader=SafeLoader)
+
+
+def load_yaml_text(contents):
+    try:
+        return safe_load(contents)
+    except (yaml.scanner.ScannerError, yaml.YAMLError) as e:
+        if hasattr(e, "problem_mark"):
+            error = contextualized_yaml_error(contents, e)
+        else:
+            error = str(e)
+
+        raise Exception(error)
+
+
+def _load_file_contents(path: str, strip: bool = True) -> str:
+    with open(path, "rb") as handle:
+        to_return = handle.read().decode("utf-8")
+
+    if strip:
+        to_return = to_return.strip()
+
+    return to_return
+
+
+def load_yaml(path):
+    contents = _load_file_contents(path)
+    return load_yaml_text(contents)

fal/dbt/new/project.py

@@ -0,0 +1,43 @@
+from typing import Optional, List, cast
+from dbt.cli.main import dbtRunner, dbtRunnerResult
+
+class falProject:
+    """
+    Represents a dbt project and access to its resources and utility functions.
+    """
+
+    def _dbt_invoke(
+        self, cmd: str, args: Optional[List[str]] = None
+    ) -> dbtRunnerResult:
+        if args is None:
+            args = []
+
+        project_args = [
+            "--project-dir",
+            self.project_dir,
+            "--profiles-dir",
+            self.profiles_dir,
+        ]
+        if self.target_name:
+            project_args.extend(["--target", self.target_name])
+
+        # TODO: Intervene the dbt logs and capture them to avoid printing them to the console
+        return self._runner.invoke([cmd] + project_args + args)
+
+    def __init__(
+        self, project_dir: str, profiles_dir: str, target_name: Optional[str] = None
+    ):
+        # TODO: Make project_dir and profiles_dir optional and use the current working directory and default profiles dir?
+        self.project_dir = project_dir
+        self.profiles_dir = profiles_dir
+        self.target_name = target_name
+
+        # Load the manifest information
+        self._runner = dbtRunner()
+        parse_result = self._dbt_invoke("parse")
+        native_manifest = cast(Manifest, parse_result.result)  # type: ignore
+        self._manifest = native_manifest
+        # self._manifest = DbtManifest(native_manifest)
+
+        # TODO: Do we need the manifest in there?
+        self._runner = dbtRunner(manifest=native_manifest)

fal/{node_graph.py → dbt/node_graph.py}

@@ -2,8 +2,8 @@ from __future__ import annotations
 from dataclasses import dataclass
 from typing import List, Tuple, Dict, cast
 
-from fal.fal_script import FalScript, TimingType
-from faldbt.project import DbtModel, FalDbt
+from fal.dbt.fal_script import FalScript, TimingType
+from fal.dbt.integration.project import DbtModel, FalDbt
 from pathlib import Path
 import networkx as nx
 import os as os

fal/{packages → dbt/packages}/dependency_analysis.py

@@ -5,13 +5,13 @@ from typing import Iterator, List, Optional, Tuple
 
 import importlib_metadata
 
-from fal.utils import cache_static
+from fal.dbt.utils import cache_static
 
 import importlib_metadata
 
 
 def _get_project_root_path(pacakge: str) -> Path:
-    import fal
+    import fal.dbt as fal
 
     # If this is a development version, we'll install
     # the current fal itself.
@@ -41,17 +41,12 @@ def _get_dbt_packages() -> Iterator[Tuple[str, Optional[str]]]:
         if dbt_plugin_name == "dbt-core":
             continue
 
-        # Handle dbt-fal separately (since it needs to be installed
-        # with its extras).
+        # Skip dbt-fal since it is already handled by _get_dbt_fal_package_name
         if dbt_plugin_name == "dbt-fal":
             continue
 
         yield dbt_plugin_name, distribution.version
 
-    package_info = _get_dbt_fal_package_name()
-    if package_info:
-        yield package_info
-
 
 def _find_fal_extras(package: str) -> set[str]:
     # Return a possible set of extras that might be required when installing
@@ -86,8 +81,8 @@ def _find_fal_extras(package: str) -> set[str]:
     # the smallest possible subset of extras that we can install.
     return available_dbt_adapters.intersection(all_extras)
 
-def _is_fal_pre_release() -> bool:
-    raw_fal_version = importlib_metadata.version("fal")
+def _running_pre_release() -> bool:
+    raw_fal_version = importlib_metadata.version("dbt-fal")
     return _version_is_prerelease(raw_fal_version)
 
 def _version_is_prerelease(raw_version: str) -> bool:
@@ -96,43 +91,42 @@ def _version_is_prerelease(raw_version: str) -> bool:
     package_version = Version(raw_version)
     return package_version.is_prerelease
 
-def _get_dbt_fal_package_name() -> Optional[Tuple[str, Optional[str]]]:
-    try:
-        dbt_fal_version = importlib_metadata.version("dbt-fal")
-    except importlib_metadata.PackageNotFoundError:
-        # It might not be installed.
-        return None
-
-    if _is_fal_pre_release():
-        dbt_fal_dep = str(_get_project_root_path("adapter"))
-        # We are going to install it from the local path.
-        dbt_fal_version = None
+def _get_dbt_fal_package() -> Tuple[str, Optional[str]]:
+    if _running_pre_release():
+        proj_path = _get_project_root_path("adapter")
+        if proj_path.exists():
+            # We are going to install it from the local path.
+            dbt_fal_dep = str(proj_path)
+            dbt_fal_version = None
+        else:
+            # We are going to install it from PyPI.
+            dbt_fal_dep = "dbt-fal"
+            try:
+                dbt_fal_version = importlib_metadata.version("dbt-fal")
+            except importlib_metadata.PackageNotFoundError:
+                # TODO: remove once `fal` is no longer a supported package
+                dbt_fal_version = importlib_metadata.version("fal")
     else:
         dbt_fal_dep = "dbt-fal"
+        try:
+            dbt_fal_version = importlib_metadata.version("dbt-fal")
+        except importlib_metadata.PackageNotFoundError:
+            # TODO: remove once `fal` is no longer a supported package
+            dbt_fal_version = importlib_metadata.version("fal")
+
+    try:
+        dbt_fal_extras = _find_fal_extras("dbt-fal")
+    except importlib_metadata.PackageNotFoundError:
+        # TODO: remove once `fal` is no longer a supported package
+        dbt_fal_extras = _find_fal_extras("fal")
 
-    dbt_fal_extras = _find_fal_extras("dbt-fal")
     if dbt_fal_extras:
         dbt_fal_dep += f"[{','.join(dbt_fal_extras)}]"
 
     return dbt_fal_dep, dbt_fal_version
 
-def _get_fal_package_name() -> Tuple[str, Optional[str]]:
-    if _is_fal_pre_release():
-        fal_dep = str(_get_project_root_path("fal"))
-        fal_version = None
-    else:
-        fal_dep = "fal"
-        fal_version = importlib_metadata.version("fal")
-
-    fal_extras = _find_fal_extras("fal")
-    if fal_extras:
-        fal_dep += f"[{','.join(fal_extras)}]"
-
-    return fal_dep, fal_version
-
-
 def get_default_requirements() -> Iterator[Tuple[str, Optional[str]]]:
-    yield _get_fal_package_name()
+    yield _get_dbt_fal_package()
     yield from _get_dbt_packages()
 
 

fal/{packages → dbt/packages}/environments/__init__.py

@@ -1,9 +1,9 @@
 from pathlib import Path
 from typing import List, Optional, Dict, Any
 
-from fal.packages.environments.base import BASE_CACHE_DIR, BaseEnvironment
-from fal.packages.environments.conda import CondaEnvironment
-from fal.packages.environments.virtual_env import VirtualPythonEnvironment
+from fal.dbt.packages.environments.base import BASE_CACHE_DIR, BaseEnvironment
+from fal.dbt.packages.environments.conda import CondaEnvironment
+from fal.dbt.packages.environments.virtual_env import VirtualPythonEnvironment
 
 REGISTERED_ENVIRONMENTS: Dict[str, BaseEnvironment] = {
     "conda": CondaEnvironment,

fal/{packages → dbt/packages}/environments/base.py

@@ -13,8 +13,8 @@ from typing import Any, Callable, ContextManager, Generic, Iterator, TypeVar, Di
 
 from platformdirs import user_cache_dir
 
-from faldbt.logger import LOGGER
-from fal.packages import bridge, isolated_runner
+from fal.dbt.integration.logger import LOGGER
+from fal.dbt.packages import bridge, isolated_runner
 
 BASE_CACHE_DIR = Path(user_cache_dir("fal", "fal"))
 BASE_CACHE_DIR.mkdir(exist_ok=True)

fal/{packages → dbt/packages}/environments/conda.py

@@ -10,15 +10,15 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import List, Dict, Any
 
-from fal.packages.environments.base import (
+from fal.dbt.packages.environments.base import (
     BASE_CACHE_DIR,
     BaseEnvironment,
     DualPythonIPC,
     log_env,
     rmdir_on_fail,
 )
-from fal.packages.environments.virtual_env import get_primary_virtual_env
-from fal.utils import cache_static
+from fal.dbt.packages.environments.virtual_env import get_primary_virtual_env
+from fal.dbt.utils import cache_static
 
 _BASE_CONDA_DIR = BASE_CACHE_DIR / "conda"
 _BASE_CONDA_DIR.mkdir(exist_ok=True)

fal/{packages → dbt/packages}/environments/virtual_env.py

@@ -7,8 +7,8 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import List, Dict, Any
 
-from fal.packages.dependency_analysis import get_default_pip_dependencies
-from fal.packages.environments.base import (
+from fal.dbt.packages.dependency_analysis import get_default_pip_dependencies
+from fal.dbt.packages.environments.base import (
     BASE_CACHE_DIR,
     BaseEnvironment,
     DualPythonIPC,
@@ -17,7 +17,7 @@ from fal.packages.environments.base import (
     python_path_for,
     rmdir_on_fail,
 )
-from fal.utils import cache_static
+from fal.dbt.utils import cache_static
 
 _BASE_VENV_DIR = BASE_CACHE_DIR / "venvs"
 _BASE_VENV_DIR.mkdir(exist_ok=True)

fal/{packages → dbt/packages}/isolated_runner.py

@@ -36,8 +36,8 @@ DEBUG_TIMEOUT = 60 * 15
 
 
 def run_client(address: str, *, with_pdb: bool = False) -> int:
-    from faldbt.logger import LOGGER
-    from fal.packages import bridge
+    from fal.dbt.integration.logger import LOGGER
+    from fal.dbt.packages import bridge
 
     if with_pdb:
         # This condition will only be activated if we want to
@@ -88,8 +88,8 @@ def _get_shell_bootstrap() -> str:
 
 
 def _fal_main() -> None:
-    from faldbt.logger import LOGGER
-    from fal.packages import bridge
+    from fal.dbt.integration.logger import LOGGER
+    from fal.dbt.packages import bridge
 
     LOGGER.debug("Starting the isolated process at PID {}", os.getpid())
 
@@ -125,7 +125,7 @@ def _process_primary_env_packages() -> None:
 def main():
     _process_primary_env_packages()
 
-    from faldbt.logger import log_manager
+    from fal.dbt.integration.logger import log_manager
 
     # TODO: do we still need this?
     with log_manager.applicationbound():

fal/{planner → dbt/planner}/executor.py

@@ -10,8 +10,8 @@ from concurrent.futures import (
 from dataclasses import dataclass, field
 from typing import Iterator, List, Optional
 
-from fal.planner.schedule import SUCCESS, Scheduler
-from fal.planner.tasks import (
+from fal.dbt.planner.schedule import SUCCESS, Scheduler
+from fal.dbt.planner.tasks import (
     TaskGroup,
     Task,
     Status,
@@ -19,9 +19,9 @@ from fal.planner.tasks import (
     FalLocalHookTask,
     HookType,
 )
-from faldbt.project import FalDbt
+from fal.dbt.integration.project import FalDbt
 
-from faldbt.logger import LOGGER
+from fal.dbt.integration.logger import LOGGER
 
 
 class State(Enum):

fal/{planner → dbt/planner}/plan.py

@@ -3,9 +3,9 @@ from __future__ import annotations
 from typing import Callable, Iterator, List, Set
 
 import networkx as nx
-from fal.node_graph import NodeKind
-from fal.cli.selectors import ExecutionPlan, _is_before_script, _is_after_script
-from faldbt.logger import LOGGER
+from fal.dbt.node_graph import NodeKind
+from fal.dbt.cli.selectors import ExecutionPlan, _is_before_script, _is_after_script
+from fal.dbt.integration.logger import LOGGER
 from dataclasses import dataclass
 
 

fal/{planner → dbt/planner}/schedule.py

@@ -6,9 +6,9 @@ from typing import Iterator, List, Optional
 
 import networkx as nx
 
-from faldbt.project import DbtModel
-from fal.node_graph import DbtModelNode, NodeGraph, NodeKind, ScriptNode
-from fal.planner.tasks import (
+from fal.dbt.integration.project import DbtModel
+from fal.dbt.node_graph import DbtModelNode, NodeGraph, NodeKind, ScriptNode
+from fal.dbt.planner.tasks import (
     SUCCESS,
     Task,
     DBTTask,
@@ -19,8 +19,8 @@ from fal.planner.tasks import (
     Status,
     HookType,
 )
-from fal.utils import DynamicIndexProvider
-from fal.fal_script import Hook, IsolatedHook, create_hook, TimingType
+from fal.dbt.utils import DynamicIndexProvider
+from fal.dbt.fal_script import Hook, IsolatedHook, create_hook, TimingType
 
 
 def create_hook_task(

fal/{planner → dbt/planner}/tasks.py

@@ -13,11 +13,11 @@ from dataclasses import dataclass, field
 from enum import Enum, auto
 from typing import Iterator, List, Any, Optional, Dict, Tuple, Union
 
-from faldbt.logger import LOGGER
+from fal.dbt.integration.logger import LOGGER
 
-from fal.fal_script import FalScript, TimingType
-from fal.utils import print_run_info, DynamicIndexProvider
-from faldbt.project import DbtModel, FalDbt, NodeStatus
+from fal.dbt.fal_script import FalScript, TimingType
+from fal.dbt.utils import print_run_info, DynamicIndexProvider
+from fal.dbt.integration.project import DbtModel, FalDbt, NodeStatus
 
 from datetime import datetime, timezone
 
@@ -162,7 +162,7 @@ class DBTTask(Task):
     model_ids: List[str]
 
     def execute(self, args: argparse.Namespace, fal_dbt: FalDbt) -> int:
-        from fal.cli.dbt_runner import dbt_run_through_python
+        from fal.dbt.cli.dbt_runner import dbt_run_through_python
 
         model_names = _unique_ids_to_model_names(self.model_ids)
         output = dbt_run_through_python(

fal/{telemetry → dbt/telemetry}/telemetry.py

@@ -24,7 +24,7 @@ import inspect
 from contextlib import contextmanager
 from dbt.config.runtime import RuntimeConfig
 
-from fal.utils import cache_static
+from fal.dbt.utils import cache_static
 
 import platform
 
@@ -127,7 +127,7 @@ def get_dbt_adapter_type(config: RuntimeConfig | None) -> str | None:
 def fal_installed_version():
     """Returns: fal version"""
     try:
-        return pkg_resources.get_distribution("fal").version
+        return pkg_resources.get_distribution("dbt-fal").version
     except ImportError:
         return
 
@@ -242,8 +242,8 @@ def write_conf_file(conf_path, to_write, error=None):
 def get_dbt_config():
     try:
         from dbt.flags import PROFILES_DIR
-        from fal.cli.args import parse_args
-        from faldbt.parse import get_dbt_config
+        from fal.dbt.cli.args import parse_args
+        from fal.dbt.integration.parse import get_dbt_config
 
         args = parse_args(sys.argv[1:])
 

fal/{typing.py → dbt/typing.py}

@@ -7,8 +7,8 @@ from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     import pandas as pd
     from typing import Optional, Dict, List, Protocol, Any
-    from faldbt.project import DbtModel, DbtTest, DbtSource, Feature
-    from fal.fal_script import Context, CurrentModel
+    from fal.dbt.integration.project import DbtModel, DbtTest, DbtSource, Feature
+    from fal.dbt.fal_script import Context, CurrentModel
 
     class _List_Sources(Protocol):
         def __call__(self) -> List[DbtSource]:

fal/{utils.py → dbt/utils.py}

@@ -1,6 +1,6 @@
 """Fal utilities."""
 import copy
-from faldbt.logger import LOGGER
+from fal.dbt.integration.logger import LOGGER
 from typing import List, TYPE_CHECKING, TypeVar
 
 try:
@@ -9,7 +9,7 @@ except ImportError:
     from backports.functools_lru_cache import lru_cache
 
 if TYPE_CHECKING:
-    from fal.fal_script import FalScript
+    from fal.dbt.fal_script import FalScript
 
 
 def print_run_info(scripts: List["FalScript"]):