fabricks 3.0.5.2__py3-none-any.whl → 3.0.7__py3-none-any.whl

fabricks/config/runtime.py

@@ -1,67 +0,0 @@
-from __future__ import annotations
-
-from typing import Any, Dict, List, Optional
-
-from fabricks.config.steps.base import ExtenderOptions, ModelBase
-from fabricks.config.steps.bronze import BronzeStepConfig
-from fabricks.config.steps.gold import GoldStepConfig
-from fabricks.config.steps.silver import SilverStepConfig
-
-
-class RuntimePathOptions(ModelBase):
-    storage: str
-    udfs: str
-    parsers: str
-    schedules: str
-    views: str
-    requirements: str
-
-
-class RuntimeTimeoutOptions(ModelBase):
-    step: int
-    job: int
-    pre_run: int
-    post_run: int
-
-
-class RuntimeOptions(ModelBase):
-    secret_scope: str
-    unity_catalog: Optional[bool] = None
-    type_widening: Optional[bool] = None
-    catalog: Optional[str] = None
-    workers: int
-    timeouts: RuntimeTimeoutOptions
-    retention_days: int
-
-
-class SparkOptions(ModelBase):
-    sql: Dict[str, Any]
-    conf: Dict[str, Any]
-
-
-class PowerBI(ModelBase):
-    name: str
-
-
-class DatabasePathOptions(ModelBase):
-    storage: str
-
-
-class Database(ModelBase):
-    name: str
-    path_options: DatabasePathOptions
-
-
-class RuntimeConfig(ModelBase):
-    name: str
-    options: RuntimeOptions
-    path_options: RuntimePathOptions
-    extender_options: Optional[ExtenderOptions] = None
-    spark_options: SparkOptions
-    bronze: Optional[List[BronzeStepConfig]] = None
-    silver: Optional[List[SilverStepConfig]] = None
-    gold: Optional[List[GoldStepConfig]] = None
-    powerbi: Optional[List[PowerBI]] = None
-    databases: Optional[List[Database]] = None
-    variables: Optional[List[Dict[str, Any]]] = None
-    credentials: Optional[List[Dict[str, Any]]] = None

fabricks/config/steps/__init__.py

@@ -1,6 +0,0 @@
-from fabricks.config.steps.base import BaseStepConfig
-from fabricks.config.steps.bronze import BronzeStepConfig
-from fabricks.config.steps.gold import GoldStepConfig
-from fabricks.config.steps.silver import SilverStepConfig
-
-__all__ = ["BaseStepConfig", "BronzeStepConfig", "SilverStepConfig", "GoldStepConfig"]

fabricks/config/steps/base.py

@@ -1,50 +0,0 @@
-from __future__ import annotations
-
-from typing import Any, Dict, List, Optional
-
-from fabricks.config.base import ModelBase
-
-
-class PathOptions(ModelBase):
-    runtime: str
-    storage: str
-
-
-class StepTimeoutOptions(ModelBase):
-    step: Optional[int] = None
-    job: Optional[int] = None
-    pre_run: Optional[int] = None
-    post_run: Optional[int] = None
-
-
-class InvokeOptions(ModelBase):
-    notebook: str
-    arguments: Optional[Dict[str, Any]] = None
-
-
-class ExtenderOptions(ModelBase):
-    extender: str
-    arguments: Optional[Dict[str, Any]] = None
-
-
-class TableOptions(ModelBase):
-    powerbi: Optional[bool] = None
-    liquid_clustering: Optional[bool] = None
-    properties: Optional[Dict[str, Any]] = None
-    retention_days: Optional[int] = None
-
-
-class DefaultOptions(ModelBase):
-    order: int
-    workers: Optional[int] = None
-    timeouts: StepTimeoutOptions
-    extenders: Optional[List[str]] = None
-    pre_run: Optional[InvokeOptions] = None
-    post_run: Optional[InvokeOptions] = None
-
-
-class BaseStepConfig(ModelBase):
-    name: str
-    options: DefaultOptions
-    path_options: PathOptions
-    table_options: Optional[TableOptions] = None

fabricks/config/steps/bronze.py

@@ -1,7 +0,0 @@
-from __future__ import annotations
-
-from fabricks.config.steps.base import BaseStepConfig
-
-
-class BronzeStepConfig(BaseStepConfig):
-    pass

fabricks/config/steps/gold.py

@@ -1,14 +0,0 @@
-from __future__ import annotations
-
-from typing import Optional
-
-from fabricks.config.steps.base import BaseStepConfig, DefaultOptions
-
-
-class GoldOptions(DefaultOptions):
-    schema_drift: Optional[bool] = None
-    metadata: Optional[bool] = None
-
-
-class GoldStepConfig(BaseStepConfig):
-    options: GoldOptions

fabricks/config/steps/silver.py

@@ -1,15 +0,0 @@
-from __future__ import annotations
-
-from typing import Optional
-
-from fabricks.config.steps.base import BaseStepConfig, DefaultOptions
-
-
-class SilverOptions(DefaultOptions):
-    parent: str
-    stream: Optional[bool] = None
-    local_checkpoint: Optional[bool] = None
-
-
-class SilverStepConfig(BaseStepConfig):
-    options: SilverOptions

fabricks/core/deploy/__init__.py

@@ -1,17 +0,0 @@
-from fabricks.core.deploy.tables import deploy_tables
-from fabricks.core.deploy.udfs import deploy_udfs
-from fabricks.core.deploy.views import deploy_views
-
-
-class deploy:
-    @staticmethod
-    def tables(drop: bool = False):
-        deploy_tables(drop=drop)
-
-    @staticmethod
-    def views():
-        deploy_views()
-
-    @staticmethod
-    def udfs():
-        deploy_udfs()

fabricks/core/schedules.py

@@ -1,142 +0,0 @@
-from typing import List, Optional, TypedDict
-
-from pyspark.sql import DataFrame
-from pyspark.sql.types import Row
-
-from fabricks.context import PATH_SCHEDULES, SPARK
-from fabricks.context.log import DEFAULT_LOGGER
-from fabricks.core.jobs.base._types import TStep
-from fabricks.utils.read.read_yaml import read_yaml
-from fabricks.utils.schema import get_schema_for_type
-from fabricks.utils.sqlglot import fix as fix_sql
-
-
-class Options(TypedDict):
-    steps: Optional[List[TStep]]
-    tag: Optional[str]
-    view: Optional[str]
-    variables: Optional[dict[str, str]]
-
-
-class Schedule(TypedDict):
-    name: str
-    options: Options
-
-
-def get_schedules():
-    return read_yaml(PATH_SCHEDULES, root="schedule")
-
-
-def get_schedules_df() -> DataFrame:
-    schema = get_schema_for_type(Schedule)
-    df = SPARK.createDataFrame(list(get_schedules()), schema=schema)  # type: ignore
-
-    assert df, "no schedules found"
-    return df
-
-
-def get_schedule(name: str) -> Row:
-    schedules = [s for s in get_schedules() if s["name"] == name]
-
-    assert schedules, "schedule not found"
-    assert len(schedules) == 1, "schedule duplicated"
-    return Row(**schedules[0])
-
-
-def create_or_replace_view_internal(name: str, options: dict):
-    step = "-- no step provided"
-    tag = "-- no tag provided"
-    view = "-- no view provided"
-
-    assert isinstance(options, dict), "options must be a dict"
-
-    if options.get("steps") is not None:
-        steps = [f"'{s}'" for s in options.get("steps")]  # type: ignore
-        step = f"and j.step in ({', '.join(steps)})"
-
-    if options.get("tag") is not None:
-        tag = f"""and array_contains(j.tags, '{options.get("tag")}')"""
-
-    if options.get("view") is not None:
-        view = f"""inner join fabricks.{options.get("view")} v on j.job_id = v.job_id"""
-
-    sql = f"""
-    create or replace view fabricks.{name}_schedule
-    as
-    select
-        j.*
-    from
-        fabricks.jobs j
-        {view}
-    where
-        true 
-        {step}
-        {tag}
-        and j.type not in ('manual')
-    """
-    sql = fix_sql(sql)
-    DEFAULT_LOGGER.debug(f"schedule - %sql\n---\n{sql}\n---")
-
-    SPARK.sql(sql)
-
-
-def create_or_replace_view(name: str):
-    row = get_schedule(name=name)
-    try:
-        create_or_replace_view_internal(row.name, row.options)
-    except Exception:
-        DEFAULT_LOGGER.exception(f"schedule - {row.name} not created nor replaced")
-
-
-def create_or_replace_views():
-    df = get_schedules_df()
-    for row in df.collect():
-        try:
-            create_or_replace_view_internal(row.name, row.options.asDict())
-        except Exception:
-            DEFAULT_LOGGER.exception(f"schedule - {row.name} not created nor replaced")
-
-
-def get_dependencies(name: str) -> DataFrame:
-    from fabricks.core.dags import DagGenerator
-
-    g = DagGenerator(schedule=name)
-    return g.get_dependencies()
-
-
-def get_mermaid_diagram(name: str) -> str:
-    df = get_dependencies(name)
-
-    df = df.withColumnRenamed("ParentId", "parent_id")
-    df = df.withColumnRenamed("Parent", "parent")
-    df = df.withColumnRenamed("JobId", "job_id")
-    df = df.withColumnRenamed("Job", "job")
-
-    dependencies = df.select("parent_id", "parent", "job_id", "job").collect()
-
-    out = "flowchart TD\n"
-
-    unique_nodes = set()
-
-    for row in dependencies:
-        parent_id = str(row["parent_id"])
-        parent_name = str(row["parent"])
-        child_id = str(row["job_id"])
-        child_name = str(row["job"])
-
-        if parent_id != "0" and parent_id is not None:
-            if parent_id not in unique_nodes:
-                out += f"    {parent_id}[{parent_name}]\n"
-                unique_nodes.add(parent_id)
-
-            if child_id not in unique_nodes:
-                out += f"    {child_id}[{child_name}]\n"
-                unique_nodes.add(child_id)
-
-            out += f"    {parent_id} --> {child_id}\n"
-        else:
-            if child_id not in unique_nodes:
-                out += f"    {child_id}[{child_name}]\n"
-                unique_nodes.add(child_id)
-
-    return out

fabricks/core/scripts/__init__.py

@@ -1,9 +0,0 @@
-from fabricks.core.scripts.generate import generate
-from fabricks.core.scripts.process import process
-from fabricks.core.scripts.terminate import terminate
-
-__all__ = [
-    "process",
-    "generate",
-    "terminate",
-]

fabricks/core/scripts/armageddon.py

@@ -1,87 +0,0 @@
-import logging
-from typing import List, Optional, Union, cast
-
-from fabricks.context import FABRICKS_STORAGE
-from fabricks.context.log import DEFAULT_LOGGER
-from fabricks.core.deploy import deploy
-from fabricks.core.jobs.base._types import Steps, TStep
-from fabricks.core.schedules import create_or_replace_views as create_or_replace_schedules_views
-from fabricks.core.steps.base import BaseStep
-from fabricks.core.views import create_or_replace_views
-from fabricks.metastore.database import Database
-
-
-def armageddon(steps: Optional[Union[TStep, List[TStep], str, List[str]]]):
-    DEFAULT_LOGGER.setLevel(logging.INFO)
-
-    if steps is None:
-        steps = Steps
-    assert steps is not None
-
-    if isinstance(steps, str):
-        steps = [cast(TStep, steps)]
-    elif isinstance(steps, List):
-        steps = [cast(TStep, s) for s in steps]
-    elif isinstance(steps, TStep):
-        steps = [steps]
-
-    DEFAULT_LOGGER.warning("armageddon")
-    print("")
-    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣀⡤⠤⠴⠾⠋⠉⠛⢾⡏⠙⠿⠦⠤⢤⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                    ")
-    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⡤⢶⣿⠉⢀⣀⡠⠆⠀⠀⠀⠀⠀⠀⠀⢤⣀⣀⠈⢹⣦⢤⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                    ")
-    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣴⣿⠁⢋⡙⠁⠀⡝⠀⠀⠀⠀⣀⡸⠋⠁⠀⠀⠹⡀⠀⠈⠈⠆⢹⢦⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                   ")
-    print("        ⠀⠀⠀⠀⠀⠀⠀⢀⣠⣤⣿⣁⡡⣴⡏⠀⠀⠀⢀⠀⢧⣀⠄⠀⠀⠀⣀⣰⠆⢀⠁⠀⠀⢈⣶⡤⣀⢹⣦⣄⡀⠀⠀⠀⠀⠀⠀⠀⠀                  ")
-    print("        ⠀⠀⠀⠀⠀⣠⢴⠟⢁⡝⠀⠁⠀⠃⠉⠀⠀⠘⣯⠀⡀⠾⣤⣄⣠⢤⠾⠄⠀⣸⠖⠀⠀⠈⠀⠃⠀⠀⠹⡄⠙⣶⢤⡀⠀⠀⠀⠀⠀                 ")
-    print("        ⠀⠀⠀⣠⠾⡇⠈⣀⡞⠀⠀⠀⠀⡀⠀⢀⣠⣄⣇⠀⣳⠴⠃⠀⠀⠀⠣⢴⠉⣰⣇⣀⣀⠀⠀⡄⠀⠀⠀⢹⣄⡘⠈⡷⣦⠀⠀⠀⠀                 ")
-    print("        ⢠⠞⠉⢻⡄⠀⠀⠈⠙⠀⠀⠀⠀⠙⣶⣏⣤⣤⠟⠉⠁⠀⠀⠀⠀⠀⠀⠀⠉⠙⢦⣱⣌⣷⠊⠀⠀⠀⠀⠈⠁⠀⠀⠀⡝⠉⠻⣄⠀                 ")
-    print("        ⠛⢀⡠⢼⡇⠀⠀⢀⡄⠀⢀⣀⡽⠚⠁⠀⠀⠀⢠⡀⢠⣀⠠⣔⢁⡀⠀⣄⠀⡄⠀⠀⠀⠈⠑⠺⣄⡀⠀⠠⡀⠀⠀⢠⡧⠄⠀⠘⢧                ")
-    print("        ⡶⠋⠀⠀⠈⣠⣈⣩⠗⠒⠋⠀⠀⠀⠀⣀⣠⣆⡼⣷⣞⠛⠻⡉⠉⡟⠒⡛⣶⠧⣀⣀⣀⠀⠀⠀⠀⠈⠓⠺⢏⣉⣠⠋⠀⠀⠀⢢⣸                ")
-    print("        ⠇⠐⠤⠤⠖⠁⣿⣀⣀⠀⠀⠀⠀⠀⠉⠁⠈⠉⠙⠛⢿⣷⡄⢣⡼⠀⣾⣿⠧⠒⠓⠚⠛⠉⠀⠀⠀⠀⠀⢀⣀⣾⡉⠓⠤⡤⠄⠸⢿                ")
-    print("        ⣆⣤⠀⠀⠠⠀⠈⠓⠈⠓⠤⡀⠀⠀⠀⠀⠀⠀⠀⠀⠈⣿⣿⢸⠀⢸⣿⠇⠀⠀⠀⠀⠀⠀⠀⠀⢀⡤⠒⠁⠰⠃⠀⠠⠀⠀⢀⣀⠞                  ")
-    print("        ⠀⠉⠓⢲⣄⡈⢀⣠⠀⠀⠀⡸⠶⠂⠀⠀⢀⠀⠀⠤⠞⢻⡇⠀⠀⢘⡟⠑⠤⠄⠀⢀⠀⠀⠐⠲⢿⡀⠀⠀⢤⣀⢈⣀⡴⠖⠋⠀⠀                 ")
-    print("        ⠀⠀⠀⠀⠈⠉⠉⠙⠓⠒⣾⣁⣀⣴⠀⣀⠙⢧⠂⢀⣆⣀⣷⣤⣀⣾⣇⣀⡆⠀⢢⠛⢁⠀⢰⣀⣀⣹⠒⠒⠛⠉⠉⠉⠀⠀⠀⠀⠀                ")
-    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠉⠁⠈⠉⠉⠛⠉⠙⠉⠀⠀⣿⡟⣿⣿⠀⠀⠈⠉⠉⠙⠋⠉⠉⠀⠉⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                   ")
-    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣿⡇⢻⣿⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                      ")
-    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣀⣤⣶⣾⣿⣿⠁⠀⢹⡛⣟⡶⢤⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                    ")
-    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣴⠛⢯⣽⡟⢿⣿⠛⠿⠳⠞⠻⣿⠻⣆⢽⠟⣶⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                   ")
-    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠛⠃⠲⠯⠴⣦⣼⣷⣤⣤⣶⣤⣩⡧⠽⠷⠐⠛⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                   ")
-    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣿⡇⠀⣿⡆⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                     ")
-    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣀⣄⡀⢀⣀⣠⡾⡿⢡⢐⠻⣿⣄⣀⡀⠀⣀⣄⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                  ")
-    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣤⢴⡏⠁⠀⠝⠉⣡⠟⣰⠃⢸⣿⠀⣷⠙⢧⡉⠻⡅⠀⠙⡷⢤⣀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                 ")
-    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣴⡟⠀⠈⣿⢄⡴⠞⠻⣄⣰⣡⠤⣞⣸⡤⢬⣧⣀⡿⠛⠦⣤⣶⡃⠀⢹⣦⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                 ")
-    print("        ⠀⠀⠀⠀⠀⠀⢀⣴⣶⡿⠃⠉⢺⠁⠙⠒⠀⠀⣠⡉⠀⠉⠚⠉⠉⠑⠈⠀⠈⣧⠀⠀⠒⠋⠀⡹⠋⠀⢻⡶⠶⡄⠀⠀⠀⠀⠀⠀⠀                 ")
-    print("        ⠀⠀⠀⠀⠀⣠⣾⣿⣇⠁⢈⡦⠀⡍⠋⠁⡀⠸⡋⠀⠀⠀⢘⠏⠉⡏⠀⠀⠀⢉⡷⠀⡌⠉⠋⡇⠠⣏⠈⢁⣦⣿⣦⠀⠀⠀⠀⠀⠀                 ")
-    print("        ⠀⠀⠀⠀⠀⠉⣁⠀⠉⠉⠉⠙⠛⠛⠒⠚⠳⠤⢼⣤⣠⠤⣮⣠⣤⣼⠦⢤⣤⣿⠤⠾⠓⠒⠛⢓⠛⠉⠉⠉⠀⠈⠉⠀⠀⠀⠀⠀⠀                ")
-    print("")
-
-    fabricks = Database("fabricks")
-    fabricks.drop()
-    for s in steps:
-        step = BaseStep(s)
-        step.drop()
-
-    tmp = FABRICKS_STORAGE.joinpath("tmp")
-    tmp.rm()
-
-    checkpoint = FABRICKS_STORAGE.joinpath("checkpoints")
-    checkpoint.rm()
-
-    schema = FABRICKS_STORAGE.joinpath("schemas")
-    schema.rm()
-
-    schedule = FABRICKS_STORAGE.joinpath("schedules")
-    schedule.rm()
-
-    fabricks.create()
-
-    deploy.tables(drop=True)
-    deploy.udfs()
-
-    for s in steps:
-        step = BaseStep(s)
-        step.create()
-
-    deploy.views()
-
-    create_or_replace_views()
-    create_or_replace_schedules_views()

fabricks/core/scripts/stats.py

@@ -1,51 +0,0 @@
-from pyspark.sql.types import Row
-
-from fabricks.cdc import NoCDC
-from fabricks.context import SPARK
-from fabricks.core.jobs.base._types import Steps
-from fabricks.utils.helpers import concat_dfs, run_in_parallel
-
-
-def collect_stats():
-    def _collect_tables(s: str):
-        df_table = SPARK.sql(f"show tables in {s}")
-        df_view = SPARK.sql(f"show views in {s}")
-
-        cond = [df_table.tableName == df_view.viewName]
-        df_table = df_table.join(df_view, cond, how="left_anti")
-
-        return df_table
-
-    dfs = run_in_parallel(_collect_tables, Steps, workers=8)
-    df_table = concat_dfs(dfs)
-    assert df_table is not None
-
-    def _collect_stats(row: Row):
-        table = row["tableName"]
-        database = row["database"]
-        job = f"{database}.{table}"
-
-        desc = SPARK.sql(f"describe detail {job}").collect()[0]
-        bytes = desc["sizeInBytes"]
-        files = desc["numFiles"]
-
-        df = SPARK.sql(
-            f"""
-            select
-              '{database}' as step,
-              md5('{job}') as job_id,
-              cast({bytes} as long) as bytes,
-              cast({files} as long) as `files`,
-              cast(count(*) as long) as `rows`
-            from
-              {job}
-            """
-        )
-
-        return df
-
-    dfs = run_in_parallel(_collect_stats, df_table, workers=64)
-    df = concat_dfs(dfs)
-    assert df is not None
-
-    NoCDC("fabricks", "statistics").overwrite(df)

fabricks/core/scripts/steps.py

@@ -1,26 +0,0 @@
-from typing import Iterable
-
-from fabricks.cdc import NoCDC
-from fabricks.context import SPARK
-from fabricks.context.runtime import BRONZE, GOLD, SILVER
-
-
-def collect_steps():
-    steps = []
-
-    def _collect(expand: str, iterable: Iterable):
-        for i in iterable:
-            steps.append(
-                {
-                    "expand": expand,
-                    "step": i.get("name"),
-                    "order": i.get("options", {}).get("order", 0),
-                },
-            )
-
-    _collect("bronze", BRONZE)
-    _collect("silver", SILVER)
-    _collect("gold", GOLD)
-
-    df = SPARK.createDataFrame(steps)
-    NoCDC("fabricks", "steps").overwrite(df)