fabricks 2024.7.1.5__py3-none-any.whl

fabricks/core/parsers/get_parser.py

@@ -0,0 +1,25 @@
+import sys
+from importlib.util import spec_from_file_location
+from typing import Optional
+
+from fabricks.context import PATH_PARSERS
+from fabricks.core.parsers.base import PARSERS, BaseParser
+from fabricks.core.parsers.types import ParserOptions
+
+
+def get_parser(name: str, parser_options: Optional[ParserOptions] = None) -> BaseParser:
+    if name not in ["json", "parquet", "avro", "csv", "tsv", "delta", "table"]:
+        sys.path.append(PATH_PARSERS.string)
+
+        path = PATH_PARSERS.join(name).append(".py")
+        assert path.exists(), f"parser not found ({path})"
+        spec = spec_from_file_location(name, path.string)
+        assert spec, f"parser not found ({path})"
+        spec.loader.load_module()  # type: ignore
+
+        parser = PARSERS[name](parser_options)
+    else:
+        parser = BaseParser(parser_options, name)
+
+    assert parser
+    return parser

fabricks/core/parsers/types.py

@@ -0,0 +1,6 @@
+from typing import Optional, TypedDict
+
+
+class ParserOptions(TypedDict):
+    file_format: Optional[str]
+    read_options: Optional[dict[str, str]]

fabricks/core/schedules.py

@@ -0,0 +1,89 @@
+from typing import List, Optional, TypedDict
+
+from databricks.sdk.runtime import spark
+from pyspark.sql import DataFrame
+
+from fabricks.context import PATH_SCHEDULES
+from fabricks.context.log import Logger
+from fabricks.core.jobs.base.types import TStep
+from fabricks.utils.read.read_yaml import read_yaml
+from fabricks.utils.schema import get_schema_for_type
+from fabricks.utils.sqlglot import fix as fix_sql
+
+
+class Options(TypedDict):
+    steps: Optional[List[TStep]]
+    tag: Optional[str]
+    view: Optional[str]
+    variables: Optional[dict[str, str]]
+
+
+class Schedule(TypedDict):
+    name: str
+    options: Options
+
+
+def get_schedules() -> DataFrame:
+    schema = get_schema_for_type(Schedule)
+    df = read_yaml(PATH_SCHEDULES, root="schedule", schema=schema)
+    assert df, "no schedules found"
+    return df
+
+
+def get_schedule(name: str) -> DataFrame:
+    df = get_schedules()
+    df = df.where(f"name == '{name}'")
+    assert not df.isEmpty(), "schedule not found"
+    assert df.count() == 1, "schedule duplicated"
+    return df
+
+
+def _create_or_replace_view(name: str, options: DataFrame):
+    step = "-- no step provided"
+    tag = "-- no tag provided"
+    view = "-- no view provided"
+
+    if options.steps is not None:
+        steps = [f"'{s}'" for s in options.steps]  # type: ignore
+        step = f"and j.step in ({', '.join(steps)})"
+    if options.tag is not None:
+        tag = f"and array_contains(j.tags, '{options.tag}')"
+    if options.view is not None:
+        view = f"inner join fabricks.{options.view} v on j.job_id = v.job_id"
+
+    sql = f"""
+    create or replace view fabricks.{name}_schedule
+    as
+    select
+        j.*
+    from
+        fabricks.jobs j
+        {view}
+    where
+        true 
+        {step}
+        {tag}
+        and j.type not in ('manual')
+    """
+    sql = fix_sql(sql)
+    Logger.debug(f"schedule - %sql\n---\n{sql}\n---")
+
+    spark.sql(sql)
+
+
+def create_or_replace_view(name: str):
+    df = get_schedule(name=name)
+    for row in df.collect():
+        try:
+            _create_or_replace_view(row.name, row.options)
+        except Exception:
+            Logger.exception(f"schedule - {row.name} not created nor replaced")
+
+
+def create_or_replace_views():
+    df = get_schedules()
+    for row in df.collect():
+        try:
+            _create_or_replace_view(row.name, row.options)
+        except Exception:
+            Logger.exception(f"schedule - {row.name} not created nor replaced")

fabricks/core/scripts/__init__.py

@@ -0,0 +1,13 @@
+from fabricks.core.scripts.generate import generate
+from fabricks.core.scripts.optimize import optimize
+from fabricks.core.scripts.process import process
+from fabricks.core.scripts.terminate import terminate
+from fabricks.core.scripts.vacuum import vacuum
+
+__all__ = [
+    "process",
+    "optimize",
+    "generate",
+    "terminate",
+    "vacuum",
+]

fabricks/core/scripts/armageddon.py

@@ -0,0 +1,82 @@
+from typing import List, Optional, Union, cast
+
+from fabricks.context import FABRICKS_STORAGE
+from fabricks.context.log import Logger
+from fabricks.core.deploy import deploy
+from fabricks.core.jobs.base.types import Steps, TStep
+from fabricks.core.schedules import create_or_replace_views as create_or_replace_schedules_views
+from fabricks.core.steps.base import BaseStep
+from fabricks.core.views import create_or_replace_views
+from fabricks.metastore.database import Database
+
+
+def armageddon(steps: Optional[Union[TStep, List[TStep], str, List[str]]]):
+    if steps is None:
+        steps = Steps
+    assert steps is not None
+
+    if isinstance(steps, str):
+        steps = [cast(TStep, steps)]
+    elif isinstance(steps, List):
+        steps = [cast(TStep, s) for s in steps]
+    elif isinstance(steps, TStep):
+        steps = [steps]
+
+    Logger.warning("armageddon")
+    print("")
+    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣀⡤⠤⠴⠾⠋⠉⠛⢾⡏⠙⠿⠦⠤⢤⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                    ")
+    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⡤⢶⣿⠉⢀⣀⡠⠆⠀⠀⠀⠀⠀⠀⠀⢤⣀⣀⠈⢹⣦⢤⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                    ")
+    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣴⣿⠁⢋⡙⠁⠀⡝⠀⠀⠀⠀⣀⡸⠋⠁⠀⠀⠹⡀⠀⠈⠈⠆⢹⢦⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                   ")
+    print("        ⠀⠀⠀⠀⠀⠀⠀⢀⣠⣤⣿⣁⡡⣴⡏⠀⠀⠀⢀⠀⢧⣀⠄⠀⠀⠀⣀⣰⠆⢀⠁⠀⠀⢈⣶⡤⣀⢹⣦⣄⡀⠀⠀⠀⠀⠀⠀⠀⠀                  ")
+    print("        ⠀⠀⠀⠀⠀⣠⢴⠟⢁⡝⠀⠁⠀⠃⠉⠀⠀⠘⣯⠀⡀⠾⣤⣄⣠⢤⠾⠄⠀⣸⠖⠀⠀⠈⠀⠃⠀⠀⠹⡄⠙⣶⢤⡀⠀⠀⠀⠀⠀                 ")
+    print("        ⠀⠀⠀⣠⠾⡇⠈⣀⡞⠀⠀⠀⠀⡀⠀⢀⣠⣄⣇⠀⣳⠴⠃⠀⠀⠀⠣⢴⠉⣰⣇⣀⣀⠀⠀⡄⠀⠀⠀⢹⣄⡘⠈⡷⣦⠀⠀⠀⠀                 ")
+    print("        ⢠⠞⠉⢻⡄⠀⠀⠈⠙⠀⠀⠀⠀⠙⣶⣏⣤⣤⠟⠉⠁⠀⠀⠀⠀⠀⠀⠀⠉⠙⢦⣱⣌⣷⠊⠀⠀⠀⠀⠈⠁⠀⠀⠀⡝⠉⠻⣄⠀                 ")
+    print("        ⠛⢀⡠⢼⡇⠀⠀⢀⡄⠀⢀⣀⡽⠚⠁⠀⠀⠀⢠⡀⢠⣀⠠⣔⢁⡀⠀⣄⠀⡄⠀⠀⠀⠈⠑⠺⣄⡀⠀⠠⡀⠀⠀⢠⡧⠄⠀⠘⢧                ")
+    print("        ⡶⠋⠀⠀⠈⣠⣈⣩⠗⠒⠋⠀⠀⠀⠀⣀⣠⣆⡼⣷⣞⠛⠻⡉⠉⡟⠒⡛⣶⠧⣀⣀⣀⠀⠀⠀⠀⠈⠓⠺⢏⣉⣠⠋⠀⠀⠀⢢⣸                ")
+    print("        ⠇⠐⠤⠤⠖⠁⣿⣀⣀⠀⠀⠀⠀⠀⠉⠁⠈⠉⠙⠛⢿⣷⡄⢣⡼⠀⣾⣿⠧⠒⠓⠚⠛⠉⠀⠀⠀⠀⠀⢀⣀⣾⡉⠓⠤⡤⠄⠸⢿                ")
+    print("        ⣆⣤⠀⠀⠠⠀⠈⠓⠈⠓⠤⡀⠀⠀⠀⠀⠀⠀⠀⠀⠈⣿⣿⢸⠀⢸⣿⠇⠀⠀⠀⠀⠀⠀⠀⠀⢀⡤⠒⠁⠰⠃⠀⠠⠀⠀⢀⣀⠞                  ")
+    print("        ⠀⠉⠓⢲⣄⡈⢀⣠⠀⠀⠀⡸⠶⠂⠀⠀⢀⠀⠀⠤⠞⢻⡇⠀⠀⢘⡟⠑⠤⠄⠀⢀⠀⠀⠐⠲⢿⡀⠀⠀⢤⣀⢈⣀⡴⠖⠋⠀⠀                 ")
+    print("        ⠀⠀⠀⠀⠈⠉⠉⠙⠓⠒⣾⣁⣀⣴⠀⣀⠙⢧⠂⢀⣆⣀⣷⣤⣀⣾⣇⣀⡆⠀⢢⠛⢁⠀⢰⣀⣀⣹⠒⠒⠛⠉⠉⠉⠀⠀⠀⠀⠀                ")
+    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠉⠁⠈⠉⠉⠛⠉⠙⠉⠀⠀⣿⡟⣿⣿⠀⠀⠈⠉⠉⠙⠋⠉⠉⠀⠉⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                   ")
+    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣿⡇⢻⣿⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                      ")
+    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣀⣤⣶⣾⣿⣿⠁⠀⢹⡛⣟⡶⢤⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                    ")
+    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣴⠛⢯⣽⡟⢿⣿⠛⠿⠳⠞⠻⣿⠻⣆⢽⠟⣶⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                   ")
+    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠛⠃⠲⠯⠴⣦⣼⣷⣤⣤⣶⣤⣩⡧⠽⠷⠐⠛⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                   ")
+    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣿⡇⠀⣿⡆⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                     ")
+    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣀⣄⡀⢀⣀⣠⡾⡿⢡⢐⠻⣿⣄⣀⡀⠀⣀⣄⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                  ")
+    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣤⢴⡏⠁⠀⠝⠉⣡⠟⣰⠃⢸⣿⠀⣷⠙⢧⡉⠻⡅⠀⠙⡷⢤⣀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                 ")
+    print("        ⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣴⡟⠀⠈⣿⢄⡴⠞⠻⣄⣰⣡⠤⣞⣸⡤⢬⣧⣀⡿⠛⠦⣤⣶⡃⠀⢹⣦⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀                 ")
+    print("        ⠀⠀⠀⠀⠀⠀⢀⣴⣶⡿⠃⠉⢺⠁⠙⠒⠀⠀⣠⡉⠀⠉⠚⠉⠉⠑⠈⠀⠈⣧⠀⠀⠒⠋⠀⡹⠋⠀⢻⡶⠶⡄⠀⠀⠀⠀⠀⠀⠀                 ")
+    print("        ⠀⠀⠀⠀⠀⣠⣾⣿⣇⠁⢈⡦⠀⡍⠋⠁⡀⠸⡋⠀⠀⠀⢘⠏⠉⡏⠀⠀⠀⢉⡷⠀⡌⠉⠋⡇⠠⣏⠈⢁⣦⣿⣦⠀⠀⠀⠀⠀⠀                 ")
+    print("        ⠀⠀⠀⠀⠀⠉⣁⠀⠉⠉⠉⠙⠛⠛⠒⠚⠳⠤⢼⣤⣠⠤⣮⣠⣤⣼⠦⢤⣤⣿⠤⠾⠓⠒⠛⢓⠛⠉⠉⠉⠀⠈⠉⠀⠀⠀⠀⠀⠀                ")
+    print("")
+
+    fabricks = Database("fabricks")
+    fabricks.drop()
+    for s in steps:
+        step = BaseStep(s)
+        step.drop()
+
+    tmp = FABRICKS_STORAGE.join("tmp")
+    tmp.rm()
+
+    checkpoint = FABRICKS_STORAGE.join("checkpoints")
+    checkpoint.rm()
+
+    schema = FABRICKS_STORAGE.join("schemas")
+    schema.rm()
+
+    schedule = FABRICKS_STORAGE.join("schedules")
+    schedule.rm()
+
+    fabricks.create()
+
+    deploy.tables(drop=True)
+    for s in steps:
+        step = BaseStep(s)
+        step.create()
+
+    deploy.views()
+
+    create_or_replace_views()
+    create_or_replace_schedules_views()

fabricks/core/scripts/generate.py

@@ -0,0 +1,20 @@
+from typing import Tuple
+
+from pyspark.sql import DataFrame
+
+from fabricks.core.dags.generator import DagGenerator
+
+
+def generate(schedule: str) -> Tuple[str, DataFrame, DataFrame]:
+    """
+    Generate a schedule, job dataframe, and dependency dataframe based on the given schedule.
+
+    Args:
+        schedule (str): The schedule to generate from.
+
+    Returns:
+        Tuple[str, DataFrame, DataFrame]: A tuple containing the schedule ID, job dataframe, and dependency dataframe.
+    """
+    g = DagGenerator(schedule)
+    schedule_id, job_df, dep_df = g.generate()
+    return schedule_id, job_df, dep_df

fabricks/core/scripts/job_schema.py

@@ -0,0 +1,28 @@
+from dataclasses import dataclass
+from typing import List
+
+from fabricks.core.jobs.base.types import JobConf
+from fabricks.utils.schema import get_json_schema_for_type
+
+
+def get_job_schema() -> str:
+    import json
+
+    @dataclass
+    class JobWrapper:
+        job: JobConf
+
+    sc = get_json_schema_for_type(List[JobWrapper])
+    defs: dict[str, dict] = sc["$defs"]
+    removals = [("Job", "job_id"), ("Job", "table")]
+
+    for key, defi in defs.items():
+        for ent, prop in removals:
+            if key.startswith(ent) and prop in defi["properties"]:
+                req: List[str] = defi["required"]
+                req.remove(prop)  # not defined in yaml
+                jobprops: dict = defi["properties"]
+                jobprops.pop(prop)
+
+    j = json.dumps(sc, indent=4)
+    return j

fabricks/core/scripts/optimize.py

@@ -0,0 +1,45 @@
+from typing import Optional
+
+from databricks.sdk.runtime import spark
+from pyspark.sql import Row
+
+from fabricks.core.jobs.get_job import get_job
+from fabricks.utils.helpers import run_in_parallel
+
+
+def optimize(schedule_id: Optional[str] = None):
+    """
+    Cleans the Fabricks jobs by vacuuming and optimizing the tables.
+
+    Args:
+        schedule_id (Optional[str]): The schedule ID to filter the jobs. If None, all jobs will be cleaned.
+
+    Returns:
+        None
+    """
+    if schedule_id is not None:
+        df = spark.sql(
+            f"""
+            select
+              j.step,
+              j.job_id
+            from
+              fabricks.logs l
+              inner join fabricks.jobs j on l.job_id = j.job_id
+            where 
+              true
+              and not j.mode = 'memory'
+              and l.schedule_id = '{schedule_id}'
+            group by
+              j.step,
+              j.job_id
+            """
+        )
+    else:
+        df = spark.sql("select * from fabricks.jobs where not mode = 'memory'")
+
+    def _optimize(row: Row):
+        job = get_job(step=row["step"], job_id=row["job_id"])
+        job.optimize()
+
+    run_in_parallel(_optimize, df, 16)

fabricks/core/scripts/process.py

@@ -0,0 +1,9 @@
+from typing import Union
+
+from fabricks.core.dags.processor import DagProcessor
+from fabricks.core.jobs.base.types import TStep
+
+
+def process(schedule_id: str, schedule: str, step: Union[TStep, str]):
+    p = DagProcessor(schedule_id=schedule_id, schedule=schedule, step=step)
+    p.process()

fabricks/core/scripts/stats.py

@@ -0,0 +1,48 @@
+from databricks.sdk.runtime import spark
+from pyspark.sql import Row
+
+from fabricks.cdc import NoCDC
+from fabricks.core.jobs.base.types import Steps
+from fabricks.utils.helpers import concat_dfs, run_in_parallel
+
+
+def collect_stats():
+    def _collect_tables(s: str):
+        df_table = spark.sql(f"show tables in {s}")
+        df_view = spark.sql(f"show views in {s}")
+
+        cond = [df_table.tableName == df_view.viewName]
+        df_table = df_table.join(df_view, cond, how="left_anti")
+
+        return df_table
+
+    dfs = run_in_parallel(_collect_tables, Steps, workers=8)
+    df_table = concat_dfs(dfs)
+
+    def _collect_stats(row: Row):
+        table = row["tableName"]
+        database = row["database"]
+        job = f"{database}.{table}"
+
+        desc = spark.sql(f"describe detail {job}").collect()[0]
+        bytes = desc["sizeInBytes"]
+        files = desc["numFiles"]
+
+        df = spark.sql(
+            f"""
+            select
+              '{database}' as step,
+              md5('{job}') as job_id,
+              cast({bytes} as long) as bytes,
+              cast({files} as long) as `files`,
+              cast(count(*) as long) as `rows`
+            from
+              {job}
+            """
+        )
+
+        return df
+
+    dfs = run_in_parallel(_collect_stats, df_table, workers=64)
+    df = concat_dfs(dfs)
+    NoCDC("fabricks", "statistics").overwrite(df)

fabricks/core/scripts/steps.py

@@ -0,0 +1,27 @@
+from typing import Iterable
+
+from databricks.sdk.runtime import spark
+
+from fabricks.cdc import NoCDC
+from fabricks.context.runtime import BRONZE, GOLD, SILVER
+
+
+def collect_steps():
+    steps = []
+
+    def _collect(extend: str, iterable: Iterable):
+        for i in iterable:
+            steps.append(
+                {
+                    "extend": extend,
+                    "step": i.get("name"),
+                    "order": i.get("options", {}).get("order", 0),
+                },
+            )
+
+    _collect("bronze", BRONZE)
+    _collect("silver", SILVER)
+    _collect("gold", GOLD)
+
+    df = spark.createDataFrame(steps)
+    NoCDC("fabricks", "steps").overwrite(df)

fabricks/core/scripts/terminate.py

@@ -0,0 +1,6 @@
+from fabricks.core.dags.terminator import DagTerminator
+
+
+def terminate(schedule_id: str):
+    t = DagTerminator(schedule_id=schedule_id)
+    t.terminate()

fabricks/core/scripts/vacuum.py

@@ -0,0 +1,45 @@
+from typing import Optional
+
+from databricks.sdk.runtime import spark
+from pyspark.sql import Row
+
+from fabricks.core.jobs.get_job import get_job
+from fabricks.utils.helpers import run_in_parallel
+
+
+def vacuum(schedule_id: Optional[str] = None):
+    """
+    Cleans the Fabricks jobs by vacuuming and optimizing the tables.
+
+    Args:
+        schedule_id (Optional[str]): The schedule ID to filter the jobs. If None, all jobs will be cleaned.
+
+    Returns:
+        None
+    """
+    if schedule_id is not None:
+        df = spark.sql(
+            f"""
+            select
+              j.step,
+              j.job_id
+            from
+              fabricks.logs l
+              inner join fabricks.jobs j on l.job_id = j.job_id
+            where 
+              true
+              and not j.mode = 'memory'
+              and l.schedule_id = '{schedule_id}'
+            group by
+              j.step,
+              j.job_id
+            """
+        )
+    else:
+        df = spark.sql("select * from fabricks.jobs where not mode = 'memory'")
+
+    def _vacuum(row: Row):
+        job = get_job(step=row["step"], job_id=row["job_id"])
+        job.vacuum()
+
+    run_in_parallel(_vacuum, df, 16)

fabricks/core/site_packages.py

@@ -0,0 +1,55 @@
+import sys
+
+from databricks.sdk.runtime import dbutils, spark
+
+from fabricks.context import FABRICKS_STORAGE, PATH_LIBRARIES, PATH_REQUIREMENTS
+from fabricks.context.log import Logger
+from fabricks.utils.pip import pip_requirements, pip_wheel
+
+
+def collect_site_packages(nofail: bool = False):
+    Logger.info(f"collect libraries ({PATH_REQUIREMENTS})")
+
+    dbfs_wheel = "dbfs:/fabricks/wheels"
+    mnt_wheel = "dbfs:/mnt/fabricks/wheels"
+
+    dbutils.fs.mkdirs(dbfs_wheel)
+
+    try:
+        w = FABRICKS_STORAGE.join("wheels")
+        Logger.info(f"pip wheel ({w})")
+        pip_wheel(PATH_REQUIREMENTS, w)
+    except (Exception, ValueError) as e:
+        if nofail:
+            Logger.exception("oops (pip wheel)")
+        else:
+            raise e
+    try:
+        for f in dbutils.fs.ls(mnt_wheel):
+            to = f"{dbfs_wheel}/{f.name}"
+            try:
+                dbutils.fs.ls(to)
+            except Exception:
+                Logger.info(f"uploading {f.name} ({to})")
+                dbutils.fs.cp(f.path, to)
+    except Exception as e:
+        if nofail:
+            Logger.exception("oops (uploading)")
+        else:
+            raise e
+
+    try:
+        p = FABRICKS_STORAGE.join("site-packages")
+        Logger.info(f"pip requirements ({p})")
+        pip_requirements(requirements_path=PATH_REQUIREMENTS, tgt_path=p)
+    except Exception as e:
+        if nofail:
+            Logger.exception("oops (pip requirements)")
+        else:
+            raise e
+
+
+def add_site_packages_to_path():
+    if PATH_LIBRARIES not in sys.path:
+        spark._sc._python_includes.append(PATH_LIBRARIES)  # type: ignore
+        sys.path.append(PATH_LIBRARIES)

fabricks/core/steps/__init__.py

@@ -0,0 +1,4 @@
+from fabricks.core.steps.base import BaseStep
+from fabricks.core.steps.get_step import get_step
+
+__all__ = ["BaseStep", "get_step"]