fabricks 3.0.11__py3-none-any.whl

fabricks/api/__init__.py

@@ -0,0 +1,11 @@
+from fabricks.api.context import init_spark_session
+from fabricks.api.core import get_job, get_jobs, get_step
+from fabricks.api.deploy import Deploy
+
+__all__ = [
+    "init_spark_session",
+    "get_job",
+    "get_jobs",
+    "get_step",
+    "Deploy",
+]

fabricks/api/cdc/__init__.py

@@ -0,0 +1,6 @@
+from fabricks.api.cdc.nocdc import NoCDC
+from fabricks.api.cdc.scd1 import SCD1
+from fabricks.api.cdc.scd2 import SCD2
+from fabricks.cdc.cdc import CDC
+
+__all__ = ["CDC", "SCD1", "SCD2", "NoCDC"]

fabricks/api/cdc/nocdc.py

@@ -0,0 +1,3 @@
+from fabricks.cdc.nocdc import NoCDC
+
+__all__ = ["NoCDC"]

fabricks/api/cdc/scd1.py

@@ -0,0 +1,3 @@
+from fabricks.cdc.scd1 import SCD1
+
+__all__ = ["SCD1"]

fabricks/api/cdc/scd2.py

@@ -0,0 +1,3 @@
+from fabricks.cdc.scd2 import SCD2
+
+__all__ = ["SCD2"]

fabricks/api/context.py

@@ -0,0 +1,27 @@
+from fabricks.context import BRONZE, DBUTILS, GOLD, SECRET_SCOPE, SILVER, SPARK, init_spark_session, pprint_runtime
+from fabricks.core.jobs.base._types import Bronzes, Golds, Silvers, Steps
+
+# step
+BRONZES = Bronzes
+SILVERS = Silvers
+GOLDS = Golds
+STEPS = Steps
+
+
+__all__ = [
+    "BRONZE",
+    "Bronzes",
+    "BRONZES",
+    "DBUTILS",
+    "GOLD",
+    "Golds",
+    "GOLDS",
+    "init_spark_session",
+    "pprint_runtime",
+    "SECRET_SCOPE",
+    "SILVER",
+    "Silvers",
+    "SILVERS",
+    "SPARK",
+    "STEPS",
+]

fabricks/api/core.py

@@ -0,0 +1,4 @@
+from fabricks.core.jobs import BaseJob, get_job, get_jobs
+from fabricks.core.steps import get_step
+
+__all__ = ["BaseJob", "get_job", "get_jobs", "get_step"]

fabricks/api/deploy.py

@@ -0,0 +1,3 @@
+from fabricks.deploy import Deploy
+
+__all__ = ["Deploy"]

fabricks/api/exceptions.py

@@ -0,0 +1,19 @@
+from fabricks.core.jobs.base.exception import (
+    CheckException,
+    CheckWarning,
+    PostRunCheckException,
+    PostRunCheckWarning,
+    PreRunCheckException,
+    PreRunCheckWarning,
+    SkipRunCheckWarning,
+)
+
+__all__ = [
+    "CheckException",
+    "CheckWarning",
+    "PreRunCheckException",
+    "PostRunCheckException",
+    "PreRunCheckWarning",
+    "PostRunCheckWarning",
+    "SkipRunCheckWarning",
+]

fabricks/api/extenders.py

@@ -0,0 +1,3 @@
+from fabricks.core.extenders import extender
+
+__all__ = ["extender"]

fabricks/api/job_schema.py

@@ -0,0 +1,3 @@
+from fabricks.core.job_schema import get_job_schema, print_job_schema
+
+__all__ = ["get_job_schema", "print_job_schema"]

fabricks/api/log.py

@@ -0,0 +1,3 @@
+from fabricks.context.log import DEFAULT_LOGGER, send_message_to_channel
+
+__all__ = ["DEFAULT_LOGGER", "send_message_to_channel"]

fabricks/api/masks.py

@@ -0,0 +1,3 @@
+from fabricks.core.masks import register_all_masks, register_mask
+
+__all__ = ["register_all_masks", "register_mask"]

fabricks/api/metastore/__init__.py

@@ -0,0 +1,10 @@
+from fabricks.api.metastore.database import Database
+from fabricks.api.metastore.table import Table
+from fabricks.api.metastore.view import View, create_or_replace_view
+
+__all__ = [
+    "create_or_replace_view",
+    "Database",
+    "Table",
+    "View",
+]

fabricks/api/metastore/database.py

@@ -0,0 +1,3 @@
+from fabricks.metastore import Database
+
+__all__ = ["Database"]

fabricks/api/metastore/table.py

@@ -0,0 +1,3 @@
+from fabricks.metastore import Table
+
+__all__ = ["Table"]

fabricks/api/metastore/view.py

@@ -0,0 +1,6 @@
+from fabricks.metastore import View
+
+create_or_replace_view = View.create_or_replace
+
+
+__all__ = ["View", "create_or_replace_view"]

fabricks/api/notebooks/cluster.py

@@ -0,0 +1,6 @@
+# Databricks notebook source
+from databricks.sdk.runtime import dbutils
+
+# COMMAND ----------
+
+dbutils.notebook.exit(value="exit (0)")  # type: ignore

fabricks/api/notebooks/initialize.py

@@ -0,0 +1,42 @@
+# Databricks notebook source
+# MAGIC %run ./add_missing_modules
+
+# COMMAND ----------
+
+from databricks.sdk.runtime import dbutils, display
+
+from fabricks.core.schedules import generate
+
+# COMMAND ----------
+
+dbutils.widgets.text("schedule", "---")
+
+# COMMAND ----------
+
+schedule = dbutils.widgets.get("schedule")
+assert schedule != "---", "no schedule provided"
+
+# COMMAND ----------
+
+print(schedule)
+
+# COMMAND ----------
+
+schedule_id, job_df, dependency_df = generate(schedule=schedule)
+
+# COMMAND ----------
+
+display(job_df)
+
+# COMMAND ----------
+
+display(dependency_df)
+
+# COMMAND ----------
+
+dbutils.jobs.taskValues.set(key="schedule_id", value=schedule_id)
+dbutils.jobs.taskValues.set(key="schedule", value=schedule)
+
+# COMMAND ----------
+
+dbutils.notebook.exit(value="exit (0)")  # type: ignore

fabricks/api/notebooks/process.py

@@ -0,0 +1,54 @@
+# Databricks notebook source
+# MAGIC %run ./add_missing_modules
+
+# COMMAND ----------
+
+from databricks.sdk.runtime import dbutils
+from pyspark.errors.exceptions.base import IllegalArgumentException
+
+from fabricks.core.schedules import process
+
+# COMMAND ----------
+
+dbutils.widgets.text("step", "---")
+dbutils.widgets.text("schedule_id", "---")
+
+# COMMAND ----------
+
+try:
+    schedule_id = dbutils.jobs.taskValues.get(taskKey="initialize", key="schedule_id")
+except (TypeError, IllegalArgumentException, ValueError):
+    schedule_id = dbutils.widgets.get("schedule_id")
+    assert schedule_id != "---", "no schedule_id provided"
+
+assert schedule_id is not None
+
+# COMMAND ----------
+
+step = dbutils.widgets.get("step")
+assert step != "---", "no step provided"
+
+# COMMAND ----------
+
+schedule = dbutils.widgets.get("schedule")
+assert schedule != "---", "no schedule provided"
+
+# COMMAND ----------
+
+print(schedule_id)
+
+# COMMAND ----------
+
+print(step)
+
+# COMMAND ----------
+
+print(schedule)
+
+# COMMAND ----------
+
+process(schedule_id=schedule_id, schedule=schedule, step=step)
+
+# COMMAND ----------
+
+dbutils.notebook.exit(value="exit (0)")  # type: ignore

fabricks/api/notebooks/run.py

@@ -0,0 +1,59 @@
+# Databricks notebook source
+# MAGIC %run ./add_missing_modules
+
+# COMMAND ----------
+
+import json
+
+from databricks.sdk.runtime import dbutils
+
+from fabricks.core.schedules import run
+
+# COMMAND ----------
+
+dbutils.widgets.text("step", "---")
+dbutils.widgets.text("job_id", "---")
+dbutils.widgets.text("job", "--")
+dbutils.widgets.text("schedule_id", "---")
+dbutils.widgets.text("schedule", "---")
+
+# COMMAND ----------
+
+step = dbutils.widgets.get("step")
+assert step != "---"
+
+# COMMAND ----------
+
+job_id = dbutils.widgets.get("job_id")
+assert job_id != "---"
+
+# COMMAND ----------
+
+job = dbutils.widgets.get("job")
+assert job != "---"
+
+# COMMAND ----------
+
+schedule_id = dbutils.widgets.get("schedule_id")
+assert schedule_id != "---"
+
+# COMMAND ----------
+
+schedule = dbutils.widgets.get("schedule")
+assert schedule != "---"
+
+# COMMAND ----------
+
+try:
+    context = json.loads(dbutils.notebook.entry_point.getDbutils().notebook().getContext().toJson())  # type: ignore
+    notebook_id = context.get("tags").get("jobId")
+except:  # noqa: E722
+    notebook_id = None
+
+# COMMAND ----------
+
+run(step=step, job_id=job_id, schedule_id=schedule_id, schedule=schedule, notebook_id=notebook_id)
+
+# COMMAND ----------
+
+dbutils.notebook.exit(value="exit (0)")  # type: ignore

fabricks/api/notebooks/schedule.py

@@ -0,0 +1,75 @@
+# Databricks notebook source
+# MAGIC %run ./add_missing_modules
+
+# COMMAND ----------
+
+from logging import DEBUG
+from typing import Any, cast
+
+from databricks.sdk.runtime import dbutils, display, spark
+
+from fabricks.context import PATH_NOTEBOOKS
+from fabricks.context.log import DEFAULT_LOGGER
+from fabricks.core import get_step
+from fabricks.core.jobs.base._types import TStep
+from fabricks.core.schedules import generate, terminate
+from fabricks.utils.helpers import run_in_parallel, run_notebook
+
+# COMMAND ----------
+
+DEFAULT_LOGGER.setLevel(DEBUG)
+
+# COMMAND ----------
+
+dbutils.widgets.text("schedule", "---")
+
+# COMMAND ----------
+
+schedule = dbutils.widgets.get("schedule")
+assert schedule != "---", "no schedule provided"
+
+# COMMAND ----------
+
+schedule_id, job_df, dependency_df = generate(schedule=schedule)
+
+# COMMAND ----------
+
+print(schedule_id)
+
+# COMMAND ----------
+
+display(job_df)
+
+# COMMAND ----------
+
+display(dependency_df)
+
+# COMMAND ----------
+steps = [row.step for row in spark.sql("select step from {df} group by step", df=job_df).collect()]
+
+# COMMAND ----------
+
+
+def _schedule(task: Any):
+    step = get_step(step=cast(TStep, task))
+    run_notebook(
+        PATH_NOTEBOOKS.joinpath("process"),
+        timeout=step.timeouts.step,
+        step=task,
+        schedule_id=schedule_id,
+        schedule=schedule,
+        workers=step.workers,
+    )
+
+
+# COMMAND ----------
+
+run_in_parallel(_schedule, steps)
+
+# COMMAND ----------
+
+terminate(schedule_id=schedule_id)
+
+# COMMAND ----------
+
+dbutils.notebook.exit(value="exit (0)")  # type: ignore

fabricks/api/notebooks/terminate.py

@@ -0,0 +1,31 @@
+# Databricks notebook source
+# MAGIC %run ./add_missing_modules
+
+# COMMAND ----------
+
+from databricks.sdk.runtime import dbutils
+from pyspark.errors.exceptions.base import IllegalArgumentException
+
+from fabricks.core.schedules import terminate
+
+# COMMAND ----------
+
+try:
+    schedule_id = dbutils.jobs.taskValues.get(taskKey="initialize", key="schedule_id")
+except (TypeError, IllegalArgumentException, ValueError):
+    schedule_id = dbutils.widgets.get("schedule_id")
+    assert schedule_id != "---", "no schedule_id provided"
+
+assert schedule_id is not None
+
+# COMMAND ----------
+
+print(schedule_id)
+
+# COMMAND ----------
+
+terminate(schedule_id=schedule_id)
+
+# COMMAND ----------
+
+dbutils.notebook.exit(value="exit (0)")  # type: ignore

fabricks/api/parsers.py

@@ -0,0 +1,3 @@
+from fabricks.core.parsers import BaseParser, ParserOptions, parser
+
+__all__ = ["BaseParser", "ParserOptions", "parser"]

fabricks/api/schedules.py

@@ -0,0 +1,3 @@
+from fabricks.core.schedules import create_or_replace_view, create_or_replace_views, generate, process, terminate
+
+__all__ = ["create_or_replace_view", "create_or_replace_views", "terminate", "generate", "process"]

fabricks/api/udfs.py

@@ -0,0 +1,3 @@
+from fabricks.core.udfs import register_all_udfs, register_udf, udf
+
+__all__ = ["udf", "register_all_udfs", "register_udf"]

fabricks/api/utils.py

@@ -0,0 +1,9 @@
+from fabricks.utils.helpers import concat_dfs, concat_ws, run_in_parallel
+from fabricks.utils.path import Path
+
+__all__ = [
+    "concat_dfs",
+    "concat_ws",
+    "Path",
+    "run_in_parallel",
+]

fabricks/api/version.py

@@ -0,0 +1,3 @@
+import importlib.metadata
+
+FABRICKS_VERSION = importlib.metadata.version("fabricks")

fabricks/api/views.py

@@ -0,0 +1,6 @@
+from fabricks.core.views import create_or_replace_view, create_or_replace_views
+
+__all__ = [
+    "create_or_replace_view",
+    "create_or_replace_views",
+]

fabricks/cdc/__init__.py

@@ -0,0 +1,14 @@
+from fabricks.cdc.base import AllowedChangeDataCaptures, BaseCDC
+from fabricks.cdc.cdc import CDC
+from fabricks.cdc.nocdc import NoCDC
+from fabricks.cdc.scd1 import SCD1
+from fabricks.cdc.scd2 import SCD2
+
+__all__ = [
+    "BaseCDC",
+    "CDC",
+    "AllowedChangeDataCaptures",
+    "NoCDC",
+    "SCD1",
+    "SCD2",
+]

fabricks/cdc/base/__init__.py

@@ -0,0 +1,4 @@
+from fabricks.cdc.base._types import AllowedChangeDataCaptures
+from fabricks.cdc.base.cdc import BaseCDC
+
+__all__ = ["BaseCDC", "AllowedChangeDataCaptures"]

fabricks/cdc/base/_types.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+from typing import Literal, Union
+
+from pyspark.sql import DataFrame
+
+from fabricks.metastore.table import Table
+
+AllowedChangeDataCaptures = Literal["nocdc", "scd1", "scd2"]
+AllowedSources = Union[DataFrame, Table, str]

fabricks/cdc/base/cdc.py

@@ -0,0 +1,5 @@
+from fabricks.cdc.base.merger import Merger
+
+
+class BaseCDC(Merger):
+    pass