fabricks 2024.7.1.5__py3-none-any.whl

fabricks/api/__init__.py

@@ -0,0 +1,7 @@
+from fabricks.api.core import get_job, get_jobs, get_step
+
+__all__ = [
+    "get_job",
+    "get_jobs",
+    "get_step",
+]

fabricks/api/cdc/__init__.py

@@ -0,0 +1,6 @@
+from fabricks.api.cdc.nocdc import NoCDC
+from fabricks.api.cdc.scd1 import SCD1
+from fabricks.api.cdc.scd2 import SCD2
+from fabricks.cdc.cdc import CDC
+
+__all__ = ["CDC", "SCD1", "SCD2", "NoCDC"]

fabricks/api/cdc/nocdc.py

@@ -0,0 +1,3 @@
+from fabricks.cdc.nocdc import NoCDC
+
+__all__ = ["NoCDC"]

fabricks/api/cdc/scd1.py

@@ -0,0 +1,3 @@
+from fabricks.cdc.scd1 import SCD1
+
+__all__ = ["SCD1"]

fabricks/api/cdc/scd2.py

@@ -0,0 +1,3 @@
+from fabricks.cdc.scd2 import SCD2
+
+__all__ = ["SCD2"]

fabricks/api/context.py

@@ -0,0 +1,31 @@
+from databricks.sdk.runtime import dbutils, spark
+
+from fabricks.context import BRONZE, GOLD, SECRET_SCOPE, SILVER
+from fabricks.core.jobs.base.types import Bronzes, Golds, Silvers
+
+# spark
+SPARK = spark
+DBUTILS = dbutils
+
+# step
+BRONZES = Bronzes
+SILVERS = Silvers
+GOLDS = Golds
+STEPS = BRONZES + SILVERS + GOLDS
+
+
+__all__ = [
+    "BRONZE",
+    "Bronzes",
+    "BRONZES",
+    "DBUTILS",
+    "GOLD",
+    "Golds",
+    "GOLDS",
+    "SECRET_SCOPE",
+    "SILVER",
+    "Silvers",
+    "SILVERS",
+    "SPARK",
+    "STEPS",
+]

fabricks/api/core.py

@@ -0,0 +1,4 @@
+from fabricks.core.jobs import get_job, get_jobs
+from fabricks.core.steps import get_step
+
+__all__ = ["get_job", "get_jobs", "get_step"]

fabricks/api/extenders.py

@@ -0,0 +1,3 @@
+from fabricks.core.extenders import extender
+
+__all__ = ["extender"]

fabricks/api/log.py

@@ -0,0 +1,3 @@
+from fabricks.context.log import Logger, TableLogger, flush
+
+__all__ = ["Logger", "TableLogger", "flush"]

fabricks/api/metastore/__init__.py

@@ -0,0 +1,10 @@
+from fabricks.api.metastore.database import Database
+from fabricks.api.metastore.table import Table
+from fabricks.api.metastore.view import View, create_or_replace_view
+
+__all__ = [
+    "create_or_replace_view",
+    "Database",
+    "Table",
+    "View",
+]

fabricks/api/metastore/database.py

@@ -0,0 +1,3 @@
+from fabricks.metastore import Database
+
+__all__ = ["Database"]

fabricks/api/metastore/table.py

@@ -0,0 +1,3 @@
+from fabricks.metastore import Table
+
+__all__ = ["Table"]

fabricks/api/metastore/view.py

@@ -0,0 +1,6 @@
+from fabricks.metastore import View
+
+create_or_replace_view = View.create_or_replace
+
+
+__all__ = ["View", "create_or_replace_view"]

fabricks/api/notebooks/cluster.py

@@ -0,0 +1,6 @@
+# Databricks notebook source
+from databricks.sdk.runtime import dbutils
+
+# COMMAND ----------
+
+dbutils.notebook.exit("exit (0)")

fabricks/api/notebooks/deploy/fabricks.py

@@ -0,0 +1,147 @@
+# Databricks notebook source
+# MAGIC %pip install python-dotenv
+
+# COMMAND ----------
+
+import os
+import subprocess
+from pathlib import Path
+
+from databricks.sdk.runtime import dbutils
+from dotenv import load_dotenv
+
+# COMMAND ----------
+
+load_dotenv()
+
+# COMMAND ----------
+
+try:
+    dbutils.fs.ls("mnt/fabricks")
+except Exception:
+    print("fabricks container not mounted")
+
+# COMMAND ----------
+
+try:
+    dbutils.fs.ls("mnt/fabricks/versions")
+except Exception:
+    print("fabricks not found")
+
+# COMMAND ----------
+
+runtime = os.environ.get("path_runtime")
+assert runtime
+if not Path(runtime).exists():
+    print("runtime not found")
+
+# COMMAND ----------
+
+notebooks = os.environ.get("path_notebooks")
+assert notebooks
+if not Path(notebooks).exists():
+    print("notebooks not found")
+
+# COMMAND ----------
+
+scripts = os.environ.get("path_scripts")
+assert scripts
+if not Path(scripts).exists():
+    print("scripts not found")
+
+# COMMAND ----------
+
+abfss_wheels = "abfss://fabricks-wheels@bmsstaprdeuwsoftware.dfs.core.windows.net"
+
+# COMMAND ----------
+
+version = os.environ.get("fabricks_version")
+
+# COMMAND ----------
+
+mnt_version = f"dbfs:/mnt/fabricks/versions/{version}"
+fuse_mnt_version = f"/dbfs/mnt/fabricks/versions/{version}"
+
+# COMMAND ----------
+
+try:
+    for f in dbutils.fs.ls(mnt_version):
+        dbutils.fs.rm(f.path, True)
+except Exception:
+    pass
+
+dbutils.fs.rm(mnt_version, True)
+dbutils.fs.mkdirs(mnt_version)
+
+# COMMAND ----------
+
+dbutils.fs.rm(mnt_version, True)
+dbutils.fs.mkdirs(mnt_version)
+
+# COMMAND ----------
+
+print("copying version to", f"{mnt_version}/version")
+
+for f in dbutils.fs.ls(f"{abfss_wheels}/{version}"):
+    to = f"{mnt_version}/{f.name}"
+
+    try:
+        dbutils.fs.ls(to)
+    except Exception:
+        print("uploading", f.name)
+        dbutils.fs.cp(f.path, to, recurse=True)
+
+# COMMAND ----------
+
+print("pip install requirements.txt")
+
+out = subprocess.run(
+    [
+        "pip",
+        "install",
+        "--no-index",
+        f"--find-links={fuse_mnt_version}/wheels",
+        "-r",
+        f"{fuse_mnt_version}/requirements.txt",
+    ],
+    capture_output=True,
+)
+
+if out.returncode == 1:
+    raise ValueError(out.stderr)
+
+# COMMAND ----------
+
+latest = os.environ.get("latest")
+assert latest
+latest = latest.lower() == "true"
+
+# COMMAND ----------
+
+versions = [version, "2023.12.*"] if latest else [version]
+
+# COMMAND ----------
+
+print("deploy init script")
+
+for v in versions:
+    path = f"{scripts}/{v}.sh"
+
+    with open(path, "w") as sh:
+        sh.write(
+            f"""
+            sudo echo FABRICKS_RUNTIME={runtime} >> /etc/environment
+            sudo echo FABRICKS_NOTEBOOKS={notebooks}/{version} >> /etc/environment
+            sudo echo FABRICKS_VERSION={version} >> /etc/environment
+
+            /databricks/python/bin/pip install --no-index --find-links='{fuse_mnt_version}/wheels' -r '{fuse_mnt_version}/requirements.txt'
+            /databricks/python/bin/pip install sqlglot
+            /databricks/python/bin/pip install jinja2
+            """.replace("            ", "").strip()
+        )
+
+# COMMAND ----------
+
+dbutils.notebook.exit("exit (0)")
+
+# COMMAND ----------

fabricks/api/notebooks/deploy/notebooks.py

@@ -0,0 +1,86 @@
+# Databricks notebook source
+# MAGIC %pip install python-dotenv
+
+# COMMAND ----------
+
+# MAGIC %pip install databricks_cli
+
+# COMMAND ----------
+
+import os
+from importlib import resources
+from pathlib import Path
+
+from databricks.sdk.runtime import dbutils
+from databricks_cli.sdk.api_client import ApiClient  # type: ignore
+from databricks_cli.workspace.api import WorkspaceApi  # type: ignore
+from dotenv import load_dotenv
+
+from fabricks.api import notebooks as src
+
+# COMMAND ----------
+
+load_dotenv()
+
+# COMMAND ----------
+
+host = os.environ.get("databricks_host")
+token = os.environ.get("databricks_token")
+version = os.environ.get("fabricks_version")
+root = os.environ.get("path_notebooks")
+latest = os.environ.get("latest")
+assert latest
+latest = latest.lower() == "true"
+runtime = os.environ.get("path_runtime")
+
+# COMMAND ----------
+
+notebooks = ["cluster", "initialize", "job", "log", "optimize", "run", "terminate"]
+
+# COMMAND ----------
+
+client = ApiClient(host=host, token=token)
+workspace_api = WorkspaceApi(client)
+
+# COMMAND ----------
+
+versions = [version, "latest"] if latest else [version]
+
+# COMMAND ----------
+
+for v in versions:
+    print(f"deploy {v}")
+
+    p = f"{root}/{v}"
+    if not Path(p).exists():
+        os.mkdir(p)
+
+    for n in notebooks:
+        notebook = resources.files(src) / f"{n}.py"
+        workspace_api.import_workspace(
+            source_path=notebook,
+            target_path=f"{p.replace('/Workspace', '')}/{n}.py",
+            fmt="AUTO",
+            language="PYTHON",
+            is_overwrite=True,
+        )
+
+# COMMAND ----------
+
+if latest:
+    p = f"{runtime}/notebooks"
+    for n in notebooks:
+        notebook = resources.files(src) / f"{n}.py"
+        workspace_api.import_workspace(
+            source_path=notebook,
+            target_path=f"{p.replace('/Workspace', '')}/{n}.py",
+            fmt="AUTO",
+            language="PYTHON",
+            is_overwrite=True,
+        )
+
+# COMMAND ----------
+
+dbutils.notebook.exit("exit (0)")
+
+# COMMAND ----------

fabricks/api/notebooks/initialize.py

@@ -0,0 +1,38 @@
+# Databricks notebook source
+from databricks.sdk.runtime import dbutils, display
+
+from fabricks.core.scripts import generate
+
+# COMMAND ----------
+
+dbutils.widgets.text("schedule", "---")
+
+# COMMAND ----------
+
+schedule = dbutils.widgets.get("schedule")
+assert schedule != "---", "no schedule provided"
+
+# COMMAND ----------
+
+print(schedule)
+
+# COMMAND ----------
+
+schedule_id, job_df, dependency_df = generate(schedule=schedule)
+
+# COMMAND ----------
+
+display(job_df)
+
+# COMMAND ----------
+
+display(dependency_df)
+
+# COMMAND ----------
+
+dbutils.jobs.taskValues.set(key="schedule_id", value=schedule_id)
+dbutils.jobs.taskValues.set(key="schedule", value=schedule)
+
+# COMMAND ----------
+
+dbutils.notebook.exit("exit (0)")

fabricks/api/notebooks/optimize.py

@@ -0,0 +1,25 @@
+# Databricks notebook source
+from databricks.sdk.runtime import dbutils
+from pyspark.errors.exceptions.base import IllegalArgumentException
+
+from fabricks.core.scripts import optimize
+
+# COMMAND ----------
+
+dbutils.widgets.text("schedule_id", "---")
+
+# COMMAND ----------
+
+try:
+    schedule_id = dbutils.jobs.taskValues.get(taskKey="initialize", key="schedule_id")
+except (TypeError, IllegalArgumentException, ValueError):
+    schedule_id = dbutils.widgets.get("schedule_id")
+    schedule_id = None if schedule_id == "---" else schedule_id
+
+# COMMAND ----------
+
+optimize(schedule_id=schedule_id)
+
+# COMMAND ----------
+
+dbutils.notebook.exit("exit (0)")

fabricks/api/notebooks/process.py

@@ -0,0 +1,50 @@
+# Databricks notebook source
+from databricks.sdk.runtime import dbutils
+from pyspark.errors.exceptions.base import IllegalArgumentException
+
+from fabricks.core.scripts import process
+
+# COMMAND ----------
+
+dbutils.widgets.text("step", "---")
+dbutils.widgets.text("schedule_id", "---")
+
+# COMMAND ----------
+
+try:
+    schedule_id = dbutils.jobs.taskValues.get(taskKey="initialize", key="schedule_id")
+except (TypeError, IllegalArgumentException, ValueError):
+    schedule_id = dbutils.widgets.get("schedule_id")
+    assert schedule_id != "---", "no schedule_id provided"
+
+assert schedule_id is not None
+
+# COMMAND ----------
+
+step = dbutils.widgets.get("step")
+assert step != "---", "no step provided"
+
+# COMMAND ----------
+
+schedule = dbutils.widgets.get("schedule")
+assert schedule != "---", "no schedule provided"
+
+# COMMAND ----------
+
+print(schedule_id)
+
+# COMMAND ----------
+
+print(step)
+
+# COMMAND ----------
+
+print(schedule)
+
+# COMMAND ----------
+
+process(schedule_id=schedule_id, schedule=schedule, step=step)
+
+# COMMAND ----------
+
+dbutils.notebook.exit("exit (0)")

fabricks/api/notebooks/run.py

@@ -0,0 +1,87 @@
+# Databricks notebook source
+import json
+
+from databricks.sdk.runtime import dbutils
+
+from fabricks.core.dags.log import DagsLogger, DagsTableLogger
+from fabricks.core.jobs import get_job
+
+# COMMAND ----------
+
+dbutils.widgets.text("step", "---")
+dbutils.widgets.text("job_id", "---")
+dbutils.widgets.text("job", "--")
+dbutils.widgets.text("schedule_id", "---")
+dbutils.widgets.text("schedule", "---")
+
+# COMMAND ----------
+
+step = dbutils.widgets.get("step")
+assert step != "---"
+
+# COMMAND ----------
+
+job_id = dbutils.widgets.get("job_id")
+assert job_id != "---"
+
+# COMMAND ----------
+
+job = dbutils.widgets.get("job")
+assert job != "---"
+
+# COMMAND ----------
+
+schedule_id = dbutils.widgets.get("schedule_id")
+assert schedule_id != "---"
+
+# COMMAND ----------
+
+schedule = dbutils.widgets.get("schedule")
+assert schedule != "---"
+
+# COMMAND ----------
+
+context = json.loads(dbutils.notebook.entry_point.getDbutils().notebook().getContext().toJson())  # type: ignore
+notebook_id = context.get("tags").get("jobId")
+
+# COMMAND ----------
+
+job = get_job(step=step, job_id=job_id)
+
+# COMMAND ----------
+
+print(job.qualified_name)
+
+# COMMAND ----------
+
+extra = {
+    "partition_key": schedule_id,
+    "schedule_id": schedule_id,
+    "schedule": schedule,
+    "step": step,
+    "job": job,
+    "notebook_id": notebook_id,
+    "target": "buffer",
+}
+
+# COMMAND ----------
+
+DagsLogger.info("running", extra=extra)
+
+
+# COMMAND ----------
+
+try:
+    job.run(schedule_id=schedule_id, schedule=schedule)
+    DagsLogger.info("done", extra=extra)
+
+except Exception as e:
+    DagsLogger.exception("failed", extra=extra)
+    raise e
+
+finally:
+    DagsTableLogger.flush()
+
+# COMMAND ----------
+
+dbutils.notebook.exit("exit (0)")

fabricks/api/notebooks/terminate.py

@@ -0,0 +1,27 @@
+# Databricks notebook source
+from databricks.sdk.runtime import dbutils
+from pyspark.errors.exceptions.base import IllegalArgumentException
+
+from fabricks.core.scripts import terminate
+
+# COMMAND ----------
+
+try:
+    schedule_id = dbutils.jobs.taskValues.get(taskKey="initialize", key="schedule_id")
+except (TypeError, IllegalArgumentException, ValueError):
+    schedule_id = dbutils.widgets.get("schedule_id")
+    assert schedule_id != "---", "no schedule_id provided"
+
+assert schedule_id is not None
+
+# COMMAND ----------
+
+print(schedule_id)
+
+# COMMAND ----------
+
+terminate(schedule_id=schedule_id)
+
+# COMMAND ----------
+
+dbutils.notebook.exit("exit (0)")

fabricks/api/notebooks/vacuum.py

@@ -0,0 +1,25 @@
+# Databricks notebook source
+from databricks.sdk.runtime import dbutils
+from pyspark.errors.exceptions.base import IllegalArgumentException
+
+from fabricks.core.scripts import vacuum
+
+# COMMAND ----------
+
+dbutils.widgets.text("schedule_id", "---")
+
+# COMMAND ----------
+
+try:
+    schedule_id = dbutils.jobs.taskValues.get(taskKey="initialize", key="schedule_id")
+except (TypeError, IllegalArgumentException, ValueError):
+    schedule_id = dbutils.widgets.get("schedule_id")
+    schedule_id = None if schedule_id == "---" else schedule_id
+
+# COMMAND ----------
+
+vacuum(schedule_id=schedule_id)
+
+# COMMAND ----------
+
+dbutils.notebook.exit("exit (0)")

fabricks/api/parsers.py

@@ -0,0 +1,3 @@
+from fabricks.core.parsers import BaseParser, ParserOptions, parser
+
+__all__ = ["BaseParser", "ParserOptions", "parser"]

fabricks/api/udfs.py

@@ -0,0 +1,3 @@
+from fabricks.core.udfs import register_all_udfs, register_udf, udf
+
+__all__ = ["udf", "register_all_udfs", "register_udf"]

fabricks/api/utils.py

@@ -0,0 +1,9 @@
+from fabricks.utils.helpers import concat_dfs, concat_ws, run_in_parallel
+from fabricks.utils.path import Path
+
+__all__ = [
+    "concat_dfs",
+    "concat_ws",
+    "Path",
+    "run_in_parallel",
+]

fabricks/cdc/__init__.py

@@ -0,0 +1,14 @@
+from fabricks.cdc.base import BaseCDC, ChangeDataCaptures
+from fabricks.cdc.cdc import CDC
+from fabricks.cdc.nocdc import NoCDC
+from fabricks.cdc.scd1 import SCD1
+from fabricks.cdc.scd2 import SCD2
+
+__all__ = [
+    "BaseCDC",
+    "CDC",
+    "ChangeDataCaptures",
+    "NoCDC",
+    "SCD1",
+    "SCD2",
+]

fabricks/cdc/base/__init__.py

@@ -0,0 +1,4 @@
+from fabricks.cdc.base.cdc import BaseCDC
+from fabricks.cdc.base.types import ChangeDataCaptures
+
+__all__ = ["BaseCDC", "ChangeDataCaptures"]

fabricks/cdc/base/cdc.py

@@ -0,0 +1,5 @@
+from fabricks.cdc.base.merger import Merger
+
+
+class BaseCDC(Merger):
+    pass