databricksedu 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- databricksedu/__init__.py +24 -0
- databricksedu/_content/catalog.json +1923 -0
- databricksedu/_content/nb/agents-tools.json.gz +0 -0
- databricksedu/_content/nb/ai-bi-dashboards.json.gz +0 -0
- databricksedu/_content/nb/ai-functions-gateway.json.gz +0 -0
- databricksedu/_content/nb/asset-bundles-cicd.json.gz +0 -0
- databricksedu/_content/nb/asset-bundles-recap.json.gz +0 -0
- databricksedu/_content/nb/auto-loader-copy-into.json.gz +0 -0
- databricksedu/_content/nb/cdc-data.json.gz +0 -0
- databricksedu/_content/nb/cdc-scd.json.gz +0 -0
- databricksedu/_content/nb/change-data-feed.json.gz +0 -0
- databricksedu/_content/nb/chunking-parsing.json.gz +0 -0
- databricksedu/_content/nb/churn-data.json.gz +0 -0
- databricksedu/_content/nb/data-privacy-recap.json.gz +0 -0
- databricksedu/_content/nb/data-quality-expectations.json.gz +0 -0
- databricksedu/_content/nb/declarative-pipelines-recap.json.gz +0 -0
- databricksedu/_content/nb/declarative-pipelines.json.gz +0 -0
- databricksedu/_content/nb/deletion-vectors.json.gz +0 -0
- databricksedu/_content/nb/delta-acid-concurrency.json.gz +0 -0
- databricksedu/_content/nb/delta-dml-merge.json.gz +0 -0
- databricksedu/_content/nb/delta-optimization.json.gz +0 -0
- databricksedu/_content/nb/delta-sharing.json.gz +0 -0
- databricksedu/_content/nb/delta-table-anatomy.json.gz +0 -0
- databricksedu/_content/nb/delta-time-travel-vacuum.json.gz +0 -0
- databricksedu/_content/nb/delta-transaction-log.json.gz +0 -0
- databricksedu/_content/nb/devops-essentials-recap.json.gz +0 -0
- databricksedu/_content/nb/embeddings-vector-search.json.gz +0 -0
- databricksedu/_content/nb/exam-data-analyst-associate.json.gz +0 -0
- databricksedu/_content/nb/exam-data-engineer-associate.json.gz +0 -0
- databricksedu/_content/nb/exam-data-engineer-professional.json.gz +0 -0
- databricksedu/_content/nb/exam-genai-engineer-associate.json.gz +0 -0
- databricksedu/_content/nb/exam-ml-associate.json.gz +0 -0
- databricksedu/_content/nb/exam-ml-professional.json.gz +0 -0
- databricksedu/_content/nb/exam-spark-developer-associate.json.gz +0 -0
- databricksedu/_content/nb/feature-engineering.json.gz +0 -0
- databricksedu/_content/nb/genai-evaluation.json.gz +0 -0
- databricksedu/_content/nb/genie-spaces.json.gz +0 -0
- databricksedu/_content/nb/hr-data.json.gz +0 -0
- databricksedu/_content/nb/hyperparameter-tuning.json.gz +0 -0
- databricksedu/_content/nb/iot-data.json.gz +0 -0
- databricksedu/_content/nb/lakeflow-connect-recap.json.gz +0 -0
- databricksedu/_content/nb/lakeflow-jobs-recap.json.gz +0 -0
- databricksedu/_content/nb/lakeflow-jobs.json.gz +0 -0
- databricksedu/_content/nb/lakehouse-architecture.json.gz +0 -0
- databricksedu/_content/nb/lineage-system-tables.json.gz +0 -0
- databricksedu/_content/nb/llm-basics.json.gz +0 -0
- databricksedu/_content/nb/medallion-architecture.json.gz +0 -0
- databricksedu/_content/nb/metric-views.json.gz +0 -0
- databricksedu/_content/nb/ml-lifecycle.json.gz +0 -0
- databricksedu/_content/nb/mlflow-tracking.json.gz +0 -0
- databricksedu/_content/nb/model-monitoring.json.gz +0 -0
- databricksedu/_content/nb/model-serving.json.gz +0 -0
- databricksedu/_content/nb/models-in-unity-catalog.json.gz +0 -0
- databricksedu/_content/nb/parquet-files.json.gz +0 -0
- databricksedu/_content/nb/performance-optimization-recap.json.gz +0 -0
- databricksedu/_content/nb/prompt-engineering.json.gz +0 -0
- databricksedu/_content/nb/rag-architecture.json.gz +0 -0
- databricksedu/_content/nb/retail-data.json.gz +0 -0
- databricksedu/_content/nb/row-filters-column-masks.json.gz +0 -0
- databricksedu/_content/nb/schema-enforcement-evolution.json.gz +0 -0
- databricksedu/_content/nb/spark-architecture.json.gz +0 -0
- databricksedu/_content/nb/spark-joins-shuffles.json.gz +0 -0
- databricksedu/_content/nb/sql-warehouses.json.gz +0 -0
- databricksedu/_content/nb/streaming-pipelines-recap.json.gz +0 -0
- databricksedu/_content/nb/structured-streaming-basics.json.gz +0 -0
- databricksedu/_content/nb/support-data.json.gz +0 -0
- databricksedu/_content/nb/uc-privileges.json.gz +0 -0
- databricksedu/_content/nb/unity-catalog-basics.json.gz +0 -0
- databricksedu/_content/nb/watermarks-stateful-streaming.json.gz +0 -0
- databricksedu/_content/web/agents-tools.b64 +1 -0
- databricksedu/_content/web/ai-bi-dashboards.b64 +1 -0
- databricksedu/_content/web/ai-functions-gateway.b64 +1 -0
- databricksedu/_content/web/asset-bundles-cicd.b64 +1 -0
- databricksedu/_content/web/asset-bundles-recap.b64 +1 -0
- databricksedu/_content/web/auto-loader-copy-into.b64 +1 -0
- databricksedu/_content/web/cdc-data.b64 +1 -0
- databricksedu/_content/web/cdc-scd.b64 +1 -0
- databricksedu/_content/web/change-data-feed.b64 +1 -0
- databricksedu/_content/web/chunking-parsing.b64 +1 -0
- databricksedu/_content/web/churn-data.b64 +1 -0
- databricksedu/_content/web/data-privacy-recap.b64 +1 -0
- databricksedu/_content/web/data-quality-expectations.b64 +1 -0
- databricksedu/_content/web/declarative-pipelines-recap.b64 +1 -0
- databricksedu/_content/web/declarative-pipelines.b64 +1 -0
- databricksedu/_content/web/deletion-vectors.b64 +1 -0
- databricksedu/_content/web/delta-acid-concurrency.b64 +1 -0
- databricksedu/_content/web/delta-dml-merge.b64 +1 -0
- databricksedu/_content/web/delta-optimization.b64 +1 -0
- databricksedu/_content/web/delta-sharing.b64 +1 -0
- databricksedu/_content/web/delta-table-anatomy.b64 +1 -0
- databricksedu/_content/web/delta-time-travel-vacuum.b64 +1 -0
- databricksedu/_content/web/delta-transaction-log.b64 +1 -0
- databricksedu/_content/web/devops-essentials-recap.b64 +1 -0
- databricksedu/_content/web/embeddings-vector-search.b64 +1 -0
- databricksedu/_content/web/exam-data-analyst-associate.b64 +1 -0
- databricksedu/_content/web/exam-data-engineer-associate.b64 +1 -0
- databricksedu/_content/web/exam-data-engineer-professional.b64 +1 -0
- databricksedu/_content/web/exam-genai-engineer-associate.b64 +1 -0
- databricksedu/_content/web/exam-ml-associate.b64 +1 -0
- databricksedu/_content/web/exam-ml-professional.b64 +1 -0
- databricksedu/_content/web/exam-spark-developer-associate.b64 +1 -0
- databricksedu/_content/web/feature-engineering.b64 +1 -0
- databricksedu/_content/web/genai-evaluation.b64 +1 -0
- databricksedu/_content/web/genie-spaces.b64 +1 -0
- databricksedu/_content/web/hr-data.b64 +1 -0
- databricksedu/_content/web/hyperparameter-tuning.b64 +1 -0
- databricksedu/_content/web/iot-data.b64 +1 -0
- databricksedu/_content/web/lakeflow-connect-recap.b64 +1 -0
- databricksedu/_content/web/lakeflow-jobs-recap.b64 +1 -0
- databricksedu/_content/web/lakeflow-jobs.b64 +1 -0
- databricksedu/_content/web/lakehouse-architecture.b64 +1 -0
- databricksedu/_content/web/lineage-system-tables.b64 +1 -0
- databricksedu/_content/web/llm-basics.b64 +1 -0
- databricksedu/_content/web/medallion-architecture.b64 +1 -0
- databricksedu/_content/web/metric-views.b64 +1 -0
- databricksedu/_content/web/ml-lifecycle.b64 +1 -0
- databricksedu/_content/web/mlflow-tracking.b64 +1 -0
- databricksedu/_content/web/model-monitoring.b64 +1 -0
- databricksedu/_content/web/model-serving.b64 +1 -0
- databricksedu/_content/web/models-in-unity-catalog.b64 +1 -0
- databricksedu/_content/web/parquet-files.b64 +1 -0
- databricksedu/_content/web/performance-optimization-recap.b64 +1 -0
- databricksedu/_content/web/prompt-engineering.b64 +1 -0
- databricksedu/_content/web/rag-architecture.b64 +1 -0
- databricksedu/_content/web/retail-data.b64 +1 -0
- databricksedu/_content/web/row-filters-column-masks.b64 +1 -0
- databricksedu/_content/web/schema-enforcement-evolution.b64 +1 -0
- databricksedu/_content/web/spark-architecture.b64 +1 -0
- databricksedu/_content/web/spark-joins-shuffles.b64 +1 -0
- databricksedu/_content/web/sql-warehouses.b64 +1 -0
- databricksedu/_content/web/streaming-pipelines-recap.b64 +1 -0
- databricksedu/_content/web/structured-streaming-basics.b64 +1 -0
- databricksedu/_content/web/support-data.b64 +1 -0
- databricksedu/_content/web/uc-privileges.b64 +1 -0
- databricksedu/_content/web/unity-catalog-basics.b64 +1 -0
- databricksedu/_content/web/watermarks-stateful-streaming.b64 +1 -0
- databricksedu/_version.py +1 -0
- databricksedu/app.py +192 -0
- databricksedu/assets/app.css +1 -0
- databricksedu/assets/app.js +353 -0
- databricksedu/assets/dbx.css +1 -0
- databricksedu/assets/quiz.js +47 -0
- databricksedu/assets/viz.css +1 -0
- databricksedu/assets/viz.js +84 -0
- databricksedu/catalog.py +95 -0
- databricksedu/data/__init__.py +58 -0
- databricksedu/data/_core.py +339 -0
- databricksedu/data/cdc.py +75 -0
- databricksedu/data/churn.py +74 -0
- databricksedu/data/hr.py +58 -0
- databricksedu/data/iot.py +86 -0
- databricksedu/data/retail.py +77 -0
- databricksedu/data/support.py +128 -0
- databricksedu/workspace.py +157 -0
- databricksedu-0.1.0.dist-info/METADATA +131 -0
- databricksedu-0.1.0.dist-info/RECORD +158 -0
- databricksedu-0.1.0.dist-info/WHEEL +4 -0
- databricksedu-0.1.0.dist-info/licenses/LICENSE +21 -0
databricksedu/catalog.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""The module index and the packaged content (built by tools/build.py)."""
|
|
2
|
+
import gzip
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
|
|
6
|
+
_HERE = os.path.dirname(os.path.abspath(__file__))
|
|
7
|
+
_CONTENT = os.path.join(_HERE, "_content")
|
|
8
|
+
_ASSETS = os.path.join(_HERE, "assets")
|
|
9
|
+
_cache = {}
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _read(path, mode="r"):
|
|
13
|
+
if mode == "rb":
|
|
14
|
+
with open(path, "rb") as f:
|
|
15
|
+
return f.read()
|
|
16
|
+
with open(path, "r", encoding="utf-8") as f:
|
|
17
|
+
return f.read()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def asset(name):
|
|
21
|
+
key = ("asset", name)
|
|
22
|
+
if key not in _cache:
|
|
23
|
+
_cache[key] = _read(os.path.join(_ASSETS, name))
|
|
24
|
+
return _cache[key]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def categories():
|
|
28
|
+
"""The categories with their modules, in display order."""
|
|
29
|
+
if "catalog" not in _cache:
|
|
30
|
+
path = os.path.join(_CONTENT, "catalog.json")
|
|
31
|
+
if not os.path.exists(path):
|
|
32
|
+
raise RuntimeError("The packaged content is missing. If you run from a git checkout, "
|
|
33
|
+
"build it first: python tools/build.py")
|
|
34
|
+
_cache["catalog"] = json.loads(_read(path))["categories"]
|
|
35
|
+
return _cache["catalog"]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def index():
|
|
39
|
+
"""{module id: (category, module)}"""
|
|
40
|
+
if "index" not in _cache:
|
|
41
|
+
_cache["index"] = {m["id"]: (c, m) for c in categories() for m in c["modules"]}
|
|
42
|
+
return _cache["index"]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def resolve(what):
|
|
46
|
+
"""'all', a category id or a module id (or a module title) -> list of (category, module)."""
|
|
47
|
+
idx = index()
|
|
48
|
+
if what in (None, "", "all", "*"):
|
|
49
|
+
return list(idx.values())
|
|
50
|
+
if what in idx:
|
|
51
|
+
return [idx[what]]
|
|
52
|
+
for c in categories():
|
|
53
|
+
if what in (c["id"], c["title"]):
|
|
54
|
+
return [(c, m) for m in c["modules"]]
|
|
55
|
+
low = str(what).lower()
|
|
56
|
+
hits = [cm for cm in idx.values() if cm[1]["title"].lower() == low]
|
|
57
|
+
if not hits:
|
|
58
|
+
hits = [cm for cm in idx.values() if low in cm[1]["id"] or low in cm[1]["title"].lower()]
|
|
59
|
+
if len(hits) == 1:
|
|
60
|
+
return hits
|
|
61
|
+
if hits:
|
|
62
|
+
raise ValueError("%r matches several modules: %s" % (what, ", ".join(m["id"] for _, m in hits)))
|
|
63
|
+
raise ValueError("Unknown module or track %r. Run databricksedu.modules() to see what is available." % (what,))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def web_blob(module_id):
|
|
67
|
+
"""Base64 text of the gzipped slide bundle, ready to embed in a page."""
|
|
68
|
+
return _read(os.path.join(_CONTENT, "web", module_id + ".b64"))
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def notebook_bundle(module_id):
|
|
72
|
+
"""{'notebooks': [{'name', 'cells'}], 'md': raw Markdown}"""
|
|
73
|
+
return json.loads(gzip.decompress(_read(os.path.join(_CONTENT, "nb", module_id + ".json.gz"), "rb")).decode("utf-8"))
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def modules(track=None, show=True):
|
|
77
|
+
"""List the modules (optionally of one track). Returns a list of dicts when show=False."""
|
|
78
|
+
rows = []
|
|
79
|
+
for c in categories():
|
|
80
|
+
if track and track not in (c["id"], c["title"]):
|
|
81
|
+
continue
|
|
82
|
+
for m in c["modules"]:
|
|
83
|
+
rows.append({"track": c["id"], "id": m["id"], "title": m["title"], "level": m["level"],
|
|
84
|
+
"minutes": m["minutes"], "summary": m["summary"]})
|
|
85
|
+
if not show:
|
|
86
|
+
return rows
|
|
87
|
+
last = None
|
|
88
|
+
for r in rows:
|
|
89
|
+
if r["track"] != last:
|
|
90
|
+
last = r["track"]
|
|
91
|
+
title = [c["title"] for c in categories() if c["id"] == last][0]
|
|
92
|
+
print("\n%s (save(\"%s\"))" % (title, last))
|
|
93
|
+
print(" %-34s %s" % (r["id"], r["title"]))
|
|
94
|
+
print("\nstart() opens the app, present(id) plays one module, save(id) copies its notebooks.")
|
|
95
|
+
return None
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""Deterministic synthetic datasets for Databricks demos and exercises.
|
|
2
|
+
|
|
3
|
+
from databricksedu import data
|
|
4
|
+
|
|
5
|
+
shop = data.generate("retail", seed=42) # same seed -> exactly the same rows, every time
|
|
6
|
+
shop.preview()
|
|
7
|
+
shop.register_views(spark) # temp views: src_dim_customer, src_dim_product, ...
|
|
8
|
+
|
|
9
|
+
data.use(CATALOG, SCHEMA) # USE CATALOG / USE SCHEMA (never creates a schema)
|
|
10
|
+
print(shop.sql()) # the CREATE TABLE / PRIMARY KEY / FOREIGN KEY script
|
|
11
|
+
shop.create_tables(CATALOG, SCHEMA) # ...or run it in one call
|
|
12
|
+
|
|
13
|
+
Datasets: retail, iot, churn, support, hr, cdc (see data.datasets()).
|
|
14
|
+
"""
|
|
15
|
+
import importlib
|
|
16
|
+
|
|
17
|
+
from ._core import Dataset, Table, get_spark, use, volume
|
|
18
|
+
|
|
19
|
+
NAMES = ["retail", "iot", "churn", "support", "hr", "cdc"]
|
|
20
|
+
|
|
21
|
+
__all__ = ["generate", "datasets", "describe", "use", "volume", "Dataset", "Table", "NAMES"]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _module(name):
|
|
25
|
+
if name not in NAMES:
|
|
26
|
+
raise ValueError("Unknown dataset %r. Available: %s" % (name, ", ".join(NAMES)))
|
|
27
|
+
return importlib.import_module("." + name, __name__)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def generate(name="retail", seed=42, scale=1.0):
|
|
31
|
+
"""Generate a dataset in memory. The same name, seed and scale always give the same rows.
|
|
32
|
+
|
|
33
|
+
``scale`` multiplies the row count of the large tables (0.1 = a tenth, 2 = double).
|
|
34
|
+
"""
|
|
35
|
+
return _module(name).generate(seed=seed, scale=scale)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def describe(name):
|
|
39
|
+
"""Static description of a dataset: title, summary and table layouts (no rows generated twice)."""
|
|
40
|
+
mod = _module(name)
|
|
41
|
+
ds = mod.generate(seed=42, scale=0.02)
|
|
42
|
+
return {
|
|
43
|
+
"name": name, "title": mod.TITLE, "icon": mod.ICON, "summary": mod.SUMMARY, "use_for": mod.USE_FOR,
|
|
44
|
+
"tables": [{"name": t.name, "comment": t.comment, "columns": [list(c) for c in t.columns],
|
|
45
|
+
"primary_key": t.primary_key, "foreign_keys": [list(fk) for fk in t.foreign_keys]}
|
|
46
|
+
for t in ds],
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def datasets(show=True):
|
|
51
|
+
"""List the available datasets."""
|
|
52
|
+
out = []
|
|
53
|
+
for name in NAMES:
|
|
54
|
+
mod = _module(name)
|
|
55
|
+
out.append({"name": name, "title": mod.TITLE, "summary": mod.SUMMARY, "use_for": mod.USE_FOR})
|
|
56
|
+
if show:
|
|
57
|
+
print('%s data.generate("%s")\n %s\n Good for: %s\n' % (mod.TITLE, name, mod.SUMMARY, mod.USE_FOR))
|
|
58
|
+
return None if show else out
|
|
@@ -0,0 +1,339 @@
|
|
|
1
|
+
"""Core classes for the synthetic datasets: Table, Dataset, and the Spark helpers.
|
|
2
|
+
|
|
3
|
+
Everything here is pure Python (standard library only). Spark is only touched by the
|
|
4
|
+
methods that take or look up a ``spark`` session.
|
|
5
|
+
"""
|
|
6
|
+
import csv
|
|
7
|
+
import datetime as _dt
|
|
8
|
+
import io
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
|
|
13
|
+
SRC_PREFIX = "src_"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
# --------------------------------------------------------------------------- helpers
|
|
17
|
+
def pick(rng, weighted):
|
|
18
|
+
"""Weighted choice built on rng.random() only, so results are stable across Python versions."""
|
|
19
|
+
r, acc = rng.random(), 0.0
|
|
20
|
+
for value, w in weighted:
|
|
21
|
+
acc += w
|
|
22
|
+
if r < acc:
|
|
23
|
+
return value
|
|
24
|
+
return weighted[-1][0]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def get_spark(spark=None):
|
|
28
|
+
"""Return the given session, or find the notebook's active Spark session."""
|
|
29
|
+
if spark is not None:
|
|
30
|
+
return spark
|
|
31
|
+
try:
|
|
32
|
+
from IPython import get_ipython
|
|
33
|
+
ip = get_ipython()
|
|
34
|
+
if ip is not None and ip.user_ns.get("spark") is not None:
|
|
35
|
+
return ip.user_ns["spark"]
|
|
36
|
+
except Exception:
|
|
37
|
+
pass
|
|
38
|
+
try:
|
|
39
|
+
from pyspark.sql import SparkSession
|
|
40
|
+
active = SparkSession.getActiveSession()
|
|
41
|
+
if active is not None:
|
|
42
|
+
return active
|
|
43
|
+
return SparkSession.builder.getOrCreate()
|
|
44
|
+
except Exception as exc: # pragma: no cover - only without Spark
|
|
45
|
+
raise RuntimeError("No Spark session found. Run this inside a Databricks notebook, "
|
|
46
|
+
"or pass spark=<your session>.") from exc
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _clean(name):
|
|
50
|
+
return str(name).strip().strip("`").strip()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _q(name):
|
|
54
|
+
return "`" + _clean(name).replace("`", "``") + "`"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _suggest(spark):
|
|
58
|
+
try:
|
|
59
|
+
user = spark.sql("SELECT current_user()").first()[0]
|
|
60
|
+
guess = re.sub(r"[^a-z0-9_]", "_", user.split("@")[0].lower())
|
|
61
|
+
if spark.sql("SHOW SCHEMAS IN dbacademy LIKE '%s'" % guess).count():
|
|
62
|
+
return ' In a Databricks Academy lab this is usually CATALOG = "dbacademy", SCHEMA = "%s".' % guess
|
|
63
|
+
except Exception:
|
|
64
|
+
pass
|
|
65
|
+
return " Run SHOW CATALOGS / SHOW SCHEMAS IN <catalog> to find a schema you can write to."
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def use(catalog, schema, spark=None):
|
|
69
|
+
"""Validate the target and run USE CATALOG / USE SCHEMA. Never creates a catalog or schema.
|
|
70
|
+
|
|
71
|
+
Returns the quoted ``catalog.schema`` name.
|
|
72
|
+
"""
|
|
73
|
+
spark = get_spark(spark)
|
|
74
|
+
catalog, schema = _clean(catalog), _clean(schema)
|
|
75
|
+
if not catalog or not schema or "FILL_IN" in (catalog + schema).upper():
|
|
76
|
+
raise ValueError("Fill in CATALOG and SCHEMA first (got %r, %r)." % (catalog, schema) + _suggest(spark))
|
|
77
|
+
spark.sql("USE CATALOG " + _q(catalog))
|
|
78
|
+
spark.sql("USE SCHEMA " + _q(schema))
|
|
79
|
+
print("Using %s.%s" % (catalog, schema))
|
|
80
|
+
return "%s.%s" % (_q(catalog), _q(schema))
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def volume(catalog, schema, name="databricksedu_files", spark=None):
|
|
84
|
+
"""Create (if needed) a volume for demo files and checkpoints. Returns its /Volumes path."""
|
|
85
|
+
spark = get_spark(spark)
|
|
86
|
+
catalog, schema = _clean(catalog), _clean(schema)
|
|
87
|
+
if not catalog or not schema or "FILL_IN" in (catalog + schema).upper():
|
|
88
|
+
raise ValueError("Fill in CATALOG and SCHEMA first (got %r, %r)." % (catalog, schema) + _suggest(spark))
|
|
89
|
+
spark.sql("CREATE VOLUME IF NOT EXISTS %s.%s.%s COMMENT 'databricksedu demo files and checkpoints'"
|
|
90
|
+
% (_q(catalog), _q(schema), _q(name)))
|
|
91
|
+
return "/Volumes/%s/%s/%s" % (catalog, schema, name)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _base_type(sql_type):
|
|
95
|
+
return re.sub(r"\s+NOT\s+NULL", "", sql_type, flags=re.I).strip()
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _sql_str(text):
|
|
99
|
+
return str(text).replace("\\", "\\\\").replace("'", "\\'")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
# --------------------------------------------------------------------------- Table
|
|
103
|
+
class Table(object):
|
|
104
|
+
"""One generated table: column definitions, rows (tuples of plain Python values) and keys."""
|
|
105
|
+
|
|
106
|
+
def __init__(self, name, columns, rows, comment="", primary_key=None, foreign_keys=None):
|
|
107
|
+
self.name = name
|
|
108
|
+
self.columns = [tuple(c) for c in columns] # (name, SQL type, comment)
|
|
109
|
+
self.rows = rows
|
|
110
|
+
self.comment = comment
|
|
111
|
+
self.primary_key = list(primary_key or [])
|
|
112
|
+
self.foreign_keys = [tuple(fk) for fk in (foreign_keys or [])] # (column, ref_table, ref_column)
|
|
113
|
+
|
|
114
|
+
@property
|
|
115
|
+
def column_names(self):
|
|
116
|
+
return [c[0] for c in self.columns]
|
|
117
|
+
|
|
118
|
+
def __len__(self):
|
|
119
|
+
return len(self.rows)
|
|
120
|
+
|
|
121
|
+
def __repr__(self):
|
|
122
|
+
return "Table(%s, %d rows, %d columns)" % (self.name, len(self.rows), len(self.columns))
|
|
123
|
+
|
|
124
|
+
def records(self):
|
|
125
|
+
names = self.column_names
|
|
126
|
+
return [dict(zip(names, r)) for r in self.rows]
|
|
127
|
+
|
|
128
|
+
def to_pandas(self):
|
|
129
|
+
import pandas as pd
|
|
130
|
+
return pd.DataFrame(self.rows, columns=self.column_names)
|
|
131
|
+
|
|
132
|
+
def spark_schema(self):
|
|
133
|
+
return ", ".join("%s %s" % (_q(c), _base_type(t)) for c, t, _ in self.columns)
|
|
134
|
+
|
|
135
|
+
def dataframe(self, spark=None):
|
|
136
|
+
return get_spark(spark).createDataFrame(self.rows, self.spark_schema())
|
|
137
|
+
|
|
138
|
+
def create_sql(self, target=""):
|
|
139
|
+
lines = []
|
|
140
|
+
for c, t, cmt in self.columns:
|
|
141
|
+
lines.append(" %-18s %s%s" % (c, t, " COMMENT '%s'" % _sql_str(cmt) if cmt else ""))
|
|
142
|
+
if self.primary_key:
|
|
143
|
+
lines.append(" CONSTRAINT %s_pk PRIMARY KEY (%s)" % (self.name, ", ".join(self.primary_key)))
|
|
144
|
+
out = "CREATE TABLE %s%s (\n%s\n)" % (target, self.name, ",\n".join(lines))
|
|
145
|
+
if self.comment:
|
|
146
|
+
out += " COMMENT '%s'" % _sql_str(self.comment)
|
|
147
|
+
return out
|
|
148
|
+
|
|
149
|
+
def foreign_key_sql(self, target=""):
|
|
150
|
+
return ["ALTER TABLE %s%s ADD CONSTRAINT %s_%s_fk FOREIGN KEY (%s) REFERENCES %s%s (%s)"
|
|
151
|
+
% (target, self.name, self.name, col, col, target, ref, ref_col)
|
|
152
|
+
for col, ref, ref_col in self.foreign_keys]
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
# --------------------------------------------------------------------------- Dataset
|
|
156
|
+
class Dataset(object):
|
|
157
|
+
"""A set of related tables generated from one seed. Parents come before children."""
|
|
158
|
+
|
|
159
|
+
def __init__(self, name, description, seed, tables, extra_files=None):
|
|
160
|
+
self.name = name
|
|
161
|
+
self.description = description
|
|
162
|
+
self.seed = seed
|
|
163
|
+
self.tables = {t.name: t for t in tables}
|
|
164
|
+
self.extra_files = extra_files or {} # {relative path: text}
|
|
165
|
+
|
|
166
|
+
def __getitem__(self, table):
|
|
167
|
+
return self.tables[table]
|
|
168
|
+
|
|
169
|
+
def __iter__(self):
|
|
170
|
+
return iter(self.tables.values())
|
|
171
|
+
|
|
172
|
+
def __repr__(self):
|
|
173
|
+
return "Dataset(%s, seed=%s, tables=[%s])" % (
|
|
174
|
+
self.name, self.seed, ", ".join("%s:%d" % (t.name, len(t)) for t in self))
|
|
175
|
+
|
|
176
|
+
# ---- looking at the data (no Spark needed)
|
|
177
|
+
def preview(self, rows=3):
|
|
178
|
+
"""Print every table with its keys and the first rows."""
|
|
179
|
+
print("Dataset '%s' (seed=%s)\n%s\n" % (self.name, self.seed, self.description))
|
|
180
|
+
for t in self:
|
|
181
|
+
keys = []
|
|
182
|
+
if t.primary_key:
|
|
183
|
+
keys.append("PK(%s)" % ", ".join(t.primary_key))
|
|
184
|
+
keys += ["FK %s -> %s.%s" % fk for fk in t.foreign_keys]
|
|
185
|
+
print("%s %s rows %s" % (t.name, format(len(t), ","), " ".join(keys)))
|
|
186
|
+
print(" " + " | ".join(t.column_names))
|
|
187
|
+
for r in t.rows[:rows]:
|
|
188
|
+
print(" " + " | ".join("" if v is None else str(v) for v in r))
|
|
189
|
+
print()
|
|
190
|
+
return self
|
|
191
|
+
|
|
192
|
+
def to_pandas(self, table):
|
|
193
|
+
return self.tables[table].to_pandas()
|
|
194
|
+
|
|
195
|
+
def fingerprint(self):
|
|
196
|
+
"""Short hash of all rows: the same seed always gives the same fingerprint."""
|
|
197
|
+
import hashlib
|
|
198
|
+
h = hashlib.sha256()
|
|
199
|
+
for t in self:
|
|
200
|
+
h.update(t.name.encode())
|
|
201
|
+
for r in t.rows:
|
|
202
|
+
h.update(repr(r).encode())
|
|
203
|
+
return h.hexdigest()[:16]
|
|
204
|
+
|
|
205
|
+
# ---- Spark: temporary views
|
|
206
|
+
def register_views(self, spark=None, prefix=SRC_PREFIX, quiet=False):
|
|
207
|
+
"""Expose every generated table as a temporary view called <prefix><table>."""
|
|
208
|
+
spark = get_spark(spark)
|
|
209
|
+
for t in self:
|
|
210
|
+
t.dataframe(spark).createOrReplaceTempView(prefix + t.name)
|
|
211
|
+
if not quiet:
|
|
212
|
+
print(" temp view %-28s %8s rows" % (prefix + t.name, format(len(t), ",")))
|
|
213
|
+
return self
|
|
214
|
+
|
|
215
|
+
# ---- SQL script
|
|
216
|
+
def _target(self, catalog=None, schema=None):
|
|
217
|
+
if catalog and schema:
|
|
218
|
+
return "%s.%s." % (_q(catalog), _q(schema))
|
|
219
|
+
return ""
|
|
220
|
+
|
|
221
|
+
def statements(self, catalog=None, schema=None, prefix=SRC_PREFIX):
|
|
222
|
+
"""The SQL that turns the temporary views into Delta tables with keys.
|
|
223
|
+
|
|
224
|
+
Returns a list of (title, [statement, ...]) steps. Without catalog and schema the names
|
|
225
|
+
are unqualified, so the statements run in the current catalog and schema.
|
|
226
|
+
"""
|
|
227
|
+
tgt = self._target(catalog, schema)
|
|
228
|
+
tables = list(self)
|
|
229
|
+
steps = [("Clean start: drop children first, so foreign keys never block a drop",
|
|
230
|
+
["DROP TABLE IF EXISTS %s%s" % (tgt, t.name) for t in reversed(tables)])]
|
|
231
|
+
for t in tables:
|
|
232
|
+
steps.append(("%s: create the table%s, then load it" % (
|
|
233
|
+
t.name, " with its primary key" if t.primary_key else ""),
|
|
234
|
+
[t.create_sql(tgt), "INSERT INTO %s%s SELECT * FROM %s%s" % (tgt, t.name, prefix, t.name)]))
|
|
235
|
+
fks = [s for t in tables for s in t.foreign_key_sql(tgt)]
|
|
236
|
+
if fks:
|
|
237
|
+
steps.append(("Foreign keys: document how the tables join", fks))
|
|
238
|
+
return steps
|
|
239
|
+
|
|
240
|
+
def sql(self, catalog=None, schema=None, prefix=SRC_PREFIX):
|
|
241
|
+
"""The whole setup as one readable SQL script."""
|
|
242
|
+
out = []
|
|
243
|
+
for title, stmts in self.statements(catalog, schema, prefix):
|
|
244
|
+
out.append("-- " + title)
|
|
245
|
+
out += [s + ";" for s in stmts]
|
|
246
|
+
out.append("")
|
|
247
|
+
return "\n".join(out).rstrip() + "\n"
|
|
248
|
+
|
|
249
|
+
def create_tables(self, catalog, schema, spark=None, show_sql=True):
|
|
250
|
+
"""Create every table in catalog.schema, running (and printing) the same SQL as .sql().
|
|
251
|
+
|
|
252
|
+
The schema must already exist: nothing here creates a catalog or a schema.
|
|
253
|
+
"""
|
|
254
|
+
spark = get_spark(spark)
|
|
255
|
+
use(catalog, schema, spark)
|
|
256
|
+
self.register_views(spark, quiet=True)
|
|
257
|
+
for title, stmts in self.statements():
|
|
258
|
+
if show_sql:
|
|
259
|
+
print("\n-- " + title)
|
|
260
|
+
for s in stmts:
|
|
261
|
+
if show_sql:
|
|
262
|
+
print(s + ";")
|
|
263
|
+
spark.sql(s)
|
|
264
|
+
print()
|
|
265
|
+
for t in self:
|
|
266
|
+
print(" %s.%s.%-24s %8s rows" % (_clean(catalog), _clean(schema), t.name,
|
|
267
|
+
format(spark.table(t.name).count(), ",")))
|
|
268
|
+
print("Dataset '%s' is ready. Current catalog and schema: %s.%s" % (self.name, _clean(catalog), _clean(schema)))
|
|
269
|
+
return self
|
|
270
|
+
|
|
271
|
+
def drop_tables(self, catalog=None, schema=None, spark=None):
|
|
272
|
+
"""Drop the tables of this dataset (children first). The schema itself is never dropped."""
|
|
273
|
+
spark = get_spark(spark)
|
|
274
|
+
tgt = self._target(catalog, schema)
|
|
275
|
+
for t in reversed(list(self)):
|
|
276
|
+
spark.sql("DROP TABLE IF EXISTS %s%s" % (tgt, t.name))
|
|
277
|
+
print(" dropped %s%s" % (tgt, t.name))
|
|
278
|
+
return self
|
|
279
|
+
|
|
280
|
+
# ---- raw files (for ingestion demos)
|
|
281
|
+
def land_files(self, path, table=None, parts=3, upto=None, fmt="csv"):
|
|
282
|
+
"""Write raw files under ``path``, like files arriving in a landing zone.
|
|
283
|
+
|
|
284
|
+
Every table goes to <path>/<table>/. The largest table (or ``table``) is split into
|
|
285
|
+
``parts`` files; ``upto`` writes only the first N of them, so calling again with a larger
|
|
286
|
+
``upto`` simulates new files arriving. Files that already exist are never rewritten.
|
|
287
|
+
``fmt`` is "csv" or "json" (JSON lines). Returns ``path``.
|
|
288
|
+
"""
|
|
289
|
+
if fmt not in ("csv", "json"):
|
|
290
|
+
raise ValueError('fmt must be "csv" or "json"')
|
|
291
|
+
tables = list(self)
|
|
292
|
+
big = table or max(tables, key=len).name
|
|
293
|
+
written = []
|
|
294
|
+
for t in tables:
|
|
295
|
+
folder = os.path.join(path, t.name)
|
|
296
|
+
os.makedirs(folder, exist_ok=True)
|
|
297
|
+
if t.name != big:
|
|
298
|
+
chunks = [("%s.%s" % (t.name, fmt), t.rows)]
|
|
299
|
+
else:
|
|
300
|
+
size = -(-len(t.rows) // parts)
|
|
301
|
+
chunks = [("part_%d.%s" % (i + 1, fmt), t.rows[i * size:(i + 1) * size])
|
|
302
|
+
for i in range(min(parts, upto or parts))]
|
|
303
|
+
for fname, rows in chunks:
|
|
304
|
+
full = os.path.join(folder, fname)
|
|
305
|
+
if os.path.exists(full): # a rewritten file would look "new" to incremental readers
|
|
306
|
+
continue
|
|
307
|
+
with open(full, "w", encoding="utf-8", newline="") as f:
|
|
308
|
+
f.write(_serialize(t.column_names, rows, fmt))
|
|
309
|
+
written.append(full)
|
|
310
|
+
for rel, text in self.extra_files.items():
|
|
311
|
+
full = os.path.join(path, rel)
|
|
312
|
+
os.makedirs(os.path.dirname(full), exist_ok=True)
|
|
313
|
+
if not os.path.exists(full):
|
|
314
|
+
with open(full, "w", encoding="utf-8", newline="") as f:
|
|
315
|
+
f.write(text)
|
|
316
|
+
written.append(full)
|
|
317
|
+
for full in written:
|
|
318
|
+
print(" " + full)
|
|
319
|
+
print("%d new file(s) under %s" % (len(written), path))
|
|
320
|
+
return path
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _plain(v):
|
|
324
|
+
if isinstance(v, (_dt.datetime, _dt.date)):
|
|
325
|
+
return v.isoformat(sep=" ") if isinstance(v, _dt.datetime) else v.isoformat()
|
|
326
|
+
return v
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def _serialize(names, rows, fmt):
|
|
330
|
+
buf = io.StringIO()
|
|
331
|
+
if fmt == "csv":
|
|
332
|
+
w = csv.writer(buf, lineterminator="\n")
|
|
333
|
+
w.writerow(names)
|
|
334
|
+
for r in rows:
|
|
335
|
+
w.writerow(["" if v is None else _plain(v) for v in r])
|
|
336
|
+
else:
|
|
337
|
+
for r in rows:
|
|
338
|
+
buf.write(json.dumps({n: _plain(v) for n, v in zip(names, r)}, ensure_ascii=False) + "\n")
|
|
339
|
+
return buf.getvalue()
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""cdc: a customer snapshot and a change feed (inserts, updates, deletes) on top of it."""
|
|
2
|
+
import datetime as dt
|
|
3
|
+
import random
|
|
4
|
+
|
|
5
|
+
from ._core import Dataset, Table, pick
|
|
6
|
+
|
|
7
|
+
TITLE = "Change data capture feed"
|
|
8
|
+
ICON = "🔁"
|
|
9
|
+
SUMMARY = ("A 200-row customer snapshot and 600 change events (INSERT, UPDATE, DELETE) with a sequence number. "
|
|
10
|
+
"Some events arrive out of order, so the sequence column matters.")
|
|
11
|
+
USE_FOR = "MERGE INTO, AUTO CDC (APPLY CHANGES), SCD Type 1 and Type 2, Change Data Feed, deduplication"
|
|
12
|
+
|
|
13
|
+
CITIES = ["Budapest", "Vienna", "Berlin", "Paris", "Madrid", "Milan", "London", "Dublin", "Warsaw", "Prague"]
|
|
14
|
+
TIERS = [("Bronze", .5), ("Silver", .3), ("Gold", .2)]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def generate(seed=42, scale=1.0):
|
|
18
|
+
rng = random.Random(seed)
|
|
19
|
+
state = {}
|
|
20
|
+
snapshot = []
|
|
21
|
+
for cid in range(1, 201):
|
|
22
|
+
row = [cid, "Customer %04d" % cid, "customer%04d@example.com" % cid,
|
|
23
|
+
CITIES[rng.randrange(len(CITIES))], pick(rng, TIERS)]
|
|
24
|
+
state[cid] = row
|
|
25
|
+
snapshot.append(tuple(row) + (dt.datetime(2026, 9, 30, 23, 59, 59),))
|
|
26
|
+
|
|
27
|
+
events = []
|
|
28
|
+
ts = dt.datetime(2026, 10, 1, 0, 0, 0)
|
|
29
|
+
next_id = 201
|
|
30
|
+
for seq in range(1, int(600 * scale) + 1):
|
|
31
|
+
ts += dt.timedelta(minutes=rng.randrange(5, 120))
|
|
32
|
+
roll = rng.random()
|
|
33
|
+
if roll < 0.20 or not state:
|
|
34
|
+
cid = next_id
|
|
35
|
+
next_id += 1
|
|
36
|
+
state[cid] = [cid, "Customer %04d" % cid, "customer%04d@example.com" % cid,
|
|
37
|
+
CITIES[rng.randrange(len(CITIES))], pick(rng, TIERS)]
|
|
38
|
+
op = "INSERT"
|
|
39
|
+
else:
|
|
40
|
+
cid = sorted(state)[rng.randrange(len(state))]
|
|
41
|
+
if roll < 0.90:
|
|
42
|
+
if rng.random() < 0.5:
|
|
43
|
+
state[cid][3] = CITIES[rng.randrange(len(CITIES))]
|
|
44
|
+
else:
|
|
45
|
+
state[cid][4] = pick(rng, TIERS)
|
|
46
|
+
op = "UPDATE"
|
|
47
|
+
else:
|
|
48
|
+
op = "DELETE"
|
|
49
|
+
row = state[cid]
|
|
50
|
+
events.append((seq, op, row[0], row[1], row[2], row[3], row[4], ts))
|
|
51
|
+
if op == "DELETE":
|
|
52
|
+
del state[cid]
|
|
53
|
+
# deliver about 8% of the events late: swap them a few positions down the feed
|
|
54
|
+
for i in range(len(events) - 6):
|
|
55
|
+
if rng.random() < 0.08:
|
|
56
|
+
j = i + rng.randrange(1, 6)
|
|
57
|
+
events[i], events[j] = events[j], events[i]
|
|
58
|
+
events = [(pos + 1,) + e for pos, e in enumerate(events)]
|
|
59
|
+
|
|
60
|
+
cols = [("customer_id", "BIGINT NOT NULL", "Unique customer identifier"),
|
|
61
|
+
("customer_name", "STRING", "Customer name"),
|
|
62
|
+
("email", "STRING", "Email address (synthetic)"),
|
|
63
|
+
("city", "STRING", "City (changes over time)"),
|
|
64
|
+
("tier", "STRING", "Loyalty tier: Bronze, Silver, Gold (changes over time)")]
|
|
65
|
+
return Dataset("cdc", SUMMARY, seed, [
|
|
66
|
+
Table("customers_snapshot", cols + [("updated_at", "TIMESTAMP", "When the snapshot was taken")],
|
|
67
|
+
snapshot, "Customers as of 2026-09-30 (the starting point)", ["customer_id"]),
|
|
68
|
+
Table("customers_changes", [
|
|
69
|
+
("arrival_order", "BIGINT NOT NULL", "Order in which the event arrived (not always the true order)"),
|
|
70
|
+
("sequence_num", "BIGINT", "True order of the change in the source system: use this to sequence"),
|
|
71
|
+
("operation", "STRING", "INSERT, UPDATE or DELETE"),
|
|
72
|
+
] + [(c, t.replace(" NOT NULL", ""), m) for c, t, m in cols] + [
|
|
73
|
+
("changed_at", "TIMESTAMP", "When the change happened in the source system"),
|
|
74
|
+
], events, "Change feed for customers; about 8% of the events arrive out of order", ["arrival_order"]),
|
|
75
|
+
])
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""churn: telco subscribers with a learnable churn label, plus monthly usage."""
|
|
2
|
+
import datetime as dt
|
|
3
|
+
import math
|
|
4
|
+
import random
|
|
5
|
+
|
|
6
|
+
from ._core import Dataset, Table, pick
|
|
7
|
+
|
|
8
|
+
TITLE = "Telco churn (machine learning)"
|
|
9
|
+
ICON = "📉"
|
|
10
|
+
SUMMARY = ("5,000 telco subscribers with a churn label that really depends on the features (contract type, "
|
|
11
|
+
"support calls, payment failures, tenure), plus 30,000 rows of monthly usage for feature engineering.")
|
|
12
|
+
USE_FOR = "MLflow tracking, feature tables, train/test splits, hyperparameter tuning, model serving, drift monitoring"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def generate(seed=42, scale=1.0):
|
|
16
|
+
rng = random.Random(seed)
|
|
17
|
+
n = int(5000 * scale)
|
|
18
|
+
subscribers, usage = [], []
|
|
19
|
+
months = [dt.date(2026, m, 1) for m in range(5, 11)]
|
|
20
|
+
for sid in range(1, n + 1):
|
|
21
|
+
plan = pick(rng, [("Basic", .45), ("Plus", .35), ("Premium", .20)])
|
|
22
|
+
contract = pick(rng, [("Monthly", .55), ("One year", .28), ("Two year", .17)])
|
|
23
|
+
tenure = int(1 + rng.random() ** 1.6 * 71)
|
|
24
|
+
age = rng.randint(18, 80)
|
|
25
|
+
fee = {"Basic": 19.0, "Plus": 39.0, "Premium": 69.0}[plan] + round(rng.uniform(-3, 6), 2)
|
|
26
|
+
support_calls = int(rng.random() ** 2.2 * 9)
|
|
27
|
+
payment_failures = int(rng.random() ** 3.0 * 5)
|
|
28
|
+
has_app = rng.random() < 0.6
|
|
29
|
+
weekly_minutes = round(max(0.0, rng.gauss(220, 90) * (1.25 if has_app else 0.85)), 1)
|
|
30
|
+
region = pick(rng, [("North", .25), ("South", .25), ("East", .25), ("West", .25)])
|
|
31
|
+
z = (-1.4
|
|
32
|
+
+ 0.42 * support_calls
|
|
33
|
+
+ 0.65 * payment_failures
|
|
34
|
+
- 0.035 * tenure
|
|
35
|
+
+ {"Monthly": 0.9, "One year": -0.4, "Two year": -1.3}[contract]
|
|
36
|
+
- (0.5 if has_app else 0.0)
|
|
37
|
+
- 0.0025 * weekly_minutes
|
|
38
|
+
+ {"Basic": 0.1, "Plus": 0.0, "Premium": -0.2}[plan])
|
|
39
|
+
churned = rng.random() < 1.0 / (1.0 + math.exp(-z))
|
|
40
|
+
subscribers.append((sid, dt.date(2026, 10, 1) - dt.timedelta(days=tenure * 30), plan, contract, region, age,
|
|
41
|
+
tenure, round(fee, 2), support_calls, payment_failures, has_app, weekly_minutes,
|
|
42
|
+
1 if churned else 0))
|
|
43
|
+
trend = -0.12 if churned else 0.01
|
|
44
|
+
for k, month in enumerate(months):
|
|
45
|
+
factor = max(0.05, 1.0 + trend * k + rng.gauss(0, 0.08))
|
|
46
|
+
minutes = round(weekly_minutes * 4.3 * factor, 1)
|
|
47
|
+
usage.append((sid, month, minutes, round(max(0.0, rng.gauss(12, 5) * factor), 2),
|
|
48
|
+
round(fee + (rng.uniform(0, 12) if rng.random() < 0.15 else 0.0), 2)))
|
|
49
|
+
|
|
50
|
+
return Dataset("churn", SUMMARY, seed, [
|
|
51
|
+
Table("subscribers", [
|
|
52
|
+
("subscriber_id", "BIGINT NOT NULL", "Unique subscriber identifier"),
|
|
53
|
+
("signup_date", "DATE", "Date the subscription started"),
|
|
54
|
+
("plan", "STRING", "Basic, Plus or Premium"),
|
|
55
|
+
("contract", "STRING", "Monthly, One year or Two year"),
|
|
56
|
+
("region", "STRING", "North, South, East or West"),
|
|
57
|
+
("age", "INT", "Age in years"),
|
|
58
|
+
("tenure_months", "INT", "Months since signup"),
|
|
59
|
+
("monthly_fee", "DOUBLE", "Monthly fee in USD"),
|
|
60
|
+
("support_calls_90d", "INT", "Support calls in the last 90 days"),
|
|
61
|
+
("payment_failures_6m", "INT", "Failed payments in the last 6 months"),
|
|
62
|
+
("has_app", "BOOLEAN", "Uses the mobile app"),
|
|
63
|
+
("avg_weekly_minutes", "DOUBLE", "Average minutes used per week"),
|
|
64
|
+
("churned", "INT", "Label: 1 = cancelled within the next month, 0 = stayed"),
|
|
65
|
+
], subscribers, "Telco subscribers with the churn label", ["subscriber_id"]),
|
|
66
|
+
Table("usage_monthly", [
|
|
67
|
+
("subscriber_id", "BIGINT NOT NULL", "FK to subscribers"),
|
|
68
|
+
("month", "DATE NOT NULL", "First day of the month"),
|
|
69
|
+
("minutes", "DOUBLE", "Minutes used in the month"),
|
|
70
|
+
("data_gb", "DOUBLE", "Mobile data used in GB"),
|
|
71
|
+
("billed_amount", "DOUBLE", "Amount billed in USD"),
|
|
72
|
+
], usage, "Monthly usage per subscriber (May - Oct 2026)", ["subscriber_id", "month"],
|
|
73
|
+
[("subscriber_id", "subscribers", "subscriber_id")]),
|
|
74
|
+
])
|