databricks360 0.1.0__py3-none-any.whl

databricks360/__init__.py

@@ -0,0 +1,67 @@
+"""Lakehouse Academy — install Databricks course lab environments.
+
+Run inside a Databricks notebook:
+
+    %pip install databricks360
+    dbutils.library.restartPython()
+
+    import databricks360 as academy
+    academy.list_courses()
+    academy.install('genie-agents')
+
+`install` writes the lab notebooks into your workspace. You then run them in
+order. Nothing is executed for you: generating the data is real work on your
+warehouse, and watching it happen is part of the lesson.
+"""
+
+from __future__ import annotations
+
+from ._catalog import Course, available_courses, get_course
+from ._install import Installation, build_notebook_source, install
+
+__version__ = "0.1.0"
+__all__ = [
+    "install",
+    "list_courses",
+    "get_course",
+    "available_courses",
+    "build_notebook_source",
+    "Course",
+    "Installation",
+    "__version__",
+]
+
+
+def list_courses() -> None:
+    """Print the available course labs."""
+    courses = available_courses()
+    if not courses:
+        print("No courses bundled in this build.")
+        return
+
+    for course in courses:
+        print(f"\n{course.id}")
+        print(f"  {course.title}")
+        if course.description:
+            for line in _wrap(course.description, 76):
+                print(f"    {line}")
+        print(f"  catalog: {course.default_catalog}   notebooks: {len(course.notebooks)}")
+        if course.tiers:
+            print("  tiers:")
+            for name, tier in course.tiers.items():
+                marker = " (default)" if name == course.default_tier else ""
+                print(f"    {name}{marker} — {tier.description}")
+    print(f"\nInstall with: academy.install('{courses[0].id}')\n")
+
+
+def _wrap(text: str, width: int) -> list[str]:
+    words, lines, current = text.split(), [], ""
+    for word in words:
+        if len(current) + len(word) + 1 > width:
+            lines.append(current)
+            current = word
+        else:
+            current = f"{current} {word}".strip()
+    if current:
+        lines.append(current)
+    return lines

databricks360/_catalog.py

@@ -0,0 +1,99 @@
+"""Course discovery: reads the manifests bundled with the package."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from importlib import resources
+from typing import Any
+
+
+@dataclass(frozen=True)
+class Notebook:
+    """One notebook in a course's lab, in run order."""
+
+    order: int
+    name: str
+    sql: str
+    title: str
+    intro: str = ""
+    requires_admin: bool = False
+    slow: bool = False
+
+
+@dataclass(frozen=True)
+class Tier:
+    name: str
+    values: dict
+    description: str = ""
+
+
+@dataclass(frozen=True)
+class Course:
+    id: str
+    title: str
+    description: str
+    default_catalog: str
+    notebooks: list = field(default_factory=list)
+    tiers: dict = field(default_factory=dict)
+    default_tier: str = "small"
+
+    @property
+    def package(self) -> str:
+        return f"databricks360.courses.{self.id.replace('-', '_')}"
+
+
+def _course_packages() -> list:
+    root = resources.files("databricks360.courses")
+    return sorted(
+        p.name
+        for p in root.iterdir()
+        if p.is_dir() and not p.name.startswith("_") and (p / "manifest.json").is_file()
+    )
+
+
+def _load(pkg_name: str) -> Course:
+    raw: dict = json.loads(
+        (resources.files(f"databricks360.courses.{pkg_name}") / "manifest.json")
+        .read_text(encoding="utf-8")
+    )
+    return Course(
+        id=raw["id"],
+        title=raw["title"],
+        description=raw.get("description", ""),
+        default_catalog=raw.get("default_catalog", "main"),
+        default_tier=raw.get("default_tier", "small"),
+        notebooks=[
+            Notebook(
+                order=n["order"],
+                name=n["name"],
+                sql=n["sql"],
+                title=n["title"],
+                intro=n.get("intro", ""),
+                requires_admin=n.get("requires_admin", False),
+                slow=n.get("slow", False),
+            )
+            for n in sorted(raw["notebooks"], key=lambda n: n["order"])
+        ],
+        tiers={
+            name: Tier(name=name, values=t["values"], description=t.get("description", ""))
+            for name, t in raw.get("tiers", {}).items()
+        },
+    )
+
+
+def available_courses() -> list:
+    return [_load(p) for p in _course_packages()]
+
+
+def get_course(course_id: str) -> Course:
+    wanted = course_id.replace("-", "_")
+    for pkg in _course_packages():
+        if pkg == wanted:
+            return _load(pkg)
+    known = ", ".join(c.id for c in available_courses()) or "none"
+    raise ValueError(f"Unknown course {course_id!r}. Available: {known}")
+
+
+def read_sql(course: Course, filename: str) -> str:
+    return (resources.files(course.package) / filename).read_text(encoding="utf-8")

databricks360/_install.py

@@ -0,0 +1,162 @@
+"""Materialise a course's lab notebooks into a Databricks workspace.
+
+Designed to run from inside a Databricks notebook, where databricks-sdk picks up
+the notebook's own identity — no host, token or profile to configure. It also
+works from a laptop if a Databricks CLI profile is present.
+"""
+
+from __future__ import annotations
+
+import base64
+from dataclasses import dataclass
+
+from ._catalog import Course, Notebook, read_sql
+from ._notebook import render_template, sql_to_notebook, unresolved_placeholders
+
+
+@dataclass
+class InstalledNotebook:
+    order: int
+    name: str
+    path: str
+    slow: bool
+    requires_admin: bool
+
+
+@dataclass
+class Installation:
+    course_id: str
+    folder: str
+    catalog: str
+    tier: str
+    notebooks: list[InstalledNotebook]
+
+    def __repr__(self) -> str:  # what a notebook cell shows
+        lines = [
+            f"Installed '{self.course_id}' → {self.folder}",
+            f"  catalog: {self.catalog}    tier: {self.tier}",
+            "",
+            "  Run these in order:",
+        ]
+        for nb in self.notebooks:
+            flags = []
+            if nb.slow:
+                flags.append("slow")
+            if nb.requires_admin:
+                flags.append("needs admin")
+            suffix = f"   ({', '.join(flags)})" if flags else ""
+            lines.append(f"    {nb.order}. {nb.name}{suffix}")
+        return "\n".join(lines)
+
+
+def _workspace_client():
+    try:
+        from databricks.sdk import WorkspaceClient
+    except ImportError as exc:  # pragma: no cover
+        raise ImportError(
+            "databricks-sdk is required. Inside a Databricks notebook run:\n"
+            "    %pip install databricks360\n"
+            "    dbutils.library.restartPython()"
+        ) from exc
+    return WorkspaceClient()
+
+
+def _default_folder(client, course: Course) -> str:
+    """Default to the caller's home folder, matching how dbdemos behaves."""
+    try:
+        user = client.current_user.me().user_name
+    except Exception:  # pragma: no cover - offline / unauthenticated
+        user = None
+    base = f"/Workspace/Users/{user}" if user else "/Workspace/Shared"
+    return f"{base}/databricks360/{course.id}"
+
+
+def build_notebook_source(
+    course: Course,
+    notebook: Notebook,
+    *,
+    catalog: str,
+    tier: str,
+) -> str:
+    """Render one notebook's source. Pure — no workspace calls, so it is testable."""
+    if tier not in course.tiers and course.tiers:
+        known = ", ".join(sorted(course.tiers))
+        raise ValueError(f"Unknown tier {tier!r} for {course.id}. Available: {known}")
+
+    values = {"CATALOG": catalog, "TIER": tier}
+    if course.tiers:
+        values.update(course.tiers[tier].values)
+
+    sql = render_template(read_sql(course, notebook.sql), values)
+    intro = render_template(notebook.intro, values)
+    source = sql_to_notebook(sql, title=notebook.title, intro=intro or None)
+
+    leftover = unresolved_placeholders(source)
+    if leftover:
+        raise ValueError(
+            f"{notebook.sql}: unresolved placeholders {leftover}. "
+            "Add them to the tier values in manifest.json, or fix the typo."
+        )
+    return source
+
+
+def install(
+    course_id: str,
+    *,
+    path: str | None = None,
+    catalog: str | None = None,
+    tier: str | None = None,
+    overwrite: bool = False,
+    dry_run: bool = False,
+) -> Installation:
+    """Create the lab notebooks for a course in the workspace.
+
+    Notebooks are written but never executed. Data generation is deliberate work
+    on the learner's warehouse, and Module 0 of the course is partly about
+    watching it happen rather than having it appear.
+    """
+    from ._catalog import get_course
+
+    course = get_course(course_id)
+    catalog = catalog or course.default_catalog
+    tier = tier or course.default_tier
+
+    client = None if dry_run else _workspace_client()
+    folder = path or (
+        f"/Workspace/Shared/databricks360/{course.id}"
+        if dry_run
+        else _default_folder(client, course)
+    )
+    folder = folder.rstrip("/")
+
+    installed: list[InstalledNotebook] = []
+
+    if not dry_run:
+        from databricks.sdk.service.workspace import ImportFormat, Language
+
+        client.workspace.mkdirs(folder)
+
+    for nb in course.notebooks:
+        source = build_notebook_source(course, nb, catalog=catalog, tier=tier)
+        target = f"{folder}/{nb.name}"
+
+        if not dry_run:
+            client.workspace.import_(
+                path=target,
+                content=base64.b64encode(source.encode("utf-8")).decode("ascii"),
+                format=ImportFormat.SOURCE,
+                language=Language.SQL,
+                overwrite=overwrite,
+            )
+
+        installed.append(
+            InstalledNotebook(
+                order=nb.order, name=nb.name, path=target,
+                slow=nb.slow, requires_admin=nb.requires_admin,
+            )
+        )
+
+    return Installation(
+        course_id=course.id, folder=folder, catalog=catalog,
+        tier=tier, notebooks=installed,
+    )

databricks360/_notebook.py

@@ -0,0 +1,107 @@
+"""Turn a plain .sql file into Databricks notebook source.
+
+Databricks notebooks are stored as a flat source file: a header comment naming
+the language, cells separated by a magic delimiter, and markdown cells prefixed
+with a MAGIC marker. Generating that here means the SQL files stay ordinary,
+runnable SQL that anyone can open in a text editor or paste into an editor tab.
+"""
+
+from __future__ import annotations
+
+import re
+
+CELL_DELIMITER = "-- COMMAND ----------"
+SQL_HEADER = "-- Databricks notebook source"
+
+# A run of dashes at least this long marks a section heading in the SQL files.
+_BANNER = re.compile(r"^-- =={2,}\s*$")
+
+
+def _md_cell(lines: list[str]) -> str:
+    """Render markdown as a %md cell."""
+    body = "\n".join(f"-- MAGIC {line}" if line else "-- MAGIC" for line in lines)
+    return f"-- MAGIC %md\n{body}"
+
+
+def split_sql_sections(sql: str) -> list[tuple[str, str]]:
+    """Split a lab SQL file into (title, body) sections on its banner comments.
+
+    The lab files use a banner of the form:
+
+        -- ====================================
+        -- dim_date  —  FLAW #2: ...
+        -- more description
+        -- ====================================
+
+    Each banner starts a new notebook cell, with the banner text becoming a
+    markdown cell above the SQL. Files without banners return a single section.
+    """
+    lines = sql.splitlines()
+    sections: list[tuple[str, list[str]]] = []
+    current_title = ""
+    current_body: list[str] = []
+    i = 0
+
+    while i < len(lines):
+        if _BANNER.match(lines[i]):
+            # Collect the comment block until the closing banner.
+            j = i + 1
+            header: list[str] = []
+            while j < len(lines) and not _BANNER.match(lines[j]):
+                header.append(re.sub(r"^--\s?", "", lines[j]))
+                j += 1
+            if j < len(lines):  # closing banner found
+                if current_body or current_title:
+                    sections.append((current_title, current_body))
+                current_title = "\n".join(header).strip()
+                current_body = []
+                i = j + 1
+                continue
+        current_body.append(lines[i])
+        i += 1
+
+    if current_body or current_title:
+        sections.append((current_title, current_body))
+
+    return [(t, "\n".join(b).strip()) for t, b in sections]
+
+
+def sql_to_notebook(
+    sql: str,
+    *,
+    title: str,
+    intro: str | None = None,
+) -> str:
+    """Build Databricks SQL notebook source from a lab SQL file."""
+    cells: list[str] = []
+
+    heading = [f"# {title}"]
+    if intro:
+        heading += ["", *intro.splitlines()]
+    cells.append(_md_cell(heading))
+
+    for section_title, body in split_sql_sections(sql):
+        if section_title:
+            first, *rest = section_title.splitlines()
+            md = [f"## {first.strip()}"]
+            if rest:
+                md += ["", *[line.strip() for line in rest]]
+            cells.append(_md_cell(md))
+        if body:
+            cells.append(body)
+
+    return f"{SQL_HEADER}\n" + f"\n\n{CELL_DELIMITER}\n\n".join(cells) + "\n"
+
+
+def render_template(sql: str, values: dict[str, str]) -> str:
+    """Substitute {{PLACEHOLDER}} tokens. Unknown tokens are left untouched so a
+    typo shows up in the notebook rather than silently becoming an empty string."""
+    out = sql
+    for key, value in values.items():
+        out = out.replace(f"{{{{{key}}}}}", str(value))
+    return out
+
+
+def unresolved_placeholders(text: str) -> list[str]:
+    """Any {{TOKEN}} left after substitution — a bug worth failing loudly on."""
+    return sorted(set(re.findall(r"\{\{([A-Z0-9_]+)\}\}", text)))

databricks360/courses/genie_agents/01_catalog_and_schemas.sql

@@ -0,0 +1,31 @@
+-- ============================================================================
+-- Meridian Financial Group — 01. Catalog, schemas, volume
+-- Run first. Idempotent.
+-- ============================================================================
+
+CREATE CATALOG IF NOT EXISTS {{CATALOG}}
+  COMMENT 'Meridian Financial Group — synthetic teaching dataset for the Genie Agents course. Contains deliberate data-quality flaws; not a reference implementation.';
+
+-- Curated business data. Everything a Genie Agent is pointed at lives here.
+CREATE SCHEMA IF NOT EXISTS {{CATALOG}}.core
+  COMMENT 'Core banking facts and dimensions for Meridian Financial Group.';
+
+-- Reference material: unstructured documents for Agent mode and Knowledge Assistant.
+CREATE SCHEMA IF NOT EXISTS {{CATALOG}}.ref
+  COMMENT 'Reference and unstructured material: credit committee memos, branch notes, complaint letters.';
+
+-- Staging holds the deliberately awful objects used in Modules 7 and 13.
+-- Kept in a separate schema so the "before" and "after" states are visibly distinct.
+CREATE SCHEMA IF NOT EXISTS {{CATALOG}}.staging
+  COMMENT 'Deliberately unfit-for-purpose objects used to teach scoping and latency. Never point a production agent here.';
+
+CREATE VOLUME IF NOT EXISTS {{CATALOG}}.ref.documents
+  COMMENT 'PDFs attached to the agent for Agent mode: credit committee memos, branch manager notes, customer complaint letters.';
+
+-- ----------------------------------------------------------------------------
+-- Verify
+-- ----------------------------------------------------------------------------
+SELECT 'catalog' AS object, '{{CATALOG}}' AS name
+UNION ALL SELECT 'schema', schema_name FROM {{CATALOG}}.information_schema.schemata
+WHERE catalog_name = '{{CATALOG}}'
+ORDER BY object, name;

databricks360/courses/genie_agents/02_dimensions.sql

@@ -0,0 +1,291 @@
+-- ============================================================================
+-- Meridian Financial Group — 02. Dimensions
+--
+-- Plants four of the nine flaws:
+--   #2  fiscal year starts 1 October          -> dim_date
+--   #3  region and state are CODES, not names -> dim_branch
+--   #5  two competing product hierarchies     -> dim_product
+--   #8  PII columns present and unmasked      -> dim_customer
+--   #9  multi-currency needs as-of-date FX    -> dim_fx_rate
+--
+-- All values derive from hash() of the row key, never rand(), so every learner
+-- gets byte-identical data and benchmark ground-truth SQL stays valid.
+--
+-- ANCHOR DATE: 2026-09-30 (last day of FY2026). Ages and tenures are computed
+-- from this fixed date rather than current_date, or the data would drift and
+-- benchmark answers would rot.
+-- ============================================================================
+
+
+-- ============================================================================
+-- dim_date  —  FLAW #2: fiscal year begins 1 October
+-- FY2025 = 2024-10-01 .. 2025-09-30
+-- FY2026 = 2025-10-01 .. 2026-09-30
+-- Two complete fiscal years, so "this year vs last year" questions work.
+-- ============================================================================
+CREATE OR REPLACE TABLE {{CATALOG}}.core.dim_date
+COMMENT 'Calendar with Meridian fiscal attributes. Meridian fiscal year starts 1 October: FY2026 = 2025-10-01 to 2026-09-30. Fiscal and calendar columns are both present and will disagree.'
+AS
+WITH d AS (
+  SELECT explode(sequence(DATE'2024-10-01', DATE'2026-09-30', INTERVAL 1 DAY)) AS date_key
+),
+calc AS (
+  SELECT
+    date_key,
+    year(date_key)  AS cal_y,
+    month(date_key) AS cal_m,
+    -- Oct-Dec belong to the NEXT fiscal year
+    year(date_key) + CASE WHEN month(date_key) >= 10 THEN 1 ELSE 0 END AS fy_num,
+    -- Fiscal quarter: Q1=Oct-Dec, Q2=Jan-Mar, Q3=Apr-Jun, Q4=Jul-Sep
+    (pmod(month(date_key) + 2, 12) DIV 3) + 1 AS fq_num,
+    -- Fiscal month 1..12 where 1 = October
+    pmod(month(date_key) + 2, 12) + 1 AS fm_num
+  FROM d
+)
+SELECT
+  date_key,
+  concat('FY', cast(fy_num AS STRING))                                AS fiscal_year,
+  concat('FY', cast(fy_num AS STRING), '-Q', cast(fq_num AS STRING))   AS fiscal_quarter,
+  fm_num                                                              AS fiscal_month,
+  cal_y                                                               AS calendar_year,
+  cal_m                                                               AS calendar_month,
+  date_format(date_key, 'EEEE')                                       AS day_name,
+  -- Approximate US banking calendar: weekends plus the fixed-date federal
+  -- holidays. Good enough to make "business days" a meaningful filter; not a
+  -- substitute for a real holiday calendar.
+  CASE
+    WHEN dayofweek(date_key) IN (1, 7) THEN false
+    WHEN date_format(date_key, 'MM-dd') IN
+         ('01-01','06-19','07-04','11-11','12-25') THEN false
+    ELSE true
+  END                                                                 AS is_business_day
+FROM calc;
+
+ALTER TABLE {{CATALOG}}.core.dim_date ALTER COLUMN date_key
+  COMMENT 'Calendar date. One row per day.';
+ALTER TABLE {{CATALOG}}.core.dim_date ALTER COLUMN fiscal_year
+  COMMENT 'Meridian fiscal year, format FY2026. The fiscal year STARTS 1 OCTOBER, so FY2026 runs 2025-10-01 to 2026-09-30. "Last year" means the prior FISCAL year unless the user says "calendar year".';
+ALTER TABLE {{CATALOG}}.core.dim_date ALTER COLUMN fiscal_quarter
+  COMMENT 'Meridian fiscal quarter, format FY2026-Q3. Q1 = Oct-Dec, Q2 = Jan-Mar, Q3 = Apr-Jun, Q4 = Jul-Sep.';
+ALTER TABLE {{CATALOG}}.core.dim_date ALTER COLUMN calendar_year
+  COMMENT 'Calendar year. Use ONLY when the user explicitly asks for calendar periods; otherwise use fiscal_year.';
+ALTER TABLE {{CATALOG}}.core.dim_date ALTER COLUMN is_business_day
+  COMMENT 'False for weekends and fixed-date US federal holidays.';
+
+
+-- ============================================================================
+-- dim_branch  —  FLAW #3: region and state are stored as CODES
+-- Users say "Northeast", "the West Coast", "California".
+-- The table holds NE, WEST, CA. Without entity matching, Genie filters on the
+-- spoken form, matches nothing, and returns a confident zero.
+-- ============================================================================
+CREATE OR REPLACE TABLE {{CATALOG}}.core.dim_branch
+COMMENT 'Meridian branch network, 340 branches. Region and state are stored as SHORT CODES, not the names people say out loud.'
+AS
+WITH b AS (SELECT id AS n FROM range(1, 341)),
+assigned AS (
+  SELECT
+    n,
+    -- Deterministic region assignment, weighted toward NE and WEST
+    CASE pmod(hash(concat('branch-region-', n)), 10)
+      WHEN 0 THEN 'NE' WHEN 1 THEN 'NE' WHEN 2 THEN 'NE'
+      WHEN 3 THEN 'SE' WHEN 4 THEN 'SE'
+      WHEN 5 THEN 'MW' WHEN 6 THEN 'MW'
+      ELSE 'WEST'
+    END AS region,
+    pmod(hash(concat('branch-state-', n)), 4)   AS state_pick,
+    pmod(hash(concat('branch-chan-', n)), 10)   AS chan_pick,
+    pmod(hash(concat('branch-open-', n)), 7300) AS open_offset
+  FROM b
+)
+SELECT
+  concat('BR', lpad(cast(n AS STRING), 4, '0')) AS branch_id,
+  concat('Branch ', lpad(cast(n AS STRING), 4, '0')) AS branch_name,
+  region,
+  -- States are real US codes, mapped to their actual region
+  CASE region
+    WHEN 'NE'   THEN element_at(array('NY','MA','NJ','PA'), state_pick + 1)
+    WHEN 'SE'   THEN element_at(array('FL','GA','NC','TN'), state_pick + 1)
+    WHEN 'MW'   THEN element_at(array('IL','OH','MI','MN'), state_pick + 1)
+    ELSE             element_at(array('CA','WA','AZ','CO'), state_pick + 1)
+  END AS state,
+  CASE WHEN chan_pick < 7 THEN 'RETAIL'
+       WHEN chan_pick < 9 THEN 'COMMERCIAL'
+       ELSE 'PRIVATE_CLIENT' END AS channel,
+  date_add(DATE'2006-01-01', open_offset) AS opened_date
+FROM assigned;
+
+ALTER TABLE {{CATALOG}}.core.dim_branch ALTER COLUMN region
+  COMMENT 'Sales region CODE. Values: NE, SE, MW, WEST. Users say "Northeast" (NE), "Southeast" (SE), "Midwest" (MW), "the West" or "West Coast" (WEST).';
+ALTER TABLE {{CATALOG}}.core.dim_branch ALTER COLUMN state
+  COMMENT 'Two-letter US state CODE, e.g. CA, NY, TX. Users say the full state name ("California"). Enable entity matching on this column.';
+ALTER TABLE {{CATALOG}}.core.dim_branch ALTER COLUMN channel
+  COMMENT 'Branch servicing model. Values: RETAIL, COMMERCIAL, PRIVATE_CLIENT.';
+
+
+-- ============================================================================
+-- dim_product  —  FLAW #5: two competing hierarchies
+-- product_category   = how the business talks about products
+-- regulatory_product_class = how they roll up for Basel/regulatory reporting
+-- Both are correct. Leaving both visible produces two different answers to
+-- "revenue by product", and nobody can tell which one they got.
+-- ============================================================================
+CREATE OR REPLACE TABLE {{CATALOG}}.core.dim_product
+COMMENT 'Product master. Carries TWO hierarchies that do not align: product_category (business view) and regulatory_product_class (Basel reporting view). Expose only one to an agent.'
+AS
+SELECT * FROM VALUES
+  ('P01','Everyday Checking',            'DEPOSITS','RETAIL_DEPOSIT'),
+  ('P02','Premier Checking',             'DEPOSITS','RETAIL_DEPOSIT'),
+  ('P03','High-Yield Savings',           'DEPOSITS','RETAIL_DEPOSIT'),
+  ('P04','12-Month CD',                  'DEPOSITS','RETAIL_DEPOSIT'),
+  ('P05','Money Market',                 'DEPOSITS','RETAIL_DEPOSIT'),
+  ('P06','Business Checking',            'DEPOSITS','CORPORATE'),
+  ('P07','Rewards Credit Card',          'CARDS','QUALIFYING_REVOLVING'),
+  ('P08','Cashback Credit Card',         'CARDS','QUALIFYING_REVOLVING'),
+  ('P09','Secured Credit Card',          'CARDS','OTHER_RETAIL'),
+  ('P10','Debit Card',                   'CARDS','RETAIL_DEPOSIT'),
+  ('P11','Commercial Card',              'CARDS','CORPORATE'),
+  ('P12','30-Year Fixed Mortgage',       'LENDING','RESIDENTIAL_MORTGAGE'),
+  ('P13','15-Year Fixed Mortgage',       'LENDING','RESIDENTIAL_MORTGAGE'),
+  ('P14','HELOC',                        'LENDING','RESIDENTIAL_MORTGAGE'),
+  ('P15','Auto Loan',                    'LENDING','OTHER_RETAIL'),
+  ('P16','Personal Loan',                'LENDING','OTHER_RETAIL'),
+  ('P17','Small Business Term Loan',     'LENDING','CORPORATE'),
+  ('P18','Managed Portfolio',            'WEALTH','OFF_BALANCE_SHEET'),
+  ('P19','Advisory Account',             'WEALTH','OFF_BALANCE_SHEET'),
+  ('P20','Traditional IRA',              'WEALTH','OFF_BALANCE_SHEET')
+AS t(product_id, product_name, product_category, regulatory_product_class);
+
+ALTER TABLE {{CATALOG}}.core.dim_product ALTER COLUMN product_category
+  COMMENT 'BUSINESS product hierarchy. Values: DEPOSITS, CARDS, LENDING, WEALTH. This is the one business users mean by "product". Prefer this and hide regulatory_product_class.';
+ALTER TABLE {{CATALOG}}.core.dim_product ALTER COLUMN regulatory_product_class
+  COMMENT 'REGULATORY (Basel) product hierarchy used for capital reporting. Does NOT align with product_category - a Debit Card is CARDS here but RETAIL_DEPOSIT there. Never mix the two in one rollup.';
+
+
+-- ============================================================================
+-- dim_customer  —  FLAW #8: PII columns, deliberately unprotected
+-- ssn_last4, email, dob and annual_income all sit here in the clear so that
+-- Module 6 can demonstrate column masks doing real work, and demonstrate a
+-- text instruction ("never show PII") failing to.
+--
+-- Every value is synthetic. Emails use example.com (RFC 2606 reserved) and
+-- ssn_last4 is a hashed 4-digit string, not derived from anything real.
+-- ============================================================================
+CREATE OR REPLACE TABLE {{CATALOG}}.core.dim_customer
+COMMENT 'Retail and commercial customers, 2.1M rows. CONTAINS PII: ssn_last4, email, dob, annual_income. All values are synthetic. Protect with Unity Catalog column masks, never with agent instructions.'
+AS
+WITH c AS (SELECT id AS n FROM range(1, 2100001)),
+calc AS (
+  SELECT
+    n,
+    pmod(hash(concat('cust-seg-',    n)), 100)  AS seg_pick,
+    pmod(hash(concat('cust-tenure-', n)), 240)  AS tenure_m,
+    pmod(hash(concat('cust-branch-', n)), 340)  AS branch_n,
+    pmod(hash(concat('cust-ssn-',    n)), 10000) AS ssn4,
+    pmod(hash(concat('cust-age-',    n)), 67)   AS age_offset,
+    pmod(hash(concat('cust-inc-',    n)), 240)  AS inc_pick
+  FROM c
+)
+SELECT
+  concat('C', lpad(cast(n AS STRING), 8, '0')) AS customer_id,
+  CASE WHEN seg_pick < 62 THEN 'MASS'
+       WHEN seg_pick < 84 THEN 'AFFLUENT'
+       WHEN seg_pick < 94 THEN 'PRIVATE'
+       ELSE 'COMMERCIAL' END                   AS segment,
+  tenure_m                                     AS tenure_months,
+  concat('BR', lpad(cast(branch_n + 1 AS STRING), 4, '0')) AS home_branch_id,
+  lpad(cast(ssn4 AS STRING), 4, '0')           AS ssn_last4,
+  concat('customer', cast(n AS STRING), '@example.com')    AS email,
+  -- Ages 18-84 as of the 2026-09-30 anchor
+  date_add(DATE'2026-09-30', -1 * ((18 + age_offset) * 365 + pmod(hash(concat('cust-dob-', n)), 365))) AS dob,
+  cast(25000 + inc_pick * 1250 AS DECIMAL(12,2)) AS annual_income
+FROM calc;
+
+ALTER TABLE {{CATALOG}}.core.dim_customer ALTER COLUMN segment
+  COMMENT 'Customer segment. Values: MASS, AFFLUENT, PRIVATE, COMMERCIAL.';
+ALTER TABLE {{CATALOG}}.core.dim_customer ALTER COLUMN tenure_months
+  COMMENT 'Months since the customer relationship began, as of 2026-09-30.';
+ALTER TABLE {{CATALOG}}.core.dim_customer ALTER COLUMN ssn_last4
+  COMMENT 'PII. Last four digits of tax identifier (synthetic). Must be masked. Never expose in an answer.';
+ALTER TABLE {{CATALOG}}.core.dim_customer ALTER COLUMN email
+  COMMENT 'PII. Contact email (synthetic, example.com). Must be masked. Never expose in an answer.';
+ALTER TABLE {{CATALOG}}.core.dim_customer ALTER COLUMN dob
+  COMMENT 'PII. Date of birth (synthetic). Must be masked. Use age bands for analysis instead.';
+ALTER TABLE {{CATALOG}}.core.dim_customer ALTER COLUMN annual_income
+  COMMENT 'PII. Self-reported annual income in USD (synthetic). Restricted - finance roles only.';
+
+
+-- ============================================================================
+-- dim_account  —  the join hub
+-- Every fact joins to a dimension THROUGH this table. Getting its cardinality
+-- wrong is how "revenue by region" quietly multiplies (Module 9).
+-- ============================================================================
+CREATE OR REPLACE TABLE {{CATALOG}}.core.dim_account
+COMMENT 'Account master, ~2.9M rows. The join hub: transactions and balances reach customer, branch and product through this table.'
+AS
+WITH a AS (SELECT id AS n FROM range(1, 2900001)),
+calc AS (
+  SELECT
+    n,
+    pmod(hash(concat('acct-cust-',   n)), 2100000) AS cust_n,
+    pmod(hash(concat('acct-prod-',   n)), 20)      AS prod_n,
+    pmod(hash(concat('acct-branch-', n)), 340)     AS branch_n,
+    pmod(hash(concat('acct-open-',   n)), 5475)    AS open_offset,
+    pmod(hash(concat('acct-status-', n)), 100)     AS status_pick
+  FROM a
+)
+SELECT
+  concat('A', lpad(cast(n AS STRING), 9, '0'))               AS account_id,
+  concat('C', lpad(cast(cust_n + 1 AS STRING), 8, '0'))      AS customer_id,
+  concat('P', lpad(cast(prod_n + 1 AS STRING), 2, '0'))      AS product_id,
+  concat('BR', lpad(cast(branch_n + 1 AS STRING), 4, '0'))   AS branch_id,
+  date_add(DATE'2011-10-01', open_offset)                    AS opened_date,
+  CASE WHEN status_pick < 8
+       THEN date_add(DATE'2011-10-01', open_offset + 400 + pmod(hash(concat('acct-close-', n)), 1500))
+       ELSE NULL END                                         AS closed_date,
+  CASE WHEN status_pick < 8 THEN 'CLOSED' ELSE 'OPEN' END    AS status
+FROM calc;
+
+ALTER TABLE {{CATALOG}}.core.dim_account ALTER COLUMN status
+  COMMENT 'Account status. Values: OPEN, CLOSED. Roughly 8% are CLOSED. Exclude CLOSED accounts from "active customer" style questions.';
+ALTER TABLE {{CATALOG}}.core.dim_account ALTER COLUMN closed_date
+  COMMENT 'Date the account was closed, NULL while open.';
+
+
+-- ============================================================================
+-- dim_fx_rate  —  FLAW #9: multi-currency needs an as-of-date join
+-- Commercial transactions settle in CAD and GBP. Summing amount across
+-- currencies without converting produces a number that does not tie to finance.
+-- ============================================================================
+CREATE OR REPLACE TABLE {{CATALOG}}.core.dim_fx_rate
+COMMENT 'Daily FX rates to USD. Transactions in CAD or GBP must be converted using the rate AS OF the transaction date, not the latest rate.'
+AS
+WITH d AS (
+  SELECT explode(sequence(DATE'2024-10-01', DATE'2026-09-30', INTERVAL 1 DAY)) AS rate_date
+),
+cur AS (SELECT explode(array('USD','CAD','GBP')) AS currency)
+SELECT
+  cur.currency,
+  d.rate_date,
+  CASE cur.currency
+    WHEN 'USD' THEN cast(1.0 AS DECIMAL(12,6))
+    -- Small deterministic drift around a plausible central rate
+    WHEN 'CAD' THEN cast(0.730 + (pmod(hash(concat('fx-cad-', d.rate_date)), 40) - 20) / 2000.0 AS DECIMAL(12,6))
+    ELSE            cast(1.265 + (pmod(hash(concat('fx-gbp-', d.rate_date)), 60) - 30) / 2000.0 AS DECIMAL(12,6))
+  END AS usd_rate
+FROM d CROSS JOIN cur;
+
+ALTER TABLE {{CATALOG}}.core.dim_fx_rate ALTER COLUMN usd_rate
+  COMMENT 'Multiply a native-currency amount by this to get USD. Join on BOTH currency AND the transaction date.';
+
+
+-- ----------------------------------------------------------------------------
+-- Row counts
+-- ----------------------------------------------------------------------------
+SELECT 'dim_date'     AS table_name, count(*) AS rows FROM {{CATALOG}}.core.dim_date
+UNION ALL SELECT 'dim_branch',   count(*) FROM {{CATALOG}}.core.dim_branch
+UNION ALL SELECT 'dim_product',  count(*) FROM {{CATALOG}}.core.dim_product
+UNION ALL SELECT 'dim_customer', count(*) FROM {{CATALOG}}.core.dim_customer
+UNION ALL SELECT 'dim_account',  count(*) FROM {{CATALOG}}.core.dim_account
+UNION ALL SELECT 'dim_fx_rate',  count(*) FROM {{CATALOG}}.core.dim_fx_rate
+ORDER BY table_name;

databricks360/courses/genie_agents/03_facts.sql

@@ -0,0 +1,295 @@
+-- ============================================================================
+-- Meridian Financial Group — 03. Facts
+--
+-- Plants the four remaining flaws:
+--   #1  fee_revenue is GROSS; reversals live in a separate table
+--   #4  DECLINED rows inflate transaction COUNTS while contributing no revenue
+--   #6  fct_loan_balances is a DAILY SNAPSHOT — SUM() across dates is ~30x wrong
+--   #7  "delinquent" / "seriously delinquent" / "default" / "charge-off" are
+--       four different things, all present, all defensible
+--
+-- TIER: the line marked  -- << TIER >>  controls scale.
+--   Small  20000000   default. Every module except 13.
+--   Large  900000000  Module 13 only (latency). Leave unclustered on purpose.
+-- ============================================================================
+
+
+-- ============================================================================
+-- fct_transactions  —  FLAWS #1, #4, #9
+-- ============================================================================
+CREATE OR REPLACE TABLE {{CATALOG}}.core.fct_transactions
+COMMENT 'Transaction-level fact. fee_revenue is GROSS - it does NOT deduct reversals, which live in fct_reversals. Only status = POSTED is revenue; DECLINED rows exist and will inflate transaction counts if not excluded.'
+AS
+WITH t AS (
+  SELECT id AS n FROM range(1, {{TXN_COUNT}} + 1)   -- << TIER >> set by the tier in manifest.json
+),
+calc AS (
+  SELECT
+    n,
+    pmod(hash(concat('txn-acct-',  n)), 2900000)  AS acct_n,
+    pmod(hash(concat('txn-day-',   n)), 730)      AS day_offset,
+    pmod(hash(concat('txn-amt-',   n)), 100000)   AS amt_pick,
+    pmod(hash(concat('txn-stat-',  n)), 1000)     AS stat_pick,
+    pmod(hash(concat('txn-mcc-',   n)), 12)       AS mcc_pick,
+    pmod(hash(concat('txn-cur-',   n)), 100)      AS cur_pick
+  FROM t
+),
+shaped AS (
+  SELECT
+    n,
+    concat('T', lpad(cast(n AS STRING), 12, '0'))              AS txn_id,
+    concat('A', lpad(cast(acct_n + 1 AS STRING), 9, '0'))      AS account_id,
+    date_add(DATE'2024-10-01', day_offset)                     AS txn_date,
+    -- Amounts are log-ish: many small, few large
+    cast(
+      CASE WHEN amt_pick < 70000 THEN 5    + amt_pick / 1000.0
+           WHEN amt_pick < 95000 THEN 120  + amt_pick / 200.0
+           ELSE                       900  + amt_pick / 50.0
+      END AS DECIMAL(14,2))                                    AS amount,
+    -- FLAW #4: only POSTED is revenue, but DECLINED is 6% of rows
+    CASE WHEN stat_pick < 880 THEN 'POSTED'
+         WHEN stat_pick < 920 THEN 'PENDING'
+         WHEN stat_pick < 980 THEN 'DECLINED'
+         ELSE                      'REVERSED'
+    END                                                        AS status,
+    element_at(array(
+      'GROCERY','RESTAURANT','FUEL','AIRLINE','HOTEL','RETAIL',
+      'UTILITIES','HEALTHCARE','ENTERTAINMENT','TRANSFER','ATM','PROFESSIONAL_SERVICES'
+    ), mcc_pick + 1)                                           AS merchant_category,
+    cur_pick
+  FROM calc
+)
+SELECT
+  s.txn_id,
+  s.account_id,
+  s.txn_date,
+  s.amount,
+  -- FLAW #1: GROSS fee revenue. Reversals are NOT deducted here.
+  -- Only POSTED transactions earn anything.
+  CASE WHEN s.status = 'POSTED'
+       THEN cast(round(s.amount * 0.0185, 2) AS DECIMAL(14,2))
+       ELSE cast(0 AS DECIMAL(14,2)) END                        AS fee_revenue,
+  CASE WHEN s.status = 'POSTED'
+       THEN cast(round(s.amount * 0.0110, 2) AS DECIMAL(14,2))
+       ELSE cast(0 AS DECIMAL(14,2)) END                        AS interchange,
+  s.merchant_category,
+  -- FLAW #9: COMMERCIAL customers settle in CAD/GBP. Everyone else is USD.
+  -- Summing amount across currencies without converting is silently wrong.
+  CASE WHEN c.segment = 'COMMERCIAL' AND s.cur_pick < 40
+         THEN CASE WHEN s.cur_pick < 25 THEN 'CAD' ELSE 'GBP' END
+       ELSE 'USD' END                                           AS currency,
+  s.status
+FROM shaped s
+JOIN {{CATALOG}}.core.dim_account  a ON a.account_id  = s.account_id
+JOIN {{CATALOG}}.core.dim_customer c ON c.customer_id = a.customer_id;
+
+ALTER TABLE {{CATALOG}}.core.fct_transactions ALTER COLUMN fee_revenue
+  COMMENT 'GROSS fee revenue in the transaction currency, before reversals and chargebacks. Do NOT sum this column alone and call it revenue - subtract fct_reversals.reversal_amount. Zero for non-POSTED rows.';
+ALTER TABLE {{CATALOG}}.core.fct_transactions ALTER COLUMN status
+  COMMENT 'Transaction status. Values: POSTED, PENDING, DECLINED, REVERSED. ONLY POSTED counts as revenue. DECLINED rows are ~6% of the table and will inflate transaction counts if not filtered out.';
+ALTER TABLE {{CATALOG}}.core.fct_transactions ALTER COLUMN currency
+  COMMENT 'Settlement currency. Values: USD, CAD, GBP. COMMERCIAL customers transact in all three. Convert with dim_fx_rate joined on currency AND txn_date before summing across currencies.';
+ALTER TABLE {{CATALOG}}.core.fct_transactions ALTER COLUMN amount
+  COMMENT 'Transaction amount in the settlement currency (see currency column), not USD.';
+ALTER TABLE {{CATALOG}}.core.fct_transactions ALTER COLUMN merchant_category
+  COMMENT 'Merchant category. 12 distinct values - a good candidate for entity matching.';
+
+
+-- ============================================================================
+-- fct_reversals  —  the other half of FLAW #1
+-- Deliberately a SEPARATE table. If reversals were a column on
+-- fct_transactions nobody would ever get gross vs net wrong.
+-- ============================================================================
+CREATE OR REPLACE TABLE {{CATALOG}}.core.fct_reversals
+COMMENT 'Reversals and chargebacks. The amount here must be SUBTRACTED from fct_transactions.fee_revenue to get net fee revenue. One row per reversed transaction.'
+AS
+SELECT
+  t.txn_id,
+  date_add(t.txn_date, 3 + pmod(hash(concat('rev-lag-', t.txn_id)), 25)) AS reversal_date,
+  cast(round(t.fee_revenue, 2) AS DECIMAL(14,2))                         AS reversal_amount,
+  element_at(array('DISPUTE','FRAUD','DUPLICATE','MERCHANT_ERROR','AUTHORISATION_FAIL'),
+             pmod(hash(concat('rev-reason-', t.txn_id)), 5) + 1)         AS reason_code
+FROM {{CATALOG}}.core.fct_transactions t
+-- ~3.5% of POSTED transactions get reversed, which is why "revenue" is
+-- overstated by a few percent and nobody notices.
+WHERE t.status = 'POSTED'
+  AND pmod(hash(concat('rev-flag-', t.txn_id)), 1000) < 35;
+
+ALTER TABLE {{CATALOG}}.core.fct_reversals ALTER COLUMN reversal_amount
+  COMMENT 'Fee revenue clawed back. Subtract the sum of this from gross fee_revenue to get NET fee revenue.';
+ALTER TABLE {{CATALOG}}.core.fct_reversals ALTER COLUMN reason_code
+  COMMENT 'Reversal reason. Values: DISPUTE, FRAUD, DUPLICATE, MERCHANT_ERROR, AUTHORISATION_FAIL.';
+
+
+-- ============================================================================
+-- fct_loan_balances  —  FLAWS #6 and #7
+--
+-- ONE ROW PER ACCOUNT PER DAY. This is the single most dangerous table in the
+-- dataset: SUM(principal_balance) over any period longer than a day produces a
+-- number that looks entirely plausible and is roughly 30x too large.
+--
+-- 45,000 lending accounts x 730 days ~ 32.8M rows (~1.4M per month).
+-- ============================================================================
+CREATE OR REPLACE TABLE {{CATALOG}}.core.fct_loan_balances
+COMMENT 'DAILY SNAPSHOT of the loan book: one row per account per day. NEVER SUM principal_balance across dates - use the latest snapshot for a point-in-time balance, or an average for a period. Also carries the four distinct delinquency concepts.'
+AS
+WITH loan_accounts AS (
+  SELECT a.account_id, a.opened_date
+  FROM {{CATALOG}}.core.dim_account a
+  JOIN {{CATALOG}}.core.dim_product p ON p.product_id = a.product_id
+  WHERE p.product_category = 'LENDING'
+    AND pmod(hash(concat('loan-pick-', a.account_id)), 100) < 6   -- ~6% sample
+),
+days AS (
+  SELECT explode(sequence(DATE'2024-10-01', DATE'2026-09-30', INTERVAL 1 DAY)) AS snapshot_date
+),
+grid AS (
+  SELECT
+    la.account_id,
+    d.snapshot_date,
+    datediff(d.snapshot_date, DATE'2024-10-01')                        AS day_n,
+    pmod(hash(concat('loan-orig-', la.account_id)), 400)               AS orig_pick,
+    pmod(hash(concat('loan-risk-', la.account_id)), 1000)              AS risk_pick
+  FROM loan_accounts la CROSS JOIN days d
+  WHERE d.snapshot_date >= la.opened_date
+),
+shaped AS (
+  SELECT
+    account_id,
+    snapshot_date,
+    -- Original principal, amortising slowly over the window
+    cast(round((15000 + orig_pick * 850) * (1 - day_n / 4000.0), 2) AS DECIMAL(14,2)) AS principal_balance,
+    -- Riskier accounts accumulate days past due over time
+    CASE
+      WHEN risk_pick < 880 THEN 0
+      WHEN risk_pick < 940 THEN pmod(day_n, 45)
+      WHEN risk_pick < 980 THEN 30 + pmod(day_n, 60)
+      ELSE                      90 + pmod(day_n, 120)
+    END AS days_past_due
+  FROM grid
+)
+SELECT
+  account_id,
+  snapshot_date,
+  principal_balance,
+  cast(round(principal_balance * 0.0625 / 365 * 30, 2) AS DECIMAL(14,2)) AS interest_accrued,
+  days_past_due,
+  -- FLAW #7, part 1: the bucket
+  CASE WHEN days_past_due = 0  THEN 'CURRENT'
+       WHEN days_past_due < 30 THEN '1-29'
+       WHEN days_past_due < 60 THEN '30-59'
+       WHEN days_past_due < 90 THEN '60-89'
+       ELSE                        '90+'
+  END AS dpd_bucket,
+  -- FLAW #7, part 2: a SEPARATE lifecycle status. "Default" and "charge-off"
+  -- are accounting events, not DPD thresholds - so "how many delinquent
+  -- loans?" has four defensible answers depending on which you mean.
+  CASE WHEN days_past_due = 0    THEN 'PERFORMING'
+       WHEN days_past_due < 90   THEN 'DELINQUENT'
+       WHEN days_past_due < 180  THEN 'DEFAULT'
+       ELSE                           'CHARGED_OFF'
+  END AS loan_status
+FROM shaped;
+
+ALTER TABLE {{CATALOG}}.core.fct_loan_balances ALTER COLUMN principal_balance
+  COMMENT 'Outstanding principal AS OF snapshot_date. This table has ONE ROW PER ACCOUNT PER DAY - never SUM across dates. For a point-in-time loan book, filter to the latest snapshot_date.';
+ALTER TABLE {{CATALOG}}.core.fct_loan_balances ALTER COLUMN days_past_due
+  COMMENT 'Days past due as of snapshot_date. 0 means current.';
+ALTER TABLE {{CATALOG}}.core.fct_loan_balances ALTER COLUMN dpd_bucket
+  COMMENT 'Delinquency bucket. Values: CURRENT, 1-29, 30-59, 60-89, 90+. Meridian defines "delinquent" as 30+ days past due and "seriously delinquent" as 90+. Ask which one the user means.';
+ALTER TABLE {{CATALOG}}.core.fct_loan_balances ALTER COLUMN loan_status
+  COMMENT 'Accounting lifecycle status, DISTINCT from dpd_bucket. Values: PERFORMING, DELINQUENT, DEFAULT, CHARGED_OFF. "Default" and "charge-off" are accounting events and are NOT the same as being 30+ or 90+ days past due.';
+
+
+-- ============================================================================
+-- fct_applications  —  approval funnel and cycle time
+-- ============================================================================
+CREATE OR REPLACE TABLE {{CATALOG}}.core.fct_applications
+COMMENT 'Credit application funnel: submitted -> decisioned -> funded. Approval rate is decisions of APPROVED over all applications. Not every approved application is funded.'
+AS
+WITH ap AS (SELECT id AS n FROM range(1, 420001)),
+calc AS (
+  SELECT
+    n,
+    pmod(hash(concat('app-cust-', n)), 2100000) AS cust_n,
+    pmod(hash(concat('app-prod-', n)), 20)      AS prod_n,
+    pmod(hash(concat('app-day-',  n)), 730)     AS day_offset,
+    pmod(hash(concat('app-dec-',  n)), 100)     AS dec_pick,
+    pmod(hash(concat('app-chan-', n)), 100)     AS chan_pick,
+    pmod(hash(concat('app-lag-',  n)), 14)      AS decision_lag,
+    pmod(hash(concat('app-fund-', n)), 100)     AS fund_pick
+  FROM ap
+)
+SELECT
+  concat('APP', lpad(cast(n AS STRING), 8, '0'))            AS app_id,
+  concat('C',   lpad(cast(cust_n + 1 AS STRING), 8, '0'))   AS customer_id,
+  concat('P',   lpad(cast(prod_n + 1 AS STRING), 2, '0'))   AS product_id,
+  cast(date_add(DATE'2024-10-01', day_offset) AS TIMESTAMP) AS submitted_ts,
+  cast(date_add(DATE'2024-10-01', day_offset + decision_lag) AS TIMESTAMP) AS decision_ts,
+  CASE WHEN dec_pick < 58 AND fund_pick < 82
+       THEN cast(date_add(DATE'2024-10-01', day_offset + decision_lag + 2) AS TIMESTAMP)
+       ELSE NULL END                                        AS funded_ts,
+  CASE WHEN dec_pick < 58 THEN 'APPROVED'
+       WHEN dec_pick < 88 THEN 'DECLINED'
+       ELSE                    'WITHDRAWN' END              AS decision,
+  CASE WHEN chan_pick < 46 THEN 'DIGITAL'
+       WHEN chan_pick < 78 THEN 'BRANCH'
+       WHEN chan_pick < 92 THEN 'CALL_CENTRE'
+       ELSE                     'BROKER' END                AS channel
+FROM calc;
+
+ALTER TABLE {{CATALOG}}.core.fct_applications ALTER COLUMN decision
+  COMMENT 'Underwriting decision. Values: APPROVED, DECLINED, WITHDRAWN. Approval rate = APPROVED / all applications, including WITHDRAWN.';
+ALTER TABLE {{CATALOG}}.core.fct_applications ALTER COLUMN funded_ts
+  COMMENT 'When the loan was actually funded. NULL if never funded - roughly 18% of APPROVED applications are never drawn down, so funded count < approved count.';
+ALTER TABLE {{CATALOG}}.core.fct_applications ALTER COLUMN channel
+  COMMENT 'Origination channel. Values: DIGITAL, BRANCH, CALL_CENTRE, BROKER.';
+
+
+-- ============================================================================
+-- fct_fraud_cases
+-- ============================================================================
+CREATE OR REPLACE TABLE {{CATALOG}}.core.fct_fraud_cases
+COMMENT 'Fraud and financial-crime cases. loss_amount is the realised loss in USD, zero for cases closed without loss.'
+AS
+WITH f AS (SELECT id AS n FROM range(1, 14001)),
+calc AS (
+  SELECT
+    n,
+    pmod(hash(concat('frd-acct-', n)), 2900000) AS acct_n,
+    pmod(hash(concat('frd-day-',  n)), 730)     AS day_offset,
+    pmod(hash(concat('frd-type-', n)), 6)       AS type_pick,
+    pmod(hash(concat('frd-loss-', n)), 100)     AS loss_pick,
+    pmod(hash(concat('frd-stat-', n)), 100)     AS stat_pick,
+    pmod(hash(concat('frd-close-',n)), 60)      AS close_lag
+  FROM f
+)
+SELECT
+  concat('FC', lpad(cast(n AS STRING), 7, '0'))            AS case_id,
+  concat('A',  lpad(cast(acct_n + 1 AS STRING), 9, '0'))   AS account_id,
+  date_add(DATE'2024-10-01', day_offset)                   AS opened_date,
+  CASE WHEN stat_pick < 84
+       THEN date_add(DATE'2024-10-01', day_offset + close_lag)
+       ELSE NULL END                                       AS closed_date,
+  cast(CASE WHEN loss_pick < 38 THEN 0
+            ELSE round(150 + loss_pick * 96.5, 2) END AS DECIMAL(14,2)) AS loss_amount,
+  element_at(array('CARD_NOT_PRESENT','ACCOUNT_TAKEOVER','APPLICATION_FRAUD',
+                   'CHECK_FRAUD','WIRE_FRAUD','FIRST_PARTY'), type_pick + 1)  AS fraud_type,
+  CASE WHEN stat_pick < 84 THEN 'CLOSED' ELSE 'OPEN' END   AS status
+FROM calc;
+
+ALTER TABLE {{CATALOG}}.core.fct_fraud_cases ALTER COLUMN fraud_type
+  COMMENT 'Fraud typology. Values: CARD_NOT_PRESENT, ACCOUNT_TAKEOVER, APPLICATION_FRAUD, CHECK_FRAUD, WIRE_FRAUD, FIRST_PARTY.';
+ALTER TABLE {{CATALOG}}.core.fct_fraud_cases ALTER COLUMN loss_amount
+  COMMENT 'Realised loss in USD. Zero for cases closed with no loss - about 38% of cases.';
+
+
+-- ----------------------------------------------------------------------------
+-- Row counts
+-- ----------------------------------------------------------------------------
+SELECT 'fct_transactions'  AS table_name, count(*) AS rows FROM {{CATALOG}}.core.fct_transactions
+UNION ALL SELECT 'fct_reversals',      count(*) FROM {{CATALOG}}.core.fct_reversals
+UNION ALL SELECT 'fct_loan_balances',  count(*) FROM {{CATALOG}}.core.fct_loan_balances
+UNION ALL SELECT 'fct_applications',   count(*) FROM {{CATALOG}}.core.fct_applications
+UNION ALL SELECT 'fct_fraud_cases',    count(*) FROM {{CATALOG}}.core.fct_fraud_cases
+ORDER BY table_name;

databricks360/courses/genie_agents/manifest.json

@@ -0,0 +1,41 @@
+{
+  "id": "genie-agents",
+  "title": "Genie Agents — Meridian Financial Group lab",
+  "description": "A synthetic US bank with nine deliberate data-quality flaws. Every flaw is the raw material for a specific module: gross vs net revenue, a fiscal year starting 1 October, region codes users never say out loud, DECLINED rows inflating counts, two competing product hierarchies, a daily-snapshot table that invites a 30x fan-out, four meanings of 'delinquent', unmasked PII, and multi-currency needing an as-of-date join.",
+  "default_catalog": "mfg",
+  "default_tier": "small",
+  "tiers": {
+    "small": {
+      "description": "20M transactions. Every module except 13. Start here.",
+      "values": { "TXN_COUNT": "20000000" }
+    },
+    "large": {
+      "description": "900M transactions, left unclustered on purpose. Module 13 (latency) only.",
+      "values": { "TXN_COUNT": "900000000" }
+    }
+  },
+  "notebooks": [
+    {
+      "order": 1,
+      "name": "01_catalog_and_schemas",
+      "sql": "01_catalog_and_schemas.sql",
+      "title": "01 · Catalog and schemas",
+      "intro": "Creates the `{{CATALOG}}` catalog, the core / ref / staging schemas, and the documents volume.\n\nRun this first. Everything else depends on it."
+    },
+    {
+      "order": 2,
+      "name": "02_dimensions",
+      "sql": "02_dimensions.sql",
+      "title": "02 · Dimensions",
+      "intro": "Six dimensions, planting five of the nine flaws: the 1 October fiscal year, region and state stored as codes, two competing product hierarchies, unmasked PII, and multi-currency FX rates.\n\nEvery value derives from `hash()` of the row key rather than `rand()`, so your data is byte-identical to everyone else's and the benchmark answers in Module 11 stay valid."
+    },
+    {
+      "order": 3,
+      "name": "03_facts",
+      "sql": "03_facts.sql",
+      "title": "03 · Facts",
+      "intro": "Five fact tables carrying the four remaining flaws.\n\n**This is the slow one** — it generates {{TXN_COUNT}} transactions plus roughly 35M daily loan-balance snapshots. Expect several minutes on a small warehouse.",
+      "slow": true
+    }
+  ]
+}

databricks360-0.1.0.dist-info/METADATA

@@ -0,0 +1,176 @@
+Metadata-Version: 2.4
+Name: databricks360
+Version: 0.1.0
+Summary: Install Databricks course lab environments: notebooks, catalogs, datasets and governance objects.
+Project-URL: Homepage, https://github.com/databrickslms/dbxdemos
+Project-URL: Source, https://github.com/databrickslms/dbxdemos
+Project-URL: Issues, https://github.com/databrickslms/dbxdemos/issues
+Author: Lakehouse Academy
+License: MIT
+License-File: LICENSE
+Keywords: databricks,genie,lakehouse,training,unity-catalog
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Education
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Education
+Requires-Python: >=3.9
+Requires-Dist: databricks-sdk>=0.38.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# databricks360
+
+Installs Databricks course lab environments — notebooks, catalogs, datasets and
+governance objects — into your own workspace.
+
+Modelled on `dbdemos`: you run it **inside a Databricks notebook**, so
+`databricks-sdk` picks up the notebook's own identity. There is no host, token or
+profile to configure.
+
+## Install
+
+Not on PyPI yet, so install from the repo. In a **Databricks notebook**:
+
+```python
+%pip install git+https://github.com/databrickslms/dbxdemos.git
+dbutils.library.restartPython()
+```
+
+Once published, that becomes `%pip install databricks360`.
+
+To pin a version, append a tag or commit:
+
+```python
+%pip install git+https://github.com/databrickslms/dbxdemos.git@v0.1.0
+```
+
+## Usage
+
+```python
+import databricks360 as academy
+
+academy.list_courses()
+academy.install('genie-agents')
+```
+
+`install` writes the lab notebooks into your workspace and prints the run order.
+Then you open them and run each in turn.
+
+```
+Installed 'genie-agents' → /Workspace/Users/you@corp.com/databricks360/genie-agents
+  catalog: mfg    tier: small
+
+  Run these in order:
+    1. 01_catalog_and_schemas
+    2. 02_dimensions
+    3. 03_facts   (slow)
+```
+
+### Options
+
+```python
+academy.install(
+    'genie-agents',
+    path='/Workspace/Shared/labs',   # default: your home folder
+    catalog='training_v2',           # default: the course's own catalog
+    tier='large',                    # default: 'small'
+    overwrite=True,                  # replace existing notebooks
+)
+```
+
+## Why it does not run the notebooks for you
+
+`dbdemos` starts a job and loads the data on your behalf. This deliberately does
+not. Generating the data is the substance of Module 0 — the point is to watch a
+warehouse chew through 20M rows and see the flaws appear, not to have a finished
+catalog materialise. It also means nothing consumes your DBUs without you asking.
+
+## Tiers
+
+| Tier | Transactions | Use |
+|---|---|---|
+| `small` | 20M | default. Every module except 13 |
+| `large` | 900M | Module 13 (latency) only. Left unclustered on purpose |
+
+Start small. The large tier exists because you cannot measure query latency on a
+toy dataset, and nowhere else needs it.
+
+## Adding a course
+
+Each course is a subpackage under `databricks360/courses/`:
+
+```
+databricks360/courses/<course_id>/
+  __init__.py
+  manifest.json      # title, default catalog, tiers, notebooks in run order
+  *.sql              # synced from content/courses/<id>/assets/lab/
+```
+
+Placeholders available in the SQL: `{{CATALOG}}`, plus anything declared under a
+tier's `values` (currently `{{TXN_COUNT}}`). An unresolved placeholder raises rather
+than silently rendering empty.
+
+Dataset documentation lives in [`docs/`](docs/).
+
+## Publishing
+
+Releases go to PyPI via **Trusted Publishing** — GitHub Actions authenticates to
+PyPI with a short-lived OIDC identity, so no API token exists in repo secrets or on
+anyone's laptop. A leaked token is the usual way a package supply chain gets
+compromised; the safest token is one that was never created.
+
+### One-time PyPI setup
+
+1. Sign in at [pypi.org](https://pypi.org) → **Your account → Publishing**
+2. Under *Add a new pending publisher*, choose **GitHub** and enter exactly:
+
+   | Field | Value |
+   |---|---|
+   | PyPI Project Name | `databricks360` |
+   | Owner | `databrickslms` |
+   | Repository name | `dbxdemos` |
+   | Workflow name | `publish.yml` |
+   | Environment name | `pypi` |
+
+3. In GitHub → **Settings → Environments → New environment** → name it `pypi`.
+   Add yourself as a required reviewer if you want to approve each release.
+
+"Pending" publisher is correct — the project does not exist on PyPI yet, and the
+first successful run creates it.
+
+### Cutting a release
+
+```bash
+# bump version in pyproject.toml, commit, then:
+git tag v0.1.0
+git push origin v0.1.0
+```
+
+The workflow runs the tests, builds, checks the tag matches `pyproject.toml`, and
+publishes. A mismatched tag fails before anything reaches the index — versions on
+PyPI are immutable, so a wrong number cannot be taken back, only yanked.
+
+### Publishing by hand instead
+
+```bash
+python -m pip install build twine
+python -m build
+twine check dist/*
+twine upload dist/*          # prompts for an API token
+```
+
+Test it against TestPyPI first if you want a dry run:
+`twine upload --repository testpypi dist/*`.
+
+## Tests
+
+```bash
+python3 run_tests.py
+```
+
+Eleven tests, no workspace required: manifest loading, notebook cell structure,
+catalog substitution, tier switching, unresolved-placeholder detection, and a
+`dry_run` install. It also asserts the flaw-teaching column comments survive into
+the generated notebooks — those comments are the curriculum, so losing them in
+rendering would be a silent failure.

databricks360-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+databricks360/__init__.py,sha256=wpwQrBwBMNxoM7msAY5NyCXnDW_9P862E6tzkGVeBLw,2028
+databricks360/_catalog.py,sha256=xBnIho0B8MZfAn-f5ggFnESIiucMOz6U6QNbpBSnfDA,2703
+databricks360/_install.py,sha256=dYKW5JMlwTAX3PQMYyFmeefxVszrRUg0yN3hTndxX3s,5019
+databricks360/_notebook.py,sha256=Y9pezGq_bI4ns1ceqChwY52gvBZU0VmEURYN84dzfN4,3653
+databricks360/courses/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databricks360/courses/genie_agents/01_catalog_and_schemas.sql,sha256=qLsiR0HmbCNM-jw6e0GRZkl6xmDMiwuBVQA1q8Iv4Sc,1778
+databricks360/courses/genie_agents/02_dimensions.sql,sha256=VsWXSA4QQqW9DUd1au3JyWpZZ_GR8HsEo3-Pa2fStO8,16186
+databricks360/courses/genie_agents/03_facts.sql,sha256=sryXtQDW-ztdDSUZCxwWKJLy6cSW1iRqPJWHjjQmyW0,15877
+databricks360/courses/genie_agents/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databricks360/courses/genie_agents/manifest.json,sha256=CSta9aQ9ObfuMC2PyjEnk36DDk_m-fnv4FvbAQYaBeE,2129
+databricks360-0.1.0.dist-info/METADATA,sha256=iJjLnmmUdHh-yOlJfmSUBKkJQ5Wd0ao53Lv2OaDaVLI,5549
+databricks360-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+databricks360-0.1.0.dist-info/licenses/LICENSE,sha256=NVi-ya6tGvg2-QS_3ThB4CzlyPDnWdGSsE09Ru_zn5k,1074
+databricks360-0.1.0.dist-info/RECORD,,

databricks360-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

databricks360-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Lakehouse Academy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.