datadoom 0.1.0.dev0__py3-none-any.whl

datadoom/templates/credit_default_challenge.datadoom.yaml

@@ -0,0 +1,147 @@
+datadoom_version: "1"
+name: "credit-default-challenge"
+description: >
+  HACKATHON FLAGSHIP — consumer credit default. A multi-hop causal economy:
+  applicant demographics and employment drive annual income, income + debt load +
+  repayment history feed a LATENT risk_score (emit: false — the true generative
+  driver, never shipped), and the risk_score drives the `defaulted` label. The
+  clean dataset is then calibrated down to the 'advanced' baseline-AUROC band
+  [0.72, 0.80] so a vanilla model is honestly hard. On top, a realistic
+  data-quality stack corrupts a copy: income is MNAR (high earners under-report),
+  debt-to-income is MAR (missing for older applicants) and DRIFTS upward over time
+  (a simulated downturn), a `collections_flag` is a LEAKED near-proxy of the label
+  (students must detect and drop it), and 2% of labels are flipped. Train/test
+  split included.
+seed: 101
+rows: 9000
+
+features:
+  applicant_age:
+    type: numeric
+    dist: normal
+    params: { mean: 41, std: 13 }
+    min: 18
+    max: 90
+    dtype: int
+  employment_type:
+    type: categorical
+    categories: [unemployed, part_time, full_time, self_employed]
+    weights: [0.08, 0.17, 0.6, 0.15]
+  education:
+    type: categorical
+    categories: [hs, college, grad]
+    weights: [0.45, 0.42, 0.13]
+  region:
+    type: categorical
+    categories: [northeast, midwest, south, west]
+    weights: [0.22, 0.24, 0.32, 0.22]
+  requested_amount:
+    type: numeric
+    dist: lognormal
+    params: { mu: 9.1, sigma: 0.55 }   # median ≈ $8,950
+    min: 500
+  credit_history_months:
+    type: numeric
+    dist: normal
+    params: { mean: 96, std: 48 }
+    min: 0
+    dtype: int
+  num_open_accounts:
+    type: numeric
+    dist: poisson
+    params: { lam: 5 }
+    min: 0
+    dtype: int
+  num_late_payments:
+    type: numeric
+    dist: poisson
+    params: { lam: 1.1 }
+    min: 0
+    dtype: int
+  debt_to_income:
+    type: numeric
+    dist: normal
+    params: { mean: 0.34, std: 0.14 }
+    min: 0
+  annual_income:
+    type: numeric          # derived (hop 1): demographics + employment → income
+    dtype: float
+    min: 0
+  risk_score:
+    type: numeric          # derived LATENT (hop 2): the true default driver
+    dtype: float
+    emit: false            # hidden — shapes the label, shows in the true graph, not shipped
+  defaulted:
+    type: boolean          # label (hop 3): logistic of the latent risk_score
+
+causal:
+  edges:
+    # hop 1 — what someone earns
+    - { from: applicant_age,   to: annual_income, fn: linear, weight: 620, bias: 15000 }
+    - { from: education,       to: annual_income, fn: map,    mapping: { hs: 0, college: 18000, grad: 41000 } }
+    - { from: employment_type, to: annual_income, fn: map,    mapping: { unemployed: -16000, part_time: -4000, full_time: 12000, self_employed: 6000 } }
+    # hop 2 — latent creditworthiness
+    - { from: annual_income,        to: risk_score, fn: linear, weight: -0.000028 }
+    - { from: debt_to_income,       to: risk_score, fn: linear, weight: 4.2 }
+    - { from: num_late_payments,    to: risk_score, fn: linear, weight: 0.55 }
+    - { from: credit_history_months, to: risk_score, fn: linear, weight: -0.006 }
+    - { from: num_open_accounts,    to: risk_score, fn: linear, weight: 0.06 }
+    # hop 3 — the label
+    - { from: risk_score, to: defaulted, fn: logistic, weight: 1.6, bias: -1.2 }
+  noise:
+    annual_income: { dist: normal, params: { mean: 0, std: 6500 } }
+    risk_score: { dist: none }
+    defaulted: { dist: none }
+
+difficulty:
+  target: advanced          # baseline ROC-AUC calibrated into [0.72, 0.80]
+  label: defaulted
+  probe: logreg
+  max_iters: 10
+  knobs: [noise, label_noise]
+
+failures:
+  - type: mnar               # high earners under-report income
+    column: annual_income
+    rate: 0.10
+    strength: 2.2
+  - type: mar                # DTI missing for older applicants
+    column: debt_to_income
+    driver: applicant_age
+    rate: 0.08
+    strength: 2.0
+  - type: drift              # a simulated downturn pushes DTI up over time
+    column: debt_to_income
+    schedule: { kind: linear, magnitude: 0.15 }
+  - type: leakage            # collections_flag ≈ the label — a trap to detect and drop
+    target: defaulted
+    into: collections_flag
+    noise: 0.05
+  - type: label_noise        # 2% mislabelled defaults
+    column: defaulted
+    rate: 0.02
+
+export:
+  formats: [csv]
+  versions: [clean, injected]
+  splits: { train: 0.8, test: 0.2 }
+
+meta:
+  level: hackathon
+  challenge:
+    title: "Predict consumer credit default"
+    task: classification
+    target: defaulted
+    metric: ROC-AUC
+    difficulty: advanced
+    baseline_auroc_band: [0.72, 0.80]
+    train_test_split: "80 / 20"
+    hidden_structure: >
+      risk_score is latent (never shipped). The signal flows
+      demographics+employment → annual_income → risk_score → defaulted.
+    gotchas:
+      - "annual_income is MNAR — missingness depends on the (high) value itself."
+      - "debt_to_income is MAR (missing for older applicants) and drifts upward over the row order."
+      - "collections_flag is a leaked proxy of the label — drop it or you'll 'win' dishonestly."
+      - "2% of labels are flipped; perfect accuracy is not the goal."
+  tags: [finance, credit-risk, causal, latent, difficulty, failure-injection, leakage, classification]

datadoom/templates/customer_churn.datadoom.yaml

@@ -0,0 +1,60 @@
+datadoom_version: "1"
+name: "customer-churn"
+description: >
+  Customer-churn starter. Tenure, monthly charges and support load feed a latent
+  satisfaction score (emit: false — it shapes the label and shows up in the true
+  causal graph, but is never shipped) behind a churn label. The dataset is
+  calibrated down to the 'intermediate' baseline-AUROC band, so a baseline model
+  lands in [0.62, 0.72] — hard in a measured, honest way.
+seed: 23
+rows: 7000
+
+features:
+  tenure_months:
+    type: numeric
+    dist: normal
+    params: { mean: 28, std: 16 }
+    min: 0
+    dtype: int
+  monthly_charges:
+    type: numeric
+    dist: normal
+    params: { mean: 70, std: 25 }
+    min: 0
+  support_tickets:
+    type: numeric
+    dist: poisson
+    params: { lam: 2.5 }
+    min: 0
+    dtype: int
+  satisfaction:
+    type: numeric          # latent: weighted combination of the drivers
+    dtype: float
+    emit: false            # hidden — drives the label but is not shipped
+  churned:
+    type: boolean          # label (derived): logistic of (low) satisfaction
+
+causal:
+  edges:
+    - { from: tenure_months,  to: satisfaction, fn: linear, weight: 0.04 }
+    - { from: monthly_charges, to: satisfaction, fn: linear, weight: -0.03 }
+    - { from: support_tickets, to: satisfaction, fn: linear, weight: -0.4 }
+    - { from: satisfaction,   to: churned, fn: logistic, weight: -2.5, bias: 0.2 }
+  noise:
+    satisfaction: { dist: none }
+    churned: { dist: none }
+
+difficulty:
+  target: intermediate      # beginner | intermediate | advanced | kaggle
+  label: churned
+  probe: logreg
+  max_iters: 10
+  knobs: [noise, label_noise]
+
+export:
+  formats: [csv]
+  versions: [clean]
+
+meta:
+  problem_statement: "Predict churned from tenure, charges, support load (satisfaction is latent)."
+  tags: [churn, difficulty, latent, causal, classification]

datadoom/templates/ecommerce_orders.datadoom.yaml

@@ -0,0 +1,46 @@
+datadoom_version: "1"
+name: "ecommerce-orders"
+description: >
+  E-commerce orders starter — a fast, distribution-only table (no causal graph):
+  order value (heavy-ish lognormal), basket quantity, product category, channel,
+  order date and a return flag. Great for quickly populating a realistic-looking
+  orders dataset.
+seed: 31
+rows: 10000
+
+features:
+  order_value:
+    type: numeric
+    dist: lognormal
+    params: { mu: 3.8, sigma: 0.6 }
+    min: 0
+  quantity:
+    type: numeric
+    dist: poisson
+    params: { lam: 2.2 }
+    min: 1
+    dtype: int
+  category:
+    type: categorical
+    categories: [electronics, apparel, home, books, toys, grocery]
+    weights: [0.22, 0.26, 0.18, 0.1, 0.1, 0.14]
+  channel:
+    type: categorical
+    categories: [web, mobile, store]
+    weights: [0.45, 0.4, 0.15]
+  order_date:
+    type: datetime
+    start: "2023-01-01"
+    end: "2024-12-31"
+    granularity: day
+  returned:
+    type: boolean
+    rate: 0.08
+
+export:
+  formats: [csv]
+  versions: [clean]
+
+meta:
+  problem_statement: "A realistic orders table for quick prototyping (distribution-only)."
+  tags: [ecommerce, tabular, datetime]

datadoom/templates/fraud_detection.datadoom.yaml

@@ -0,0 +1,57 @@
+datadoom_version: "1"
+name: "transaction-fraud"
+description: >
+  Transaction fraud starter. Customer age and card type drive monthly spend, which
+  drives a fraud-risk label; a stack of realistic failures (under-reported spend,
+  random gaps in age, mislabelled cases) then corrupts a copy — the clean baseline
+  is always kept, so you can compare clean vs. injected in Results.
+seed: 7
+rows: 8000
+
+features:
+  customer_age:
+    type: numeric
+    dist: normal
+    params: { mean: 42, std: 13 }
+    min: 18
+    max: 95
+    dtype: int
+  card_type:
+    type: categorical
+    categories: [standard, gold, platinum]
+    weights: [0.6, 0.3, 0.1]
+  monthly_spend:
+    type: numeric          # derived
+    dtype: float
+    min: 0
+  is_fraud:
+    type: boolean          # derived (target)
+
+causal:
+  edges:
+    - { from: customer_age, to: monthly_spend, fn: linear, weight: 35, bias: 1200 }
+    - { from: card_type,    to: monthly_spend, fn: map, mapping: { standard: 0, gold: 900, platinum: 2600 } }
+    - { from: monthly_spend, to: is_fraud, fn: logistic, weight: -0.0008, bias: 1.2 }
+  noise:
+    monthly_spend: { dist: normal, params: { mean: 0, std: 300 } }
+    is_fraud: { dist: none }
+
+failures:
+  - type: mnar               # big spenders under-report
+    column: monthly_spend
+    rate: 0.10
+    strength: 2.0
+  - type: mcar               # random gaps in age
+    columns: [customer_age]
+    rate: 0.05
+  - type: label_noise        # mislabelled fraud cases
+    column: is_fraud
+    rate: 0.03
+
+export:
+  formats: [csv]
+  versions: [clean, injected]
+
+meta:
+  problem_statement: "Predict is_fraud from customer_age, card_type, monthly_spend under realistic failures."
+  tags: [fraud, causal, failure-injection, classification]

datadoom/templates/hospital_readmission.datadoom.yaml

@@ -0,0 +1,61 @@
+datadoom_version: "1"
+name: "hospital-readmission"
+description: >
+  Hospital 30-day readmission starter. Patient age, diagnosis count, length of stay
+  and prior admissions drive a latent severity score (emit: false) behind a
+  readmission label. A clean causal dataset with a hidden confounder documented in
+  the true graph — a good base to add failures or a difficulty target onto.
+seed: 11
+rows: 6000
+
+features:
+  patient_age:
+    type: numeric
+    dist: normal
+    params: { mean: 58, std: 17 }
+    min: 18
+    max: 100
+    dtype: int
+  num_diagnoses:
+    type: numeric
+    dist: poisson
+    params: { lam: 4 }
+    min: 0
+    dtype: int
+  length_of_stay:
+    type: numeric
+    dist: normal
+    params: { mean: 5, std: 3 }
+    min: 1
+    dtype: int
+  prior_admissions:
+    type: numeric
+    dist: poisson
+    params: { lam: 1.2 }
+    min: 0
+    dtype: int
+  severity:
+    type: numeric          # latent: weighted combination of the drivers
+    dtype: float
+    emit: false            # hidden confounder — not shipped
+  readmitted:
+    type: boolean          # label (derived): logistic of severity
+
+causal:
+  edges:
+    - { from: patient_age,      to: severity, fn: linear, weight: 0.02 }
+    - { from: num_diagnoses,    to: severity, fn: linear, weight: 0.25 }
+    - { from: length_of_stay,   to: severity, fn: linear, weight: 0.18 }
+    - { from: prior_admissions, to: severity, fn: linear, weight: 0.6 }
+    - { from: severity,         to: readmitted, fn: logistic, weight: 1.8, bias: -2.0 }
+  noise:
+    severity: { dist: none }
+    readmitted: { dist: none }
+
+export:
+  formats: [csv]
+  versions: [clean]
+
+meta:
+  problem_statement: "Predict readmitted from age, diagnoses, length of stay, prior admissions (severity is latent)."
+  tags: [healthcare, causal, latent, classification]

datadoom/templates/insurance_claims.datadoom.yaml

@@ -0,0 +1,43 @@
+datadoom_version: "1"
+name: "insurance-claims"
+description: >
+  Insurance claims starter showcasing a heavy-tailed claim amount (Pareto) — most
+  claims are small, a few are very large. Policyholder age, region, prior-claim
+  count and a fraud flag round it out. Distribution-only, generates instantly.
+seed: 13
+rows: 8000
+
+features:
+  policyholder_age:
+    type: numeric
+    dist: normal
+    params: { mean: 47, std: 15 }
+    min: 18
+    max: 90
+    dtype: int
+  region:
+    type: categorical
+    categories: [north, south, east, west]
+    weights: [0.25, 0.3, 0.2, 0.25]
+  claim_amount:
+    type: numeric
+    dist: pareto
+    params: { alpha: 2.5, xm: 800 }
+    min: 0
+  num_prior_claims:
+    type: numeric
+    dist: poisson
+    params: { lam: 1.1 }
+    min: 0
+    dtype: int
+  is_fraudulent:
+    type: boolean
+    rate: 0.05
+
+export:
+  formats: [csv]
+  versions: [clean]
+
+meta:
+  problem_statement: "Insurance claims with a heavy-tailed (Pareto) claim amount."
+  tags: [insurance, heavy-tail, tabular]

datadoom/templates/iot_sensors.datadoom.yaml

@@ -0,0 +1,44 @@
+datadoom_version: "1"
+name: "iot-sensor-readings"
+description: >
+  IoT sensor telemetry starter — a fast, purely numeric table of environmental
+  readings (temperature, humidity, pressure, battery) tagged by device and
+  hourly timestamp. Bounded distributions keep the values physically plausible.
+seed: 19
+rows: 12000
+
+features:
+  device_id:
+    type: categorical
+    categories: [sensor-a, sensor-b, sensor-c, sensor-d]
+  temperature_c:
+    type: numeric
+    dist: normal
+    params: { mean: 21.5, std: 4.0 }
+  humidity_pct:
+    type: numeric
+    dist: normal
+    params: { mean: 45, std: 12 }
+    min: 0
+    max: 100
+  pressure_hpa:
+    type: numeric
+    dist: normal
+    params: { mean: 1013, std: 8 }
+  battery_pct:
+    type: numeric
+    dist: uniform
+    params: { low: 20, high: 100 }
+  reading_time:
+    type: datetime
+    start: "2024-01-01"
+    end: "2024-03-31"
+    granularity: hour
+
+export:
+  formats: [csv]
+  versions: [clean]
+
+meta:
+  problem_statement: "Hourly multi-sensor telemetry for quick prototyping (distribution-only)."
+  tags: [iot, numeric, datetime, timeseries-ish]

datadoom/templates/people_directory.datadoom.yaml

@@ -0,0 +1,56 @@
+datadoom_version: "1"
+name: "people-directory"
+description: >
+  People directory starter — realistic-but-deterministic identities via the
+  bundled mimesis providers (name, email, phone, company, job, city, country).
+  Same (spec_hash, seed) → byte-identical strings. Ideal for demos and UIs that
+  need believable names and contact details.
+seed: 42
+rows: 3000
+
+features:
+  full_name:
+    type: text
+    generator: name
+  email:
+    type: text
+    generator: email
+  phone:
+    type: text
+    generator: phone
+  company:
+    type: text
+    generator: company
+  job_title:
+    type: text
+    generator: occupation
+  city:
+    type: text
+    generator: city
+  country:
+    type: text
+    generator: country
+  age:
+    type: numeric
+    dist: normal
+    params: { mean: 38, std: 12 }
+    min: 18
+    max: 85
+    dtype: int
+  plan:
+    type: categorical
+    categories: [free, pro, enterprise]
+    weights: [0.7, 0.25, 0.05]
+  signup_date:
+    type: datetime
+    start: "2021-01-01"
+    end: "2024-12-31"
+    granularity: day
+
+export:
+  formats: [csv]
+  versions: [clean]
+
+meta:
+  problem_statement: "Believable people records (names, emails, companies) for demos and UIs."
+  tags: [people, realistic-text, pii-like]

datadoom/templates/predictive_maintenance.datadoom.yaml

@@ -0,0 +1,107 @@
+datadoom_version: "1"
+name: "predictive-maintenance"
+description: >
+  HACKATHON — turbine predictive maintenance on multi-sensor TIME-SERIES. Three
+  additive sensor streams (vibration, bearing temperature, oil pressure) run over
+  the row index (row order IS time): each carries trend + operational seasonality +
+  AR(1) autocorrelation + noise. Together with the operating `load_factor` and the
+  `component_grade` they drive a LATENT `wear_index` (emit: false) behind a
+  `needs_maintenance` label. A realistic operations stack then corrupts a copy:
+  the load regime DRIFTS upward over time, the load sensor gains noise and drops
+  readings (MCAR), and a `maintenance_alarm` is planted as a LEAKED proxy of the
+  label. This is a sequential problem — preserve row order; do not shuffle the
+  series across the train/test boundary.
+seed: 303
+rows: 4500
+
+features:
+  vibration:
+    type: timeseries       # mm/s RMS — slow upward trend as the unit wears
+    trend: { slope: 0.0009, intercept: 2.0 }
+    seasonality:
+      - { amplitude: 0.45, period: 24, phase: 0.0 }    # daily duty cycle
+    ar: [0.5]
+    noise_std: 0.2
+    dtype: float
+  bearing_temp:
+    type: timeseries       # °C — daily + weekly cycle on a warming trend
+    trend: { slope: 0.0018, intercept: 64.0 }
+    seasonality:
+      - { amplitude: 4.0, period: 24, phase: 0.0 }
+      - { amplitude: 1.5, period: 168, phase: 1.0 }
+    ar: [0.6]
+    noise_std: 1.0
+    dtype: float
+  oil_pressure:
+    type: timeseries       # bar — gently declining as seals degrade
+    trend: { slope: -0.0011, intercept: 45.0 }
+    ar: [0.4]
+    noise_std: 0.7
+    dtype: float
+  load_factor:
+    type: numeric          # 0..1 operating load (root; the failures target this)
+    dist: normal
+    params: { mean: 0.68, std: 0.14 }
+    min: 0
+    max: 1
+  component_grade:
+    type: categorical
+    categories: [economy, standard, premium]
+    weights: [0.4, 0.4, 0.2]
+  wear_index:
+    type: numeric          # LATENT degradation index — the true driver
+    dtype: float
+    emit: false
+  needs_maintenance:
+    type: boolean          # label (derived): logistic of wear_index
+
+causal:
+  edges:
+    - { from: vibration,       to: wear_index, fn: linear, weight: 0.9 }
+    - { from: bearing_temp,    to: wear_index, fn: linear, weight: 0.03 }
+    - { from: oil_pressure,    to: wear_index, fn: linear, weight: -0.04 }
+    - { from: load_factor,     to: wear_index, fn: linear, weight: 1.3 }
+    - { from: component_grade, to: wear_index, fn: map,    mapping: { economy: 0.8, standard: 0.0, premium: -0.6 } }
+    - { from: wear_index,      to: needs_maintenance, fn: logistic, weight: 1.1, bias: -6.8 }
+  noise:
+    wear_index: { dist: none }
+    needs_maintenance: { dist: none }
+
+failures:
+  - type: drift              # load regime creeps up over the campaign
+    column: load_factor
+    schedule: { kind: linear, magnitude: 0.18 }
+  - type: feature_noise      # sensor noise on the load reading
+    column: load_factor
+    dist: normal
+    params: { mean: 0, std: 0.04 }
+  - type: mcar               # dropped telemetry on load
+    columns: [load_factor]
+    rate: 0.05
+  - type: leakage            # an alarm that is essentially the label — drop it
+    target: needs_maintenance
+    into: maintenance_alarm
+    noise: 0.06
+
+export:
+  formats: [csv]
+  versions: [clean, injected]
+
+meta:
+  level: hackathon
+  challenge:
+    title: "Predict turbine maintenance needs from sensor streams"
+    task: classification
+    target: needs_maintenance
+    metric: ROC-AUC / PR-AUC (class-imbalanced)
+    time_axis: row_order
+    hidden_structure: >
+      wear_index is latent and accumulates from the sensor streams + load + grade.
+      The series are autocorrelated — vibration/temp trend up, oil pressure trends
+      down as the unit degrades.
+    gotchas:
+      - "Row order is the time axis — do NOT shuffle across the train/test split."
+      - "load_factor drifts and is noisy/MCAR in the injected copy — a covariate-shift between early and late rows."
+      - "maintenance_alarm is a leaked proxy of the label — drop it."
+      - "Positive class is the minority — prefer PR-AUC over plain accuracy."
+  tags: [iot, predictive-maintenance, time-series, causal, latent, drift, leakage, classification]