tech-hub-skills 1.2.0 → 1.5.2

package/.claude/roles/data-engineer/skills/01-lakehouse-architecture/bronze_ingestion.py

@@ -0,0 +1,337 @@
+"""
+Bronze Layer: Raw Data Ingestion
+Ingest data from multiple sources with validation and error handling.
+"""
+
+import json
+import os
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, Any, List, Optional, Union
+import pandas as pd
+from pyspark.sql import SparkSession, DataFrame
+from pyspark.sql.types import StructType, StructField, StringType, IntegerType, TimestampType
+from pyspark.sql import functions as F
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class BronzeLoader:
+    """
+    Bronze layer ingestion with schema validation and audit logging.
+
+    Bronze layer principles:
+    - Append-only (preserve full history)
+    - Raw data with minimal transformation
+    - Add metadata (ingestion timestamp, source, file name)
+    - Schema validation
+    - Error quarantine
+    """
+
+    def __init__(
+        self,
+        spark: Optional[SparkSession] = None,
+        bronze_path: str = "/lakehouse/bronze",
+        quarantine_path: str = "/lakehouse/quarantine"
+    ):
+        """
+        Initialize Bronze loader.
+
+        Args:
+            spark: SparkSession (creates one if not provided)
+            bronze_path: Path to bronze layer storage
+            quarantine_path: Path for invalid records
+        """
+        self.spark = spark or self._create_spark_session()
+        self.bronze_path = bronze_path
+        self.quarantine_path = quarantine_path
+
+        # Create directories if they don't exist
+        Path(bronze_path).mkdir(parents=True, exist_ok=True)
+        Path(quarantine_path).mkdir(parents=True, exist_ok=True)
+
+    def _create_spark_session(self) -> SparkSession:
+        """Create Spark session with Delta Lake support."""
+        return SparkSession.builder \
+            .appName("BronzeIngestion") \
+            .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \
+            .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \
+            .config("spark.databricks.delta.retentionDurationCheck.enabled", "false") \
+            .getOrCreate()
+
+    def ingest_from_source(
+        self,
+        source_path: str,
+        table_name: str,
+        source_format: str = "json",
+        schema: Optional[StructType] = None,
+        options: Optional[Dict[str, str]] = None
+    ) -> Dict[str, Any]:
+        """
+        Ingest data from source into Bronze layer.
+
+        Args:
+            source_path: Path to source data
+            table_name: Name for bronze table
+            source_format: Format (json, csv, parquet, etc.)
+            schema: Optional schema to enforce
+            options: Additional read options
+
+        Returns:
+            Ingestion metrics
+        """
+        logger.info(f"Starting ingestion: {table_name} from {source_path}")
+
+        try:
+            # Read source data
+            df = self._read_source(source_path, source_format, schema, options)
+
+            # Add bronze layer metadata
+            df_bronze = self._add_bronze_metadata(df, source_path, table_name)
+
+            # Validate schema if provided
+            if schema:
+                df_bronze = self._validate_schema(df_bronze, schema)
+
+            # Write to bronze layer
+            bronze_table_path = f"{self.bronze_path}/{table_name}"
+
+            df_bronze.write \
+                .format("delta") \
+                .mode("append") \
+                .option("mergeSchema", "true") \
+                .save(bronze_table_path)
+
+            # Collect metrics
+            record_count = df_bronze.count()
+
+            metrics = {
+                "status": "success",
+                "table_name": table_name,
+                "records_ingested": record_count,
+                "source_path": source_path,
+                "ingestion_timestamp": datetime.now().isoformat(),
+                "bronze_path": bronze_table_path
+            }
+
+            logger.info(f"✅ Successfully ingested {record_count} records to {table_name}")
+
+            return metrics
+
+        except Exception as e:
+            logger.error(f"❌ Ingestion failed: {str(e)}")
+
+            return {
+                "status": "failed",
+                "table_name": table_name,
+                "error": str(e),
+                "ingestion_timestamp": datetime.now().isoformat()
+            }
+
+    def _read_source(
+        self,
+        source_path: str,
+        source_format: str,
+        schema: Optional[StructType] = None,
+        options: Optional[Dict[str, str]] = None
+    ) -> DataFrame:
+        """Read data from source."""
+        options = options or {}
+
+        reader = self.spark.read.format(source_format)
+
+        if schema:
+            reader = reader.schema(schema)
+
+        for key, value in options.items():
+            reader = reader.option(key, value)
+
+        return reader.load(source_path)
+
+    def _add_bronze_metadata(
+        self,
+        df: DataFrame,
+        source_path: str,
+        table_name: str
+    ) -> DataFrame:
+        """Add bronze layer audit columns."""
+        return df \
+            .withColumn("_bronze_ingestion_timestamp", F.current_timestamp()) \
+            .withColumn("_bronze_source_path", F.lit(source_path)) \
+            .withColumn("_bronze_table_name", F.lit(table_name)) \
+            .withColumn("_bronze_ingestion_date", F.current_date())
+
+    def _validate_schema(
+        self,
+        df: DataFrame,
+        expected_schema: StructType
+    ) -> DataFrame:
+        """
+        Validate DataFrame against expected schema.
+
+        Quarantine records that don't match schema.
+        """
+        # In production, implement sophisticated schema validation
+        # For now, we return the df as-is
+        return df
+
+    def ingest_csv(
+        self,
+        csv_path: str,
+        table_name: str,
+        delimiter: str = ",",
+        header: bool = True,
+        schema: Optional[StructType] = None
+    ) -> Dict[str, Any]:
+        """Convenience method for CSV ingestion."""
+        options = {
+            "delimiter": delimiter,
+            "header": str(header).lower(),
+            "inferSchema": "true" if schema is None else "false"
+        }
+
+        return self.ingest_from_source(
+            source_path=csv_path,
+            table_name=table_name,
+            source_format="csv",
+            schema=schema,
+            options=options
+        )
+
+    def ingest_json(
+        self,
+        json_path: str,
+        table_name: str,
+        multiline: bool = False,
+        schema: Optional[StructType] = None
+    ) -> Dict[str, Any]:
+        """Convenience method for JSON ingestion."""
+        options = {
+            "multiLine": str(multiline).lower()
+        }
+
+        return self.ingest_from_source(
+            source_path=json_path,
+            table_name=table_name,
+            source_format="json",
+            schema=schema,
+            options=options
+        )
+
+    def ingest_parquet(
+        self,
+        parquet_path: str,
+        table_name: str
+    ) -> Dict[str, Any]:
+        """Convenience method for Parquet ingestion."""
+        return self.ingest_from_source(
+            source_path=parquet_path,
+            table_name=table_name,
+            source_format="parquet"
+        )
+
+    def create_bronze_table(
+        self,
+        table_name: str,
+        schema: StructType,
+        partition_by: Optional[List[str]] = None
+    ) -> None:
+        """Create an empty bronze table with schema."""
+        bronze_table_path = f"{self.bronze_path}/{table_name}"
+
+        # Create empty DataFrame with schema
+        empty_df = self.spark.createDataFrame([], schema)
+
+        # Add bronze metadata columns
+        bronze_df = self._add_bronze_metadata(empty_df, "initialized", table_name)
+
+        # Write table
+        writer = bronze_df.write.format("delta").mode("overwrite")
+
+        if partition_by:
+            writer = writer.partitionBy(*partition_by)
+
+        writer.save(bronze_table_path)
+
+        logger.info(f"✅ Created bronze table: {table_name}")
+
+
+# Example CRM schema
+CRM_LEADS_SCHEMA = StructType([
+    StructField("lead_id", StringType(), False),
+    StructField("email", StringType(), True),
+    StructField("company", StringType(), True),
+    StructField("industry", StringType(), True),
+    StructField("company_size", StringType(), True),
+    StructField("job_title", StringType(), True),
+    StructField("lead_source", StringType(), True),
+    StructField("created_date", TimestampType(), True),
+    StructField("lead_score", IntegerType(), True),
+    StructField("status", StringType(), True)
+])
+
+
+# Example usage
+if __name__ == "__main__":
+    print("=" * 80)
+    print("Bronze Layer Ingestion Demo")
+    print("=" * 80)
+
+    # Create sample data
+    sample_data = [
+        {
+            "lead_id": "L001",
+            "email": "john@techcorp.com",
+            "company": "TechCorp",
+            "industry": "Software",
+            "company_size": "100-500",
+            "job_title": "Data Scientist",
+            "lead_source": "Website",
+            "created_date": "2025-01-15T10:30:00",
+            "lead_score": 85,
+            "status": "New"
+        },
+        {
+            "lead_id": "L002",
+            "email": "sarah@datainc.com",
+            "company": "Data Inc",
+            "industry": "Analytics",
+            "company_size": "50-100",
+            "job_title": "ML Engineer",
+            "lead_source": "LinkedIn",
+            "created_date": "2025-01-16T14:20:00",
+            "lead_score": 92,
+            "status": "Qualified"
+        }
+    ]
+
+    # Save as JSON
+    sample_path = "/tmp/sample_crm_leads.json"
+    with open(sample_path, 'w') as f:
+        json.dump(sample_data, f)
+
+    # Initialize Bronze loader
+    bronze = BronzeLoader(
+        bronze_path="./lakehouse/bronze",
+        quarantine_path="./lakehouse/quarantine"
+    )
+
+    # Ingest data
+    metrics = bronze.ingest_json(
+        json_path=sample_path,
+        table_name="crm_leads",
+        multiline=True,
+        schema=CRM_LEADS_SCHEMA
+    )
+
+    print("\n📊 Ingestion Metrics:")
+    print(json.dumps(metrics, indent=2))
+
+    # Query bronze table
+    print("\n📋 Bronze Table Sample:")
+    bronze_df = bronze.spark.read.format("delta").load("./lakehouse/bronze/crm_leads")
+    bronze_df.show(truncate=False)
+
+    print(f"\nBronze table row count: {bronze_df.count()}")

package/.claude/roles/data-engineer/skills/01-lakehouse-architecture/medallion_queries.sql

@@ -0,0 +1,300 @@
+-- Medallion Architecture SQL Patterns
+-- Bronze → Silver → Gold transformations for Data Lakehouse
+
+-- ================================================================
+-- BRONZE LAYER - Raw Data Ingestion
+-- ================================================================
+
+-- View bronze layer with metadata
+SELECT
+    *,
+    _bronze_ingestion_timestamp,
+    _bronze_source_path,
+    _bronze_table_name
+FROM bronze.crm_leads
+WHERE _bronze_ingestion_date >= CURRENT_DATE - INTERVAL '7 days'
+ORDER BY _bronze_ingestion_timestamp DESC;
+
+-- Check for duplicate records in bronze
+SELECT
+    lead_id,
+    COUNT(*) as duplicate_count,
+    MIN(_bronze_ingestion_timestamp) as first_seen,
+    MAX(_bronze_ingestion_timestamp) as last_seen
+FROM bronze.crm_leads
+GROUP BY lead_id
+HAVING COUNT(*) > 1;
+
+-- Bronze layer data quality check
+SELECT
+    _bronze_ingestion_date,
+    COUNT(*) as total_records,
+    COUNT(DISTINCT lead_id) as unique_leads,
+    COUNT(*) - COUNT(DISTINCT lead_id) as duplicates,
+    COUNT(CASE WHEN email IS NULL THEN 1 END) as missing_email,
+    COUNT(CASE WHEN company IS NULL THEN 1 END) as missing_company
+FROM bronze.crm_leads
+GROUP BY _bronze_ingestion_date
+ORDER BY _bronze_ingestion_date DESC;
+
+-- ================================================================
+-- SILVER LAYER - Cleaned & Standardized
+-- ================================================================
+
+-- Transform Bronze → Silver (Deduplication & Cleaning)
+CREATE OR REPLACE TABLE silver.crm_leads_clean AS
+WITH deduplicated AS (
+    SELECT *,
+        ROW_NUMBER() OVER (
+            PARTITION BY lead_id
+            ORDER BY _bronze_ingestion_timestamp DESC
+        ) as rn
+    FROM bronze.crm_leads
+),
+cleaned AS (
+    SELECT
+        lead_id,
+        LOWER(TRIM(email)) as email,
+        TRIM(company) as company,
+        UPPER(industry) as industry,
+        company_size,
+        job_title,
+        lead_source,
+        created_date,
+        COALESCE(lead_score, 0) as lead_score,
+        UPPER(status) as status,
+        -- Silver metadata
+        CURRENT_TIMESTAMP() as _silver_processed_timestamp,
+        _bronze_ingestion_timestamp as _bronze_ingestion_timestamp
+    FROM deduplicated
+    WHERE rn = 1  -- Keep only most recent version
+        AND email IS NOT NULL  -- Basic validation
+        AND email LIKE '%@%'  -- Email format check
+)
+SELECT * FROM cleaned;
+
+-- Silver layer quality metrics
+SELECT
+    COUNT(*) as total_records,
+    COUNT(DISTINCT email) as unique_emails,
+    COUNT(CASE WHEN lead_score >= 80 THEN 1 END) as high_score_leads,
+    AVG(lead_score) as avg_lead_score,
+    COUNT(DISTINCT industry) as unique_industries,
+    COUNT(DISTINCT company) as unique_companies,
+    MAX(_silver_processed_timestamp) as last_processed
+FROM silver.crm_leads_clean;
+
+-- Schema drift detection (Silver)
+SELECT
+    column_name,
+    data_type,
+    is_nullable,
+    COUNT(*) OVER () as total_columns
+FROM information_schema.columns
+WHERE table_schema = 'silver'
+  AND table_name = 'crm_leads_clean'
+ORDER BY ordinal_position;
+
+-- ================================================================
+-- GOLD LAYER - Business Logic & Aggregations
+-- ================================================================
+
+-- Transform Silver → Gold (Lead Segmentation)
+CREATE OR REPLACE TABLE gold.lead_segments AS
+SELECT
+    lead_id,
+    email,
+    company,
+    industry,
+    company_size,
+    job_title,
+    lead_source,
+    created_date,
+    lead_score,
+    status,
+    -- Business logic: Lead segment
+    CASE
+        WHEN lead_score >= 90 THEN 'HOT'
+        WHEN lead_score >= 70 THEN 'WARM'
+        WHEN lead_score >= 50 THEN 'QUALIFIED'
+        ELSE 'COLD'
+    END as lead_segment,
+    -- Seniority level from job title
+    CASE
+        WHEN UPPER(job_title) LIKE '%VP%' OR UPPER(job_title) LIKE '%VICE PRESIDENT%' THEN 'VP+'
+        WHEN UPPER(job_title) LIKE '%DIRECTOR%' THEN 'Director'
+        WHEN UPPER(job_title) LIKE '%MANAGER%' THEN 'Manager'
+        WHEN UPPER(job_title) LIKE '%SENIOR%' OR UPPER(job_title) LIKE '%SR%' THEN 'Senior IC'
+        ELSE 'IC'
+    END as seniority_level,
+    -- Company size category
+    CASE
+        WHEN company_size IN ('1000+', '500-1000') THEN 'Enterprise'
+        WHEN company_size IN ('100-500', '50-100') THEN 'Mid-Market'
+        ELSE 'SMB'
+    END as company_category,
+    -- Days since creation
+    DATEDIFF(CURRENT_DATE, created_date) as days_since_created,
+    -- Gold metadata
+    CURRENT_TIMESTAMP() as _gold_created_timestamp
+FROM silver.crm_leads_clean;
+
+-- Gold Layer: Daily Lead Metrics
+CREATE OR REPLACE TABLE gold.daily_lead_metrics AS
+SELECT
+    DATE(created_date) as metric_date,
+    lead_source,
+    lead_segment,
+    company_category,
+    COUNT(*) as lead_count,
+    AVG(lead_score) as avg_lead_score,
+    COUNT(CASE WHEN lead_segment = 'HOT' THEN 1 END) as hot_leads,
+    COUNT(CASE WHEN status = 'QUALIFIED' THEN 1 END) as qualified_leads,
+    COUNT(DISTINCT company) as unique_companies,
+    COUNT(DISTINCT industry) as unique_industries
+FROM gold.lead_segments
+GROUP BY
+    DATE(created_date),
+    lead_source,
+    lead_segment,
+    company_category;
+
+-- Gold Layer: Lead Source Performance
+CREATE OR REPLACE VIEW gold.lead_source_performance AS
+SELECT
+    lead_source,
+    COUNT(*) as total_leads,
+    AVG(lead_score) as avg_score,
+    COUNT(CASE WHEN lead_segment = 'HOT' THEN 1 END) as hot_leads,
+    COUNT(CASE WHEN lead_segment IN ('HOT', 'WARM') THEN 1 END) as quality_leads,
+    ROUND(100.0 * COUNT(CASE WHEN lead_segment IN ('HOT', 'WARM') THEN 1 END) / COUNT(*), 2) as quality_rate,
+    COUNT(DISTINCT company) as unique_companies,
+    MAX(created_date) as latest_lead_date,
+    DATEDIFF(CURRENT_DATE, MAX(created_date)) as days_since_last_lead
+FROM gold.lead_segments
+GROUP BY lead_source
+ORDER BY quality_rate DESC;
+
+-- Gold Layer: Industry Analysis
+CREATE OR REPLACE VIEW gold.industry_analysis AS
+SELECT
+    industry,
+    company_category,
+    COUNT(*) as lead_count,
+    AVG(lead_score) as avg_lead_score,
+    COUNT(CASE WHEN lead_segment = 'HOT' THEN 1 END) as hot_leads,
+    COUNT(CASE WHEN seniority_level IN ('VP+', 'Director') THEN 1 END) as senior_decision_makers,
+    COUNT(DISTINCT company) as unique_companies,
+    ROUND(AVG(days_since_created), 1) as avg_age_days
+FROM gold.lead_segments
+GROUP BY industry, company_category
+HAVING COUNT(*) >= 10
+ORDER BY hot_leads DESC, avg_lead_score DESC;
+
+-- ================================================================
+-- INCREMENTAL PROCESSING PATTERNS
+-- ================================================================
+
+-- Incremental load: Bronze to Silver (only new/updated records)
+MERGE INTO silver.crm_leads_clean AS target
+USING (
+    SELECT
+        lead_id,
+        LOWER(TRIM(email)) as email,
+        TRIM(company) as company,
+        UPPER(industry) as industry,
+        company_size,
+        job_title,
+        lead_source,
+        created_date,
+        COALESCE(lead_score, 0) as lead_score,
+        UPPER(status) as status,
+        _bronze_ingestion_timestamp
+    FROM (
+        SELECT *,
+            ROW_NUMBER() OVER (
+                PARTITION BY lead_id
+                ORDER BY _bronze_ingestion_timestamp DESC
+            ) as rn
+        FROM bronze.crm_leads
+        WHERE _bronze_ingestion_timestamp > (
+            SELECT COALESCE(MAX(_bronze_ingestion_timestamp), '1900-01-01')
+            FROM silver.crm_leads_clean
+        )
+    )
+    WHERE rn = 1
+        AND email IS NOT NULL
+        AND email LIKE '%@%'
+) AS source
+ON target.lead_id = source.lead_id
+WHEN MATCHED THEN
+    UPDATE SET
+        email = source.email,
+        company = source.company,
+        industry = source.industry,
+        company_size = source.company_size,
+        job_title = source.job_title,
+        lead_source = source.lead_source,
+        created_date = source.created_date,
+        lead_score = source.lead_score,
+        status = source.status,
+        _silver_processed_timestamp = CURRENT_TIMESTAMP(),
+        _bronze_ingestion_timestamp = source._bronze_ingestion_timestamp
+WHEN NOT MATCHED THEN
+    INSERT (
+        lead_id, email, company, industry, company_size,
+        job_title, lead_source, created_date, lead_score, status,
+        _silver_processed_timestamp, _bronze_ingestion_timestamp
+    )
+    VALUES (
+        source.lead_id, source.email, source.company, source.industry,
+        source.company_size, source.job_title, source.lead_source,
+        source.created_date, source.lead_score, source.status,
+        CURRENT_TIMESTAMP(), source._bronze_ingestion_timestamp
+    );
+
+-- ================================================================
+-- DATA QUALITY MONITORING
+-- ================================================================
+
+-- Cross-layer data quality dashboard
+SELECT
+    'Bronze' as layer,
+    COUNT(*) as record_count,
+    COUNT(DISTINCT lead_id) as unique_ids,
+    MAX(_bronze_ingestion_timestamp) as last_update
+FROM bronze.crm_leads
+
+UNION ALL
+
+SELECT
+    'Silver' as layer,
+    COUNT(*) as record_count,
+    COUNT(DISTINCT lead_id) as unique_ids,
+    MAX(_silver_processed_timestamp) as last_update
+FROM silver.crm_leads_clean
+
+UNION ALL
+
+SELECT
+    'Gold' as layer,
+    COUNT(*) as record_count,
+    COUNT(DISTINCT lead_id) as unique_ids,
+    MAX(_gold_created_timestamp) as last_update
+FROM gold.lead_segments;
+
+-- ================================================================
+-- PERFORMANCE OPTIMIZATION
+-- ================================================================
+
+-- Optimize Silver table (Vacuum + Optimize)
+-- OPTIMIZE silver.crm_leads_clean ZORDER BY (lead_id, created_date);
+-- VACUUM silver.crm_leads_clean RETAIN 168 HOURS;  -- 7 days
+
+-- Optimize Gold table
+-- OPTIMIZE gold.lead_segments ZORDER BY (lead_segment, created_date, company_category);
+-- VACUUM gold.lead_segments RETAIN 168 HOURS;
+
+-- Table statistics for query optimization
+-- ANALYZE TABLE silver.crm_leads_clean COMPUTE STATISTICS;
+-- ANALYZE TABLE gold.lead_segments COMPUTE STATISTICS FOR ALL COLUMNS;