mangleframes 0.1.9__tar.gz → 0.2.1__tar.gz

{mangleframes-0.1.9 → mangleframes-0.2.1}/Cargo.lock

@@ -418,6 +418,7 @@ dependencies = [
  "matchit",
  "memchr",
  "mime",
+ "multer",
  "percent-encoding",
  "pin-project-lite",
  "serde_core",
@@ -1280,6 +1281,15 @@ version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
 
+[[package]]
+name = "encoding_rs"
+version = "0.8.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "equivalent"
 version = "1.0.2"
@@ -1957,6 +1967,7 @@ dependencies = [
  "arrow-ipc",
  "arrow-json",
  "axum",
+ "base64",
  "chrono",
  "clap",
  "datafusion",
@@ -2041,6 +2052,23 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
+[[package]]
+name = "multer"
+version = "3.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b"
+dependencies = [
+ "bytes",
+ "encoding_rs",
+ "futures-util",
+ "http",
+ "httparse",
+ "memchr",
+ "mime",
+ "spin",
+ "version_check",
+]
+
 [[package]]
 name = "ndk-context"
 version = "0.1.1"
@@ -2767,6 +2795,12 @@ dependencies = [
  "windows-sys 0.60.2",
 ]
 
+[[package]]
+name = "spin"
+version = "0.9.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+
 [[package]]
 name = "sqlparser"
 version = "0.53.0"

{mangleframes-0.1.9 → mangleframes-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mangleframes
-Version: 0.1.9
+Version: 0.2.1
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Rust
 Classifier: License :: OSI Approved :: MIT License

{mangleframes-0.1.9 → mangleframes-0.2.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "mangleframes"
-version = "0.1.9"
+version = "0.2.1"
 description = "PySpark DataFrame viewer with modern web UI"
 requires-python = ">=3.11"
 license = { text = "MIT" }

{mangleframes-0.1.9 → mangleframes-0.2.1}/python/mangleframes/__init__.py

@@ -16,7 +16,7 @@ from .server import DataFrameServer
 if TYPE_CHECKING:
     from pyspark.sql import DataFrame
 
-__version__ = "0.1.9"
+__version__ = "0.2.1"
 __all__ = ["register", "unregister", "show", "cleanup"]
 
 _registry: dict[str, DataFrame] = {}

{mangleframes-0.1.9 → mangleframes-0.2.1}/python/mangleframes/protocol.py

@@ -131,8 +131,9 @@ def handle_get(registry: dict[str, DataFrame], name: str, limit: int) -> bytes:
         return encode_error(f"DataFrame '{name}' not found")
 
     # Check cache first - return cached data if limit is sufficient
+    # limit=0 means "fetch all rows", so bypass cache in that case
     with _arrow_cache_lock:
-        if name in _arrow_cache:
+        if name in _arrow_cache and limit > 0:
             cached_limit, cached_payload = _arrow_cache[name]
             if cached_limit >= limit:
                 return encode_response(STATUS_OK, cached_payload)

{mangleframes-0.1.9 → mangleframes-0.2.1}/viewer/Cargo.toml

@@ -9,7 +9,7 @@ path = "src/main.rs"
 
 [dependencies]
 tokio = { version = "1", features = ["full"] }
-axum = { version = "0.8", features = ["ws"] }
+axum = { version = "0.8", features = ["ws", "multipart"] }
 tower-http = { version = "0.6", features = ["cors", "fs"] }
 rust-embed = "8"
 arrow = { version = "54", features = ["ipc"] }
@@ -28,6 +28,7 @@ thiserror = "2"
 anyhow = "1"
 futures-util = "0.3"
 chrono = "0.4"
+base64 = "0.22"
 
 [profile.release]
 lto = true

mangleframes-0.2.1/viewer/src/dashboard.rs

@@ -0,0 +1,428 @@
+//! HTML dashboard generation for reconciliation reports.
+
+use crate::reconcile::{AggregatedReconcileResult, ColumnTotalComparison};
+use chrono::Utc;
+
+/// Metadata for the dashboard report.
+pub struct DashboardMetadata {
+    pub source_frame: String,
+    pub target_frame: String,
+    pub source_type: String,
+    pub group_by_source: Vec<String>,
+    pub group_by_target: Vec<String>,
+    pub join_keys_source: Vec<String>,
+    pub join_keys_target: Vec<String>,
+}
+
+/// Generate a self-contained HTML dashboard report.
+pub fn generate_reconcile_dashboard(
+    result: &AggregatedReconcileResult,
+    metadata: &DashboardMetadata,
+) -> String {
+    let timestamp = Utc::now().format("%Y-%m-%d %H:%M:%S UTC").to_string();
+    let stats = &result.statistics;
+
+    let status_class = if stats.match_rate >= 0.99 { "pass" } else { "warn" };
+    let status_text = if stats.match_rate >= 0.99 { "PASS" } else { "REVIEW" };
+
+    format!(
+        r#"<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Reconciliation Report - {} vs {}</title>
+    <style>{}</style>
+</head>
+<body>
+    <header>
+        <h1>Reconciliation Report</h1>
+        <div class="status-badge {}">{}</div>
+    </header>
+
+    <section class="metadata">
+        <h2>Report Details</h2>
+        <table class="metadata-table">
+            <tr><td>Generated</td><td>{}</td></tr>
+            <tr><td>Source Frame</td><td>{} ({})</td></tr>
+            <tr><td>Target Frame</td><td>{}</td></tr>
+            <tr><td>Source Group By</td><td>{}</td></tr>
+            <tr><td>Target Group By</td><td>{}</td></tr>
+            <tr><td>Join Keys (Source)</td><td>{}</td></tr>
+            <tr><td>Join Keys (Target)</td><td>{}</td></tr>
+        </table>
+    </section>
+
+    <section class="summary">
+        <h2>Executive Summary</h2>
+        <div class="stats-grid">
+            <div class="stat-card">
+                <div class="stat-value">{:.1}%</div>
+                <div class="stat-label">Match Rate</div>
+            </div>
+            <div class="stat-card">
+                <div class="stat-value">{}</div>
+                <div class="stat-label">Source Groups</div>
+            </div>
+            <div class="stat-card">
+                <div class="stat-value">{}</div>
+                <div class="stat-label">Target Groups</div>
+            </div>
+            <div class="stat-card">
+                <div class="stat-value">{}</div>
+                <div class="stat-label">Matched Groups</div>
+            </div>
+            <div class="stat-card highlight-warn">
+                <div class="stat-value">{}</div>
+                <div class="stat-label">Source Only</div>
+            </div>
+            <div class="stat-card highlight-warn">
+                <div class="stat-value">{}</div>
+                <div class="stat-label">Target Only</div>
+            </div>
+        </div>
+    </section>
+
+    <section class="column-totals">
+        <h2>Column Totals Comparison</h2>
+        {}
+    </section>
+
+    <section class="differences">
+        <h2>Detailed Differences</h2>
+
+        <div class="subsection">
+            <h3>Source Only ({} groups)</h3>
+            {}
+        </div>
+
+        <div class="subsection">
+            <h3>Target Only ({} groups)</h3>
+            {}
+        </div>
+    </section>
+
+    <section class="matched-sample">
+        <h2>Matched Sample ({} of {} groups)</h2>
+        {}
+    </section>
+
+    <footer>
+        <p>Generated by MangleFrames | {}</p>
+    </footer>
+</body>
+</html>"#,
+        metadata.source_frame,
+        metadata.target_frame,
+        get_css(),
+        status_class,
+        status_text,
+        timestamp,
+        metadata.source_frame,
+        metadata.source_type,
+        metadata.target_frame,
+        metadata.group_by_source.join(", "),
+        metadata.group_by_target.join(", "),
+        metadata.join_keys_source.join(", "),
+        metadata.join_keys_target.join(", "),
+        stats.match_rate * 100.0,
+        stats.source_groups,
+        stats.target_groups,
+        stats.matched_groups,
+        stats.source_only_groups,
+        stats.target_only_groups,
+        render_column_totals_table(&result.column_totals),
+        result.source_only.total,
+        render_json_rows_table(&result.source_only.rows, 100),
+        result.target_only.total,
+        render_json_rows_table(&result.target_only.rows, 100),
+        result.matched_rows.rows.len(),
+        result.matched_rows.total,
+        render_json_rows_table(&result.matched_rows.rows, 50),
+        timestamp
+    )
+}
+
+fn get_css() -> &'static str {
+    r#"
+* { box-sizing: border-box; margin: 0; padding: 0; }
+
+body {
+    font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+    line-height: 1.5;
+    color: #1a1a2e;
+    background: #fff;
+    padding: 40px;
+    max-width: 1200px;
+    margin: 0 auto;
+}
+
+header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    border-bottom: 2px solid #1a1a2e;
+    padding-bottom: 20px;
+    margin-bottom: 30px;
+}
+
+h1 { font-size: 28px; font-weight: 700; }
+h2 { font-size: 20px; font-weight: 600; margin-bottom: 16px; color: #1a1a2e; }
+h3 { font-size: 16px; font-weight: 600; margin-bottom: 12px; color: #444; }
+
+.status-badge {
+    padding: 8px 20px;
+    border-radius: 20px;
+    font-weight: 700;
+    font-size: 14px;
+    text-transform: uppercase;
+}
+
+.status-badge.pass { background: #d4edda; color: #155724; }
+.status-badge.warn { background: #fff3cd; color: #856404; }
+
+section { margin-bottom: 40px; }
+
+.metadata-table {
+    width: 100%;
+    border-collapse: collapse;
+    font-size: 14px;
+}
+
+.metadata-table td {
+    padding: 8px 12px;
+    border-bottom: 1px solid #e0e0e0;
+}
+
+.metadata-table td:first-child {
+    font-weight: 600;
+    width: 200px;
+    color: #666;
+}
+
+.stats-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
+    gap: 16px;
+}
+
+.stat-card {
+    background: #f8f9fa;
+    border: 1px solid #e0e0e0;
+    border-radius: 8px;
+    padding: 20px;
+    text-align: center;
+}
+
+.stat-card.highlight-warn { border-left: 4px solid #ffc107; }
+
+.stat-value {
+    font-size: 28px;
+    font-weight: 700;
+    color: #1a1a2e;
+    font-family: 'SF Mono', Monaco, monospace;
+}
+
+.stat-label {
+    font-size: 12px;
+    color: #666;
+    text-transform: uppercase;
+    margin-top: 4px;
+}
+
+table.data-table {
+    width: 100%;
+    border-collapse: collapse;
+    font-size: 13px;
+    margin-top: 12px;
+}
+
+table.data-table th,
+table.data-table td {
+    padding: 10px 12px;
+    text-align: left;
+    border-bottom: 1px solid #e0e0e0;
+}
+
+table.data-table th {
+    background: #f8f9fa;
+    font-weight: 600;
+    font-size: 11px;
+    text-transform: uppercase;
+    color: #666;
+}
+
+table.data-table tr:hover { background: #f8f9fa; }
+
+.diff-positive { color: #28a745; }
+.diff-negative { color: #dc3545; }
+.diff-zero { color: #666; }
+
+.subsection { margin-bottom: 24px; }
+
+.empty-notice {
+    padding: 20px;
+    text-align: center;
+    color: #666;
+    font-style: italic;
+    background: #f8f9fa;
+    border-radius: 4px;
+}
+
+footer {
+    margin-top: 40px;
+    padding-top: 20px;
+    border-top: 1px solid #e0e0e0;
+    text-align: center;
+    font-size: 12px;
+    color: #666;
+}
+
+@media print {
+    body { padding: 20px; }
+    section { page-break-inside: avoid; }
+}
+"#
+}
+
+fn render_column_totals_table(totals: &[ColumnTotalComparison]) -> String {
+    if totals.is_empty() {
+        return r#"<div class="empty-notice">No aggregation columns configured</div>"#.to_string();
+    }
+
+    let mut rows = String::new();
+    for total in totals {
+        let diff_class = match total.difference {
+            Some(d) if d > 0.001 => "diff-positive",
+            Some(d) if d < -0.001 => "diff-negative",
+            _ => "diff-zero",
+        };
+
+        rows.push_str(&format!(
+            r#"<tr>
+                <td>{}</td>
+                <td>{}</td>
+                <td>{}</td>
+                <td>{}</td>
+                <td class="{}">{}</td>
+                <td>{}</td>
+            </tr>"#,
+            total.column,
+            total.aggregation,
+            format_number(total.source_total),
+            format_number(total.target_total),
+            diff_class,
+            format_number(total.difference),
+            format_percent(total.percent_diff)
+        ));
+    }
+
+    format!(
+        r#"<table class="data-table">
+            <thead>
+                <tr>
+                    <th>Column</th>
+                    <th>Aggregation</th>
+                    <th>Source Total</th>
+                    <th>Target Total</th>
+                    <th>Difference</th>
+                    <th>% Diff</th>
+                </tr>
+            </thead>
+            <tbody>{}</tbody>
+        </table>"#,
+        rows
+    )
+}
+
+fn render_json_rows_table(rows: &[serde_json::Value], max_rows: usize) -> String {
+    if rows.is_empty() {
+        return r#"<div class="empty-notice">No rows to display</div>"#.to_string();
+    }
+
+    let columns: Vec<String> = if let Some(obj) = rows[0].as_object() {
+        obj.keys().cloned().collect()
+    } else {
+        return r#"<div class="empty-notice">Invalid data format</div>"#.to_string();
+    };
+
+    let header: String = columns.iter()
+        .map(|c| format!("<th>{}</th>", html_escape(c)))
+        .collect();
+
+    let display_rows = rows.iter().take(max_rows);
+    let mut body = String::new();
+    for row in display_rows {
+        if let Some(obj) = row.as_object() {
+            let cells: String = columns.iter()
+                .map(|col| {
+                    let val = obj.get(col)
+                        .map(|v| format_json_value(v))
+                        .unwrap_or_default();
+                    format!("<td>{}</td>", html_escape(&val))
+                })
+                .collect();
+            body.push_str(&format!("<tr>{}</tr>", cells));
+        }
+    }
+
+    let truncated_notice = if rows.len() > max_rows {
+        format!(r#"<div class="empty-notice">Showing {} of {} rows</div>"#, max_rows, rows.len())
+    } else {
+        String::new()
+    };
+
+    format!(
+        r#"<table class="data-table">
+            <thead><tr>{}</tr></thead>
+            <tbody>{}</tbody>
+        </table>
+        {}"#,
+        header, body, truncated_notice
+    )
+}
+
+fn format_number(val: Option<f64>) -> String {
+    match val {
+        Some(v) if v.abs() >= 1_000_000.0 => format!("{:.2}M", v / 1_000_000.0),
+        Some(v) if v.abs() >= 1_000.0 => format!("{:.2}K", v / 1_000.0),
+        Some(v) => format!("{:.2}", v),
+        None => "-".to_string(),
+    }
+}
+
+fn format_percent(val: Option<f64>) -> String {
+    match val {
+        Some(v) => format!("{:.2}%", v),
+        None => "-".to_string(),
+    }
+}
+
+fn format_json_value(val: &serde_json::Value) -> String {
+    match val {
+        serde_json::Value::Null => "".to_string(),
+        serde_json::Value::Bool(b) => b.to_string(),
+        serde_json::Value::Number(n) => {
+            if let Some(f) = n.as_f64() {
+                if f.abs() >= 1_000_000.0 {
+                    format!("{:.2}M", f / 1_000_000.0)
+                } else if f.abs() >= 1_000.0 {
+                    format!("{:.2}K", f / 1_000.0)
+                } else {
+                    format!("{:.2}", f)
+                }
+            } else {
+                n.to_string()
+            }
+        }
+        serde_json::Value::String(s) => s.clone(),
+        _ => val.to_string(),
+    }
+}
+
+fn html_escape(s: &str) -> String {
+    s.replace('&', "&amp;")
+        .replace('<', "&lt;")
+        .replace('>', "&gt;")
+        .replace('"', "&quot;")
+}

{mangleframes-0.1.9 → mangleframes-0.2.1}/viewer/src/join_handlers.rs

@@ -1,7 +1,6 @@
 //! HTTP handlers for join analysis endpoints.
 
 use std::sync::Arc;
-use std::time::Instant;
 
 use arrow::record_batch::RecordBatch;
 use axum::body::Body;
@@ -15,7 +14,10 @@ use serde_json::json;
 use crate::arrow_reader;
 use crate::export;
 use crate::join_analysis::JoinAnalyzer;
-use crate::web_server::{AppState, CachedFrame};
+use crate::web_server::AppState;
+
+/// Limit of 0 means fetch all rows (no limit)
+const JOIN_FETCH_LIMIT: usize = 0;
 
 #[derive(Deserialize)]
 pub struct JoinRequest {
@@ -171,30 +173,15 @@ async fn get_or_fetch_frame(
     state: &AppState,
     frame_name: &str,
 ) -> Result<Vec<RecordBatch>, axum::response::Response> {
-    // Check cache first
-    {
-        let cache = state.cache.read().await;
-        if let Some(cached) = cache.get(frame_name) {
-            return Ok(cached.batches.clone());
-        }
-    }
-
-    // Fetch from Python server
-    let response = state.client.get_frame(frame_name, 100000).map_err(|e| {
+    // Always fetch fresh for join analysis to ensure we get all rows
+    // (UI cache may have truncated data from paginated viewing)
+    let response = state.client.get_frame(frame_name, JOIN_FETCH_LIMIT).map_err(|e| {
         error_response(StatusCode::NOT_FOUND, &format!("Failed to fetch {}: {}", frame_name, e))
     })?;
 
     let batches = arrow_reader::parse_arrow_stream(&response.data)
         .map_err(|e| error_response(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()))?;
 
-    // Cache for future use
-    state.evict_frame_if_needed().await;
-    let mut cache = state.cache.write().await;
-    cache.insert(
-        frame_name.to_string(),
-        CachedFrame { batches: batches.clone(), stats: None, last_access: Instant::now() },
-    );
-
     Ok(batches)
 }
 

{mangleframes-0.1.9 → mangleframes-0.2.1}/viewer/src/main.rs

@@ -1,6 +1,7 @@
 //! MangleFrames Viewer - Web-based PySpark DataFrame viewer.
 
 mod arrow_reader;
+mod dashboard;
 mod dq_handlers;
 mod export;
 mod handlers;
@@ -9,6 +10,8 @@ mod history_handlers;
 mod join_analysis;
 mod join_handlers;
 mod perf;
+mod reconcile;
+mod reconcile_handlers;
 mod query_engine;
 mod socket_client;
 mod stats;