mangleframes 0.1.1__tar.gz → 0.1.2__tar.gz

{mangleframes-0.1.1 → mangleframes-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mangleframes
-Version: 0.1.1
+Version: 0.1.2
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Rust
 Classifier: License :: OSI Approved :: MIT License

{mangleframes-0.1.1 → mangleframes-0.1.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "mangleframes"
-version = "0.1.1"
+version = "0.1.2"
 description = "PySpark DataFrame viewer with modern web UI"
 requires-python = ">=3.9"
 license = { text = "MIT" }

{mangleframes-0.1.1 → mangleframes-0.1.2}/python/mangleframes/__init__.py

@@ -5,12 +5,13 @@ import time
 from typing import TYPE_CHECKING
 
 from .launcher import launch_viewer, open_browser
+from .protocol import clear_stats_cache
 from .server import DataFrameServer
 
 if TYPE_CHECKING:
     from pyspark.sql import DataFrame
 
-__version__ = "0.1.1"
+__version__ = "0.1.2"
 __all__ = ["register", "unregister", "show"]
 
 _registry: dict[str, DataFrame] = {}
@@ -23,6 +24,7 @@ def register(name: str, df: DataFrame) -> None:
     global _server
 
     _registry[name] = df
+    clear_stats_cache(name)  # Invalidate cached stats for this name
 
     if _server is None:
         _server = DataFrameServer(_registry)
@@ -33,6 +35,7 @@ def unregister(name: str) -> None:
     """Remove a DataFrame from the viewer."""
     if name in _registry:
         del _registry[name]
+        clear_stats_cache(name)
 
 
 def show(port: int = 8765, block: bool = True) -> None:

{mangleframes-0.1.1 → mangleframes-0.1.2}/python/mangleframes/protocol.py

@@ -6,6 +6,7 @@ import struct
 from typing import TYPE_CHECKING, Any
 
 import pyarrow as pa
+from pyspark.sql import functions as F
 
 if TYPE_CHECKING:
     from pyspark.sql import DataFrame
@@ -13,6 +14,17 @@ if TYPE_CHECKING:
 STATUS_OK = 0
 STATUS_ERROR = 1
 
+# Cache for computed stats (cleared when DataFrame is re-registered)
+_stats_cache: dict[str, dict] = {}
+
+
+def clear_stats_cache(name: str | None = None) -> None:
+    """Clear cached stats for a DataFrame or all DataFrames."""
+    if name is None:
+        _stats_cache.clear()
+    elif name in _stats_cache:
+        del _stats_cache[name]
+
 
 def encode_response(status: int, payload: bytes) -> bytes:
     """Encode response with status and length prefix."""
@@ -66,37 +78,59 @@ def handle_get(registry: dict[str, DataFrame], name: str, limit: int) -> bytes:
     return encode_response(STATUS_OK, sink.getvalue().to_pybytes())
 
 
+def _is_numeric_type(dtype_str: str) -> bool:
+    """Check if a Spark type string represents a numeric type."""
+    dtype_lower = dtype_str.lower()
+    return any(t in dtype_lower for t in ["int", "long", "double", "float", "decimal"])
+
+
 def handle_stats(registry: dict[str, DataFrame], name: str) -> bytes:
-    """Return basic statistics for a DataFrame."""
+    """Return basic statistics for a DataFrame using single aggregation."""
     if name not in registry:
         return encode_error(f"DataFrame '{name}' not found")
 
+    # Return cached stats if available
+    if name in _stats_cache:
+        return encode_json_response(_stats_cache[name])
+
     df = registry[name]
-    row_count = df.count()
+    fields = df.schema.fields
 
+    # Build all aggregation expressions in one pass
+    agg_exprs = [F.count(F.lit(1)).alias("__total")]
+    for field in fields:
+        col_name = field.name
+        agg_exprs.append(
+            F.sum(F.when(F.col(col_name).isNull(), 1).otherwise(0)).alias(f"{col_name}__nulls")
+        )
+        if _is_numeric_type(str(field.dataType)):
+            agg_exprs.append(F.min(col_name).alias(f"{col_name}__min"))
+            agg_exprs.append(F.max(col_name).alias(f"{col_name}__max"))
+
+    # Single Spark action
+    result = df.agg(*agg_exprs).collect()[0]
+    row_count = result["__total"]
+
+    # Extract stats from result
     column_stats = []
-    for field in df.schema.fields:
+    for field in fields:
         col_name = field.name
-        stats = {"name": col_name, "type": str(field.dataType), "nullable": field.nullable}
-
-        null_count_row = df.filter(df[col_name].isNull()).count()
-        stats["null_count"] = null_count_row
+        dtype_str = str(field.dataType)
+        stats = {"name": col_name, "type": dtype_str, "nullable": field.nullable}
+        stats["null_count"] = result[f"{col_name}__nulls"] or 0
 
-        dtype = str(field.dataType).lower()
-        if any(t in dtype for t in ["int", "long", "double", "float", "decimal"]):
-            agg_result = df.agg({col_name: "min"}).collect()[0][0]
-            stats["min"] = str(agg_result) if agg_result is not None else None
-
-            agg_result = df.agg({col_name: "max"}).collect()[0][0]
-            stats["max"] = str(agg_result) if agg_result is not None else None
+        if _is_numeric_type(dtype_str):
+            min_val = result[f"{col_name}__min"]
+            max_val = result[f"{col_name}__max"]
+            stats["min"] = str(min_val) if min_val is not None else None
+            stats["max"] = str(max_val) if max_val is not None else None
 
         column_stats.append(stats)
 
-    return encode_json_response({
-        "name": name,
-        "row_count": row_count,
-        "columns": column_stats,
-    })
+    stats_data = {"name": name, "row_count": row_count, "columns": column_stats}
+    _stats_cache[name] = stats_data  # Cache for future requests
+
+    return encode_json_response(stats_data)
 
 
 def dispatch_command(

{mangleframes-0.1.1 → mangleframes-0.1.2}/viewer/src/main.rs

@@ -17,7 +17,7 @@ use tracing::info;
 use tracing_subscriber::EnvFilter;
 
 use crate::socket_client::SocketClient;
-use crate::web_server::AppState;
+use crate::web_server::{AppState, CachedFrame};
 
 #[derive(Parser)]
 #[command(name = "mangleframes-viewer")]
@@ -43,7 +43,20 @@ async fn main() -> anyhow::Result<()> {
 
     info!("Connecting to Python server at {:?}", args.socket);
     let client = Arc::new(SocketClient::new(&args.socket));
-    let state = AppState::new(client);
+    let state = AppState::new(client.clone());
+
+    // Preload first frame into cache for instant display
+    if let Ok(frames) = client.list_frames() {
+        if let Some(first) = frames.first() {
+            info!("Preloading frame: {}", first);
+            if let Ok(data) = client.get_frame(first, 1000) {
+                if let Ok(batches) = arrow_reader::parse_arrow_stream(&data) {
+                    let mut cache = state.cache.write().await;
+                    cache.insert(first.clone(), CachedFrame { batches, stats: None });
+                }
+            }
+        }
+    }
 
     if !args.no_browser {
         let url = format!("http://localhost:{}", args.port);

{mangleframes-0.1.1 → mangleframes-0.1.2}/viewer/static/app.js

@@ -297,9 +297,9 @@ function init() {
         state.offset = 0;
         state.sortCol = null;
 
-        await loadSchema(name);
-        await loadData(name);
-        loadStats(name);
+        // Fetch schema and data in parallel for faster display
+        await Promise.all([loadSchema(name), loadData(name)]);
+        loadStats(name);  // Fire-and-forget, updates UI when ready
     };
 
     $('refresh-btn').onclick = () => {