mangleframes 0.2.9__tar.gz → 0.3.0__tar.gz

{mangleframes-0.2.9 → mangleframes-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mangleframes
-Version: 0.2.9
+Version: 0.3.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Rust
 Classifier: License :: OSI Approved :: MIT License

{mangleframes-0.2.9 → mangleframes-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "mangleframes"
-version = "0.2.9"
+version = "0.3.0"
 description = "PySpark DataFrame viewer with modern web UI"
 requires-python = ">=3.12"
 license = { text = "MIT" }

{mangleframes-0.2.9 → mangleframes-0.3.0}/python/mangleframes/__init__.py

@@ -32,7 +32,7 @@ from .session import SparkSession, get_proxy_port, get_spark_session
 if TYPE_CHECKING:
     from pyspark.sql import DataFrame
 
-__version__ = "0.2.9"
+__version__ = "0.3.0"
 
 # Import alert classes for convenience (optional dependency)
 try:

{mangleframes-0.2.9 → mangleframes-0.3.0}/spark-connect/src/client.rs

@@ -5,6 +5,7 @@ use std::time::Instant;
 
 use arrow::record_batch::RecordBatch;
 use arrow_ipc::reader::StreamReader;
+use arrow_ipc::writer::StreamWriter;
 use tonic::metadata::MetadataValue;
 use tonic::service::Interceptor;
 use tonic::transport::{Channel, ClientTlsConfig};
@@ -364,6 +365,102 @@ impl SparkConnectClient {
         info!("Found {} tables", tables.len());
         Ok(tables)
     }
+
+    /// Register Arrow record batches as a temporary view in Spark.
+    pub async fn create_temp_view(
+        &self,
+        view_name: &str,
+        batches: &[RecordBatch],
+    ) -> Result<(), SparkConnectError> {
+        if batches.is_empty() {
+            return Err(SparkConnectError::Config("No batches to register".to_string()));
+        }
+
+        info!("Registering temp view '{}' with {} batches", view_name, batches.len());
+
+        // Serialize batches to Arrow IPC streaming format
+        let ipc_data = serialize_batches_to_ipc(batches)?;
+        info!("Serialized {} bytes of Arrow IPC data", ipc_data.len());
+
+        // Create LocalRelation with the IPC data
+        let local_relation = Relation {
+            common: None,
+            rel_type: Some(crate::proto::relation::RelType::LocalRelation(
+                crate::proto::LocalRelation {
+                    data: Some(ipc_data),
+                    schema: None,
+                },
+            )),
+        };
+
+        // Create the temp view command
+        let command = crate::proto::Command {
+            command_type: Some(crate::proto::command::CommandType::CreateDataframeView(
+                crate::proto::CreateDataFrameViewCommand {
+                    input: Some(local_relation),
+                    name: view_name.to_string(),
+                    is_global: false,
+                    replace: true,
+                },
+            )),
+        };
+
+        self.execute_command(command).await?;
+        info!("Successfully registered temp view '{}'", view_name);
+        Ok(())
+    }
+
+    /// Execute a Spark Connect command (non-query operation).
+    async fn execute_command(
+        &self,
+        command: crate::proto::Command,
+    ) -> Result<(), SparkConnectError> {
+        let plan = Plan {
+            op_type: Some(crate::proto::plan::OpType::Command(command)),
+        };
+
+        let request = ExecutePlanRequest {
+            session_id: self.session_id.clone(),
+            user_context: Some(UserContext {
+                user_id: "spark-connect-rs".to_string(),
+                user_name: "spark-connect-rs".to_string(),
+                extensions: vec![],
+            }),
+            operation_id: Some(Uuid::new_v4().to_string()),
+            plan: Some(plan),
+            client_type: Some("spark-connect-rs".to_string()),
+            request_options: vec![],
+            tags: vec![],
+            client_observed_server_side_session_id: None,
+        };
+
+        let response = match &self.inner {
+            ClientInner::Direct(c) => c.clone().execute_plan(request).await?,
+            ClientInner::Proxy(c) => c.clone().execute_plan(request).await?,
+        };
+
+        // Drain the response stream
+        let mut stream = response.into_inner();
+        while stream.message().await?.is_some() {}
+
+        Ok(())
+    }
+}
+
+/// Serialize Arrow RecordBatches to IPC streaming format.
+fn serialize_batches_to_ipc(batches: &[RecordBatch]) -> Result<Vec<u8>, SparkConnectError> {
+    let schema = batches[0].schema();
+    let mut ipc_data = Vec::new();
+
+    {
+        let mut writer = StreamWriter::try_new(&mut ipc_data, &schema)?;
+        for batch in batches {
+            writer.write(batch)?;
+        }
+        writer.finish()?;
+    }
+
+    Ok(ipc_data)
 }
 
 /// Extract a JSON value from an Arrow array at a specific index.

{mangleframes-0.2.9 → mangleframes-0.3.0}/viewer/src/reconcile_handlers.rs

@@ -23,6 +23,14 @@ use crate::web_server::{AppState, CachedFrame};
 
 const CSV_FRAME_PREFIX: &str = "__csv__";
 
+/// Sanitize a name for use as a Spark temp view name.
+/// Replaces non-alphanumeric characters (except underscores) with underscores.
+fn sanitize_name(name: &str) -> String {
+    name.chars()
+        .map(|c| if c.is_alphanumeric() || c == '_' { c } else { '_' })
+        .collect()
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(rename_all = "snake_case")]
 pub enum JoinType {
@@ -229,13 +237,44 @@ pub async fn reconcile(
         }
     };
 
+    // Check if source is a CSV frame and register it as a temp view in Spark
+    let source_frame_name = {
+        let csv_cache_key = format!("{}{}", CSV_FRAME_PREFIX, req.source_frame);
+        let cache = state.cache.read().await;
+
+        if let Some(cached) = cache.get(&csv_cache_key) {
+            let view_name = format!("__mf_csv_{}", sanitize_name(&req.source_frame));
+            tracing::info!(
+                "Registering CSV '{}' as temp view '{}' ({} batches)",
+                req.source_frame,
+                view_name,
+                cached.batches.len()
+            );
+
+            // Drop the cache lock before async operation
+            let batches = cached.batches.clone();
+            drop(cache);
+
+            if let Err(e) = dbx.create_temp_view(&view_name, &batches).await {
+                return error_response(
+                    StatusCode::INTERNAL_SERVER_ERROR,
+                    &format!("Failed to register CSV as temp view: {}", e),
+                );
+            }
+            view_name
+        } else {
+            drop(cache);
+            req.source_frame.clone()
+        }
+    };
+
     let config = build_reconcile_config(&req);
     let sample_limit = req.sample_limit.unwrap_or(100);
 
-    tracing::info!("Executing reconciliation: {} vs {}", req.source_frame, req.target_frame);
+    tracing::info!("Executing reconciliation: {} vs {}", source_frame_name, req.target_frame);
 
-    // Execute stats query
-    let stats = match execute_stats_query(dbx, &req.source_frame, &req.target_frame, &config).await
+    // Execute stats query (use source_frame_name for CSV temp view)
+    let stats = match execute_stats_query(dbx, &source_frame_name, &req.target_frame, &config).await
     {
         Ok(s) => s,
         Err(e) => return error_response(StatusCode::INTERNAL_SERVER_ERROR, &e),
@@ -243,7 +282,7 @@ pub async fn reconcile(
 
     // Execute source-only query
     let source_only_rows =
-        match execute_source_only_query(dbx, &req.source_frame, &req.target_frame, &config, sample_limit).await
+        match execute_source_only_query(dbx, &source_frame_name, &req.target_frame, &config, sample_limit).await
         {
             Ok(rows) => rows,
             Err(e) => return error_response(StatusCode::INTERNAL_SERVER_ERROR, &e),
@@ -251,7 +290,7 @@ pub async fn reconcile(
 
     // Execute target-only query
     let target_only_rows =
-        match execute_target_only_query(dbx, &req.source_frame, &req.target_frame, &config, sample_limit).await
+        match execute_target_only_query(dbx, &source_frame_name, &req.target_frame, &config, sample_limit).await
         {
             Ok(rows) => rows,
             Err(e) => return error_response(StatusCode::INTERNAL_SERVER_ERROR, &e),
@@ -259,7 +298,7 @@ pub async fn reconcile(
 
     // Execute matched rows query
     let matched_rows =
-        match execute_matched_query(dbx, &req.source_frame, &req.target_frame, &config, sample_limit).await
+        match execute_matched_query(dbx, &source_frame_name, &req.target_frame, &config, sample_limit).await
         {
             Ok(rows) => rows,
             Err(e) => return error_response(StatusCode::INTERNAL_SERVER_ERROR, &e),
@@ -267,7 +306,7 @@ pub async fn reconcile(
 
     // Execute totals query
     let column_totals =
-        match execute_totals_query(dbx, &req.source_frame, &req.target_frame, &config).await {
+        match execute_totals_query(dbx, &source_frame_name, &req.target_frame, &config).await {
             Ok(totals) => totals,
             Err(e) => {
                 tracing::warn!("Failed to get column totals: {}", e);
@@ -536,6 +575,35 @@ pub async fn export_reconciliation(
         }
     };
 
+    // Check if source is a CSV frame and register it as a temp view in Spark
+    let source_frame_name = {
+        let csv_cache_key = format!("{}{}", CSV_FRAME_PREFIX, req.source_frame);
+        let cache = state.cache.read().await;
+
+        if let Some(cached) = cache.get(&csv_cache_key) {
+            let view_name = format!("__mf_csv_{}", sanitize_name(&req.source_frame));
+            tracing::info!(
+                "Registering CSV '{}' as temp view '{}' for export",
+                req.source_frame,
+                view_name
+            );
+
+            let batches = cached.batches.clone();
+            drop(cache);
+
+            if let Err(e) = dbx.create_temp_view(&view_name, &batches).await {
+                return error_response(
+                    StatusCode::INTERNAL_SERVER_ERROR,
+                    &format!("Failed to register CSV as temp view: {}", e),
+                );
+            }
+            view_name
+        } else {
+            drop(cache);
+            req.source_frame.clone()
+        }
+    };
+
     let config = json!({
         "source_type": req.source_type,
         "source_group_by": req.source_group_by,
@@ -550,13 +618,13 @@ pub async fn export_reconciliation(
 
     tracing::info!(
         "Exporting reconciliation dashboard: {} vs {}",
-        req.source_frame,
+        source_frame_name,
         req.target_frame
     );
 
     // Execute stats query
     let stats =
-        match execute_stats_query(dbx, &req.source_frame, &req.target_frame, &config).await {
+        match execute_stats_query(dbx, &source_frame_name, &req.target_frame, &config).await {
             Ok(s) => s,
             Err(e) => return error_response(StatusCode::INTERNAL_SERVER_ERROR, &e),
         };
@@ -564,7 +632,7 @@ pub async fn export_reconciliation(
     // Execute source-only query
     let source_only_rows = match execute_source_only_query(
         dbx,
-        &req.source_frame,
+        &source_frame_name,
         &req.target_frame,
         &config,
         sample_limit,
@@ -578,7 +646,7 @@ pub async fn export_reconciliation(
     // Execute target-only query
     let target_only_rows = match execute_target_only_query(
         dbx,
-        &req.source_frame,
+        &source_frame_name,
         &req.target_frame,
         &config,
         sample_limit,
@@ -592,7 +660,7 @@ pub async fn export_reconciliation(
     // Execute matched rows query
     let matched_rows = match execute_matched_query(
         dbx,
-        &req.source_frame,
+        &source_frame_name,
         &req.target_frame,
         &config,
         sample_limit,
@@ -605,7 +673,7 @@ pub async fn export_reconciliation(
 
     // Execute totals query
     let column_totals =
-        match execute_totals_query(dbx, &req.source_frame, &req.target_frame, &config).await {
+        match execute_totals_query(dbx, &source_frame_name, &req.target_frame, &config).await {
             Ok(totals) => totals,
             Err(e) => {
                 tracing::warn!("Failed to get column totals: {}", e);
@@ -905,4 +973,31 @@ mod tests {
         let row = json!({"count": "not a number"});
         assert_eq!(extract_i64(&row, "count"), 0);
     }
+
+    // ============ sanitize_name tests ============
+
+    #[test]
+    fn test_sanitize_name_spaces_and_parens() {
+        assert_eq!(sanitize_name("orders (1)"), "orders__1_");
+    }
+
+    #[test]
+    fn test_sanitize_name_dots_and_hyphens() {
+        assert_eq!(sanitize_name("my-table.csv"), "my_table_csv");
+    }
+
+    #[test]
+    fn test_sanitize_name_already_valid() {
+        assert_eq!(sanitize_name("normal_name"), "normal_name");
+    }
+
+    #[test]
+    fn test_sanitize_name_alphanumeric() {
+        assert_eq!(sanitize_name("table123"), "table123");
+    }
+
+    #[test]
+    fn test_sanitize_name_special_chars() {
+        assert_eq!(sanitize_name("data@2024#test!"), "data_2024_test_");
+    }
 }

{mangleframes-0.2.9 → mangleframes-0.3.0}/viewer/src/spark_client.rs

@@ -102,6 +102,20 @@ impl DatabricksClient {
             execution_ms,
         })
     }
+
+    /// Register Arrow batches as a temporary view in Spark.
+    pub async fn create_temp_view(
+        &self,
+        view_name: &str,
+        batches: &[RecordBatch],
+    ) -> Result<(), SparkConnectError> {
+        let guard = self.client.read().await;
+        let client = guard
+            .as_ref()
+            .ok_or_else(|| SparkConnectError::Config("Not connected".to_string()))?;
+
+        client.create_temp_view(view_name, batches).await
+    }
 }
 
 impl Default for DatabricksClient {