mangleframes 0.3.0__tar.gz → 0.3.1__tar.gz

{mangleframes-0.3.0 → mangleframes-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mangleframes
-Version: 0.3.0
+Version: 0.3.1
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Rust
 Classifier: License :: OSI Approved :: MIT License

{mangleframes-0.3.0 → mangleframes-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "mangleframes"
-version = "0.3.0"
+version = "0.3.1"
 description = "PySpark DataFrame viewer with modern web UI"
 requires-python = ">=3.12"
 license = { text = "MIT" }

{mangleframes-0.3.0 → mangleframes-0.3.1}/python/mangleframes/__init__.py

@@ -32,7 +32,7 @@ from .session import SparkSession, get_proxy_port, get_spark_session
 if TYPE_CHECKING:
     from pyspark.sql import DataFrame
 
-__version__ = "0.3.0"
+__version__ = "0.3.1"
 
 # Import alert classes for convenience (optional dependency)
 try:

{mangleframes-0.3.0 → mangleframes-0.3.1}/spark-connect/Cargo.toml

@@ -6,7 +6,7 @@ description = "Rust client for Apache Spark Connect gRPC protocol"
 
 [dependencies]
 tonic = { version = "0.12", features = ["tls", "tls-roots", "gzip"] }
-prost = "0.13"
+prost = { version = "0.13", features = ["no-recursion-limit"] }
 prost-types = "0.13"
 tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
 arrow = { version = "54", features = ["ipc"] }

mangleframes-0.3.1/spark-connect/src/lib.rs

@@ -0,0 +1,67 @@
+//! Spark Connect Rust client for Apache Spark gRPC protocol.
+//!
+//! This crate provides a Rust client for connecting to Spark clusters via the
+//! Spark Connect gRPC protocol, with specific support for Databricks.
+
+pub mod client;
+pub mod error;
+
+#[allow(clippy::all)]
+mod proto {
+    include!("proto/spark.connect.rs");
+}
+
+pub use client::{ColumnSchema, SparkConnectClient};
+pub use error::SparkConnectError;
+
+#[cfg(test)]
+mod tests {
+    use super::proto::{relation, Limit, Relation};
+    use prost::Message;
+
+    /// Build a deeply nested Relation with `depth` levels of Limit operations.
+    fn build_nested_relation(depth: usize) -> Relation {
+        let mut current = Relation {
+            common: None,
+            rel_type: None,
+        };
+
+        for i in 0..depth {
+            current = Relation {
+                common: None,
+                rel_type: Some(relation::RelType::Limit(Box::new(Limit {
+                    input: Some(Box::new(current)),
+                    limit: i as i32,
+                }))),
+            };
+        }
+
+        current
+    }
+
+    /// Verifies the no-recursion-limit feature works by encoding/decoding
+    /// a deeply nested protobuf message (150+ levels exceeds old 100 limit).
+    #[test]
+    fn test_deeply_nested_relation_decodes() {
+        let depth = 150;
+        let relation = build_nested_relation(depth);
+
+        let encoded = relation.encode_to_vec();
+        let decoded = Relation::decode(encoded.as_slice())
+            .expect("should decode deeply nested relation without recursion limit");
+
+        // Verify the structure by walking down to confirm depth
+        let mut count = 0;
+        let mut current = &decoded;
+        while let Some(relation::RelType::Limit(limit)) = &current.rel_type {
+            count += 1;
+            if let Some(input) = &limit.input {
+                current = input;
+            } else {
+                break;
+            }
+        }
+
+        assert_eq!(count, depth, "decoded relation should have correct nesting depth");
+    }
+}

mangleframes-0.3.1/viewer/frontend/src/lib/api.ts

@@ -0,0 +1,174 @@
+const BASE_URL = ''
+
+export interface Column {
+  name: string
+  type: string
+  nullable: boolean
+}
+
+export interface Timing {
+  spark_ms: number
+  ipc_ms: number
+  json_ms: number
+  total_ms: number
+  cached: boolean
+}
+
+export interface DataResponse {
+  rows: Record<string, unknown>[]
+  total: number
+  offset: number
+  timing: Timing
+}
+
+export interface SchemaResponse {
+  columns: Column[]
+}
+
+export interface FramesResponse {
+  frames: string[]
+}
+
+export interface StatsResponse {
+  row_count: number
+  columns: ColumnStats[]
+}
+
+export interface ColumnStats {
+  name: string
+  type: string
+  null_count: number
+  distinct_count?: number
+  min?: string | number
+  max?: string | number
+  avg?: number
+}
+
+export interface QueryResponse {
+  rows: Record<string, unknown>[]
+  total: number
+  timing: { execution_ms: number }
+}
+
+export interface CsvUploadResponse {
+  frame_name: string
+  columns: { name: string; data_type: string; nullable: boolean }[]
+  row_count: number
+}
+
+export interface CsvFramesResponse {
+  frames: string[]
+}
+
+export interface ReconcileStatistics {
+  match_rate: number
+  matched_groups: number
+  source_groups: number
+  source_only_groups: number
+  target_groups: number
+  target_only_groups: number
+}
+
+export interface ColumnTotal {
+  column: string
+  aggregation: string
+  source_total: number
+  target_total: number
+  difference: number
+  percent_diff: number
+}
+
+export interface ReconcileResponse {
+  statistics: ReconcileStatistics
+  column_totals: ColumnTotal[]
+  source_only: { total: number; rows: Record<string, unknown>[] }
+  target_only: { total: number; rows: Record<string, unknown>[] }
+  matched_rows: { rows: Record<string, unknown>[] }
+  source_frame: string
+  target_frame: string
+}
+
+export interface ReconcileRequest {
+  source_frame: string
+  target_frame: string
+  source_type?: string
+  source_group_by: string[]
+  target_group_by: string[]
+  source_join_keys: string[]
+  target_join_keys: string[]
+  join_type: 'inner' | 'left' | 'right' | 'full'
+  aggregations: { column: string; aggregations: string[] }[]
+  sample_limit?: number
+}
+
+async function request<T>(path: string, options?: RequestInit): Promise<T> {
+  const response = await fetch(`${BASE_URL}${path}`, {
+    headers: { 'Content-Type': 'application/json' },
+    ...options,
+  })
+
+  if (!response.ok) {
+    const error = await response.json().catch(() => ({ error: response.statusText }))
+    throw new Error(error.error || `Request failed: ${response.status}`)
+  }
+
+  return response.json()
+}
+
+export const api = {
+  listFrames: () => request<FramesResponse>('/api/frames'),
+
+  getSchema: (name: string) =>
+    request<SchemaResponse>(`/api/frames/${encodeURIComponent(name)}/schema`),
+
+  getData: (name: string, offset = 0, limit = 100) =>
+    request<DataResponse>(
+      `/api/frames/${encodeURIComponent(name)}/data?offset=${offset}&limit=${limit}`
+    ),
+
+  getStats: (name: string) =>
+    request<StatsResponse>(`/api/frames/${encodeURIComponent(name)}/stats`),
+
+  executeQuery: (sql: string) =>
+    request<QueryResponse>('/api/query', {
+      method: 'POST',
+      body: JSON.stringify({ sql }),
+    }),
+
+  exportFrame: (name: string, format: 'csv' | 'json' | 'parquet') =>
+    fetch(`${BASE_URL}/api/frames/${encodeURIComponent(name)}/export`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ format }),
+    }),
+
+  uploadCsv: async (file: File, frameName: string): Promise<CsvUploadResponse> => {
+    const formData = new FormData()
+    formData.append('file', file)
+    formData.append('frame_name', frameName)
+    const response = await fetch(`${BASE_URL}/api/reconcile/upload`, {
+      method: 'POST',
+      body: formData,
+    })
+    if (!response.ok) {
+      const error = await response.json().catch(() => ({ error: response.statusText }))
+      throw new Error(error.error || `Upload failed: ${response.status}`)
+    }
+    return response.json()
+  },
+
+  listCsvFrames: () => request<CsvFramesResponse>('/api/reconcile/frames'),
+
+  reconcile: (req: ReconcileRequest) =>
+    request<ReconcileResponse>('/api/reconcile', {
+      method: 'POST',
+      body: JSON.stringify(req),
+    }),
+
+  exportReconciliation: (req: ReconcileRequest) =>
+    fetch(`${BASE_URL}/api/reconcile/export`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(req),
+    }),
+}

mangleframes-0.3.0/spark-connect/src/lib.rs

@@ -1,15 +0,0 @@
-//! Spark Connect Rust client for Apache Spark gRPC protocol.
-//!
-//! This crate provides a Rust client for connecting to Spark clusters via the
-//! Spark Connect gRPC protocol, with specific support for Databricks.
-
-pub mod client;
-pub mod error;
-
-#[allow(clippy::all)]
-mod proto {
-    include!("proto/spark.connect.rs");
-}
-
-pub use client::{ColumnSchema, SparkConnectClient};
-pub use error::SparkConnectError;