mangleframes 0.1.6__tar.gz → 0.1.7__tar.gz

{mangleframes-0.1.6 → mangleframes-0.1.7}/Cargo.lock

@@ -604,7 +604,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2"
 dependencies = [
  "iana-time-zone",
+ "js-sys",
  "num-traits",
+ "wasm-bindgen",
  "windows-link",
 ]
 
@@ -1955,6 +1957,7 @@ dependencies = [
  "arrow-ipc",
  "arrow-json",
  "axum",
+ "chrono",
  "clap",
  "datafusion",
  "futures-util",

{mangleframes-0.1.6 → mangleframes-0.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mangleframes
-Version: 0.1.6
+Version: 0.1.7
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Rust
 Classifier: License :: OSI Approved :: MIT License
@@ -8,6 +8,7 @@ Requires-Dist: databricks-connect>=16.1.7
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: maturin>=1.11.2
 Requires-Dist: pyarrow>=11.0.0
+Requires-Dist: python-dateutil>=2.8.0
 Requires-Dist: pytest>=7.0 ; extra == 'dev'
 Requires-Dist: maturin>=1.4 ; extra == 'dev'
 Requires-Dist: pyspark>=3.4.0 ; extra == 'spark'

{mangleframes-0.1.6 → mangleframes-0.1.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "mangleframes"
-version = "0.1.6"
+version = "0.1.7"
 description = "PySpark DataFrame viewer with modern web UI"
 requires-python = ">=3.11"
 license = { text = "MIT" }
@@ -18,6 +18,7 @@ dependencies = [
     "loguru>=0.7.3",
     "maturin>=1.11.2",
     "pyarrow>=11.0.0",
+    "python-dateutil>=2.8.0",
 ]
 
 [project.optional-dependencies]

{mangleframes-0.1.6 → mangleframes-0.1.7}/python/mangleframes/__init__.py

@@ -16,7 +16,7 @@ from .server import DataFrameServer
 if TYPE_CHECKING:
     from pyspark.sql import DataFrame
 
-__version__ = "0.1.6"
+__version__ = "0.1.7"
 __all__ = ["register", "unregister", "show", "cleanup"]
 
 _registry: dict[str, DataFrame] = {}

{mangleframes-0.1.6 → mangleframes-0.1.7}/python/mangleframes/protocol.py

@@ -298,6 +298,172 @@ def handle_join_temporal(
     })
 
 
+def _generate_expected_buckets(min_date: str, max_date: str, granularity: str) -> list[str]:
+    """Generate all expected bucket keys between min and max dates."""
+    from datetime import datetime, timedelta
+    from dateutil.relativedelta import relativedelta
+
+    start = datetime.strptime(min_date[:10], "%Y-%m-%d")
+    end = datetime.strptime(max_date[:10], "%Y-%m-%d")
+    buckets = []
+
+    if granularity == "day":
+        current = start
+        while current <= end:
+            buckets.append(current.strftime("%Y-%m-%d"))
+            current += timedelta(days=1)
+    elif granularity == "week":
+        current = start - timedelta(days=start.weekday())
+        end_aligned = end - timedelta(days=end.weekday())
+        while current <= end_aligned:
+            buckets.append(current.strftime("%Y-%m-%d"))
+            current += timedelta(weeks=1)
+    elif granularity == "month":
+        current = start.replace(day=1)
+        end_aligned = end.replace(day=1)
+        while current <= end_aligned:
+            buckets.append(current.strftime("%Y-%m-%d"))
+            current += relativedelta(months=1)
+
+    return buckets
+
+
+def _find_gaps(expected: list[str], actual: set[str]) -> list[dict]:
+    """Find contiguous ranges of missing buckets."""
+    gaps = []
+    gap_start = None
+    gap_count = 0
+
+    for bucket in expected:
+        if bucket not in actual:
+            if gap_start is None:
+                gap_start = bucket
+                gap_count = 1
+            else:
+                gap_count += 1
+        else:
+            if gap_start is not None:
+                gaps.append({
+                    "start": gap_start,
+                    "end": expected[expected.index(gap_start) + gap_count - 1],
+                    "periods": gap_count
+                })
+                gap_start = None
+                gap_count = 0
+
+    if gap_start is not None:
+        gaps.append({
+            "start": gap_start,
+            "end": expected[expected.index(gap_start) + gap_count - 1],
+            "periods": gap_count
+        })
+
+    return gaps
+
+
+def handle_temporal_range(
+    registry: dict[str, DataFrame], name: str, column: str, granularity: str
+) -> bytes:
+    """Return detailed temporal range statistics for a frame."""
+    if name not in registry:
+        return encode_error(f"DataFrame '{name}' not found")
+
+    df = registry[name]
+    if column not in [f.name for f in df.schema.fields]:
+        return encode_error(f"Column '{column}' not found in '{name}'")
+
+    field = next(f for f in df.schema.fields if f.name == column)
+    if not _is_temporal_type(str(field.dataType)):
+        return encode_error(f"Column '{column}' is not a temporal type")
+
+    if granularity not in ("day", "week", "month"):
+        return encode_error(f"Invalid granularity: {granularity}")
+
+    start_time = time.perf_counter()
+
+    agg_result = df.agg(
+        F.min(column).alias("min_date"),
+        F.max(column).alias("max_date"),
+        F.count(F.lit(1)).alias("total_rows"),
+        F.sum(F.when(F.col(column).isNull(), 1).otherwise(0)).alias("null_dates"),
+        F.countDistinct(column).alias("distinct_dates")
+    ).collect()[0]
+
+    min_date = agg_result["min_date"]
+    max_date = agg_result["max_date"]
+    total_rows = agg_result["total_rows"]
+    null_dates = agg_result["null_dates"] or 0
+    distinct_dates = agg_result["distinct_dates"]
+
+    internal_gaps = []
+    if min_date and max_date:
+        truncated = df.withColumn("__bucket", F.date_trunc(granularity, F.col(column)))
+        bucket_rows = truncated.select("__bucket").distinct().collect()
+        actual_buckets = {
+            str(row["__bucket"])[:10] for row in bucket_rows if row["__bucket"]
+        }
+        expected = _generate_expected_buckets(str(min_date), str(max_date), granularity)
+        internal_gaps = _find_gaps(expected, actual_buckets)
+
+    compute_ms = int((time.perf_counter() - start_time) * 1000)
+
+    return encode_json_response({
+        "frame": name,
+        "column": column,
+        "granularity": granularity,
+        "min_date": str(min_date)[:10] if min_date else None,
+        "max_date": str(max_date)[:10] if max_date else None,
+        "total_rows": total_rows,
+        "null_dates": null_dates,
+        "distinct_dates": distinct_dates,
+        "internal_gaps": internal_gaps,
+        "compute_ms": compute_ms
+    })
+
+
+def handle_temporal_loss(
+    registry: dict[str, DataFrame],
+    name: str, column: str,
+    overlap_start: str, overlap_end: str
+) -> bytes:
+    """Return row counts outside the overlap zone."""
+    if name not in registry:
+        return encode_error(f"DataFrame '{name}' not found")
+
+    df = registry[name]
+    if column not in [f.name for f in df.schema.fields]:
+        return encode_error(f"Column '{column}' not found in '{name}'")
+
+    field = next(f for f in df.schema.fields if f.name == column)
+    if not _is_temporal_type(str(field.dataType)):
+        return encode_error(f"Column '{column}' is not a temporal type")
+
+    start_time = time.perf_counter()
+
+    result = df.agg(
+        F.count(F.lit(1)).alias("total"),
+        F.sum(F.when(F.col(column) < overlap_start, 1).otherwise(0)).alias("before"),
+        F.sum(F.when(F.col(column) > overlap_end, 1).otherwise(0)).alias("after")
+    ).collect()[0]
+
+    total = result["total"]
+    rows_before = result["before"] or 0
+    rows_after = result["after"] or 0
+    total_lost = rows_before + rows_after
+    pct_lost = (total_lost / total * 100) if total > 0 else 0.0
+
+    compute_ms = int((time.perf_counter() - start_time) * 1000)
+
+    return encode_json_response({
+        "frame": name,
+        "rows_before": rows_before,
+        "rows_after": rows_after,
+        "total_lost": total_lost,
+        "pct_lost": round(pct_lost, 2),
+        "compute_ms": compute_ms
+    })
+
+
 def handle_join_overlap(
     registry: dict[str, DataFrame],
     frame1: str, frame2: str,
@@ -418,4 +584,18 @@ def dispatch_command(
             return encode_error("At least one column required per frame")
         return handle_join_overlap(registry, frame1, frame2, cols1, cols2)
 
+    if command.startswith("TEMPORAL_RANGE:"):
+        parts = command[15:].split(":")
+        if len(parts) != 3:
+            return encode_error("Invalid TEMPORAL_RANGE format. Use TEMPORAL_RANGE:name:column:granularity")
+        name, column, granularity = parts
+        return handle_temporal_range(registry, name, column, granularity)
+
+    if command.startswith("TEMPORAL_LOSS:"):
+        parts = command[14:].split(":")
+        if len(parts) != 4:
+            return encode_error("Invalid TEMPORAL_LOSS format. Use TEMPORAL_LOSS:name:column:start:end")
+        name, column, overlap_start, overlap_end = parts
+        return handle_temporal_loss(registry, name, column, overlap_start, overlap_end)
+
     return encode_error(f"Unknown command: {command}")

{mangleframes-0.1.6 → mangleframes-0.1.7}/viewer/Cargo.toml

@@ -27,6 +27,7 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 thiserror = "2"
 anyhow = "1"
 futures-util = "0.3"
+chrono = "0.4"
 
 [profile.release]
 lto = true

{mangleframes-0.1.6 → mangleframes-0.1.7}/viewer/src/history_analysis.rs

@@ -61,6 +61,46 @@ pub struct JoinPrediction {
     pub coverage_pct: f64,
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DateGap {
+    pub start: String,
+    pub end: String,
+    pub periods: usize,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct TemporalRangeStats {
+    pub frame: String,
+    pub column: String,
+    pub granularity: String,
+    pub min_date: Option<String>,
+    pub max_date: Option<String>,
+    pub total_rows: usize,
+    pub null_dates: usize,
+    pub distinct_dates: usize,
+    pub internal_gaps: Vec<DateGap>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct OverlapZone {
+    pub start: String,
+    pub end: String,
+    pub span: String,
+    pub days: i64,
+    pub valid: bool,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct FrameDataLoss {
+    pub frame: String,
+    pub rows_before_overlap: usize,
+    pub rows_after_overlap: usize,
+    pub total_lost: usize,
+    pub pct_lost: f64,
+    pub range_lost_before: Option<String>,
+    pub range_lost_after: Option<String>,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct CoverageResult {
     pub frames: Vec<String>,
@@ -69,12 +109,17 @@ pub struct CoverageResult {
     pub pairwise_overlaps: Vec<PairwiseOverlap>,
     pub timeline: Vec<TimelineBucket>,
     pub predictions: Vec<JoinPrediction>,
+    pub temporal_ranges: Vec<TemporalRangeStats>,
+    pub overlap_zone: Option<OverlapZone>,
+    pub data_loss: Vec<FrameDataLoss>,
 }
 
 pub struct HistoryAnalyzer {
     key_stats: HashMap<String, FrameKeyStats>,
     temporal_stats: HashMap<String, FrameTemporalStats>,
     overlaps: Vec<PairwiseOverlap>,
+    temporal_ranges: HashMap<String, TemporalRangeStats>,
+    data_loss: HashMap<String, FrameDataLoss>,
 }
 
 impl HistoryAnalyzer {
@@ -83,6 +128,8 @@ impl HistoryAnalyzer {
             key_stats: HashMap::new(),
             temporal_stats: HashMap::new(),
             overlaps: Vec::new(),
+            temporal_ranges: HashMap::new(),
+            data_loss: HashMap::new(),
         }
     }
 
@@ -98,6 +145,14 @@ impl HistoryAnalyzer {
         self.overlaps.push(overlap);
     }
 
+    pub fn add_temporal_range(&mut self, stats: TemporalRangeStats) {
+        self.temporal_ranges.insert(stats.frame.clone(), stats);
+    }
+
+    pub fn add_data_loss(&mut self, loss: FrameDataLoss) {
+        self.data_loss.insert(loss.frame.clone(), loss);
+    }
+
     pub fn compute_coverage(&self, frames: &[String]) -> Result<CoverageResult, HistoryError> {
         if frames.is_empty() {
             return Err(HistoryError::NoFrames);
@@ -123,6 +178,18 @@ impl HistoryAnalyzer {
         let timeline = self.build_timeline(frames);
         let predictions = self.compute_predictions(frames, &pairwise_overlaps);
 
+        let temporal_ranges: Vec<TemporalRangeStats> = frames
+            .iter()
+            .filter_map(|f| self.temporal_ranges.get(f).cloned())
+            .collect();
+
+        let overlap_zone = self.compute_overlap_zone(&temporal_ranges);
+
+        let data_loss: Vec<FrameDataLoss> = frames
+            .iter()
+            .filter_map(|f| self.data_loss.get(f).cloned())
+            .collect();
+
         Ok(CoverageResult {
             frames: frames.to_vec(),
             key_stats,
@@ -130,9 +197,87 @@ impl HistoryAnalyzer {
             pairwise_overlaps,
             timeline,
             predictions,
+            temporal_ranges,
+            overlap_zone,
+            data_loss,
         })
     }
 
+    fn compute_overlap_zone(&self, ranges: &[TemporalRangeStats]) -> Option<OverlapZone> {
+        if ranges.is_empty() {
+            return None;
+        }
+
+        let min_dates: Vec<&str> = ranges
+            .iter()
+            .filter_map(|r| r.min_date.as_deref())
+            .collect();
+
+        let max_dates: Vec<&str> = ranges
+            .iter()
+            .filter_map(|r| r.max_date.as_deref())
+            .collect();
+
+        if min_dates.is_empty() || max_dates.is_empty() {
+            return None;
+        }
+
+        let overlap_start = min_dates.iter().max()?.to_string();
+        let overlap_end = max_dates.iter().min()?.to_string();
+
+        if overlap_start > overlap_end {
+            return Some(OverlapZone {
+                start: overlap_start,
+                end: overlap_end,
+                span: "No overlap".to_string(),
+                days: 0,
+                valid: false,
+            });
+        }
+
+        let days = Self::compute_days_between(&overlap_start, &overlap_end);
+        let span = Self::format_date_span(days);
+
+        Some(OverlapZone {
+            start: overlap_start,
+            end: overlap_end,
+            span,
+            days,
+            valid: true,
+        })
+    }
+
+    fn compute_days_between(start: &str, end: &str) -> i64 {
+        use chrono::NaiveDate;
+        let start_date = NaiveDate::parse_from_str(start, "%Y-%m-%d").ok();
+        let end_date = NaiveDate::parse_from_str(end, "%Y-%m-%d").ok();
+        match (start_date, end_date) {
+            (Some(s), Some(e)) => (e - s).num_days(),
+            _ => 0,
+        }
+    }
+
+    fn format_date_span(days: i64) -> String {
+        if days < 0 {
+            return "No overlap".to_string();
+        }
+        let years = days / 365;
+        let months = (days % 365) / 30;
+        let remaining_days = days % 30;
+
+        let mut parts = Vec::new();
+        if years > 0 {
+            parts.push(format!("{}y", years));
+        }
+        if months > 0 {
+            parts.push(format!("{}m", months));
+        }
+        if remaining_days > 0 || parts.is_empty() {
+            parts.push(format!("{}d", remaining_days));
+        }
+        parts.join(" ")
+    }
+
     fn build_timeline(&self, frames: &[String]) -> Vec<TimelineBucket> {
         let mut all_buckets: HashMap<String, HashMap<String, usize>> = HashMap::new();