datapilot-kit 0.4.0__tar.gz → 0.4.1__tar.gz

{datapilot_kit-0.4.0/datapilot_kit.egg-info → datapilot_kit-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapilot-kit
-Version: 0.4.0
+Version: 0.4.1
 Summary: An open-source Python library for deterministic exploratory data analysis and dataset understanding.
 Author: Avishkar D. Gopale
 License-Expression: MIT

{datapilot_kit-0.4.0 → datapilot_kit-0.4.1}/datapilot/__init__.py

@@ -4,8 +4,9 @@ Datapilot
 An intelligent Python library for exploratory data analysis.
 """
 
-__version__ = "0.4.0"
+__version__ = "0.4.1"
 
 from .api import analyze
 
 __all__ = ["analyze"]
+

datapilot_kit-0.4.1/datapilot/analysis/consistency.py

@@ -0,0 +1,68 @@
+"""
+Deterministic consistency analysis for Datapilot v0.4.1.
+"""
+
+from collections import Counter
+
+import pandas as pd
+
+from .models import ConsistencySummary
+
+
+def generate_consistency_summary(
+    dataframe: pd.DataFrame,
+) -> ConsistencySummary:
+    """
+    Detect objectively observable type inconsistencies.
+
+    Missing values are excluded from type analysis because they are
+    assessed separately by Dataset Completeness.
+
+    Consistency degradation is based on the proportion of non-missing
+    values whose underlying type differs from the dominant type within
+    their column.
+    """
+
+    inconsistent_columns: list[str] = []
+    inconsistent_value_count = 0
+    total_non_missing = 0
+
+    for column in dataframe.columns:
+        series = dataframe[column].dropna()
+
+        if series.empty:
+            continue
+
+        type_counts = Counter(
+            type(value)
+            for value in series
+        )
+
+        column_total = len(series)
+        dominant_count = max(type_counts.values())
+        column_inconsistent_count = (
+            column_total - dominant_count
+        )
+
+        total_non_missing += column_total
+
+        if column_inconsistent_count > 0:
+            inconsistent_columns.append(str(column))
+            inconsistent_value_count += column_inconsistent_count
+
+    if total_non_missing == 0:
+        consistency_percentage = 0.0
+    else:
+        consistency_percentage = (
+            inconsistent_value_count
+            / total_non_missing
+        ) * 100.0
+
+    return ConsistencySummary(
+        inconsistent_columns=inconsistent_columns,
+        inconsistent_value_count=inconsistent_value_count,
+        consistency_percentage=round(
+            consistency_percentage,
+            2,
+        ),
+    )

{datapilot_kit-0.4.0 → datapilot_kit-0.4.1}/datapilot/analysis/health.py

@@ -5,9 +5,11 @@ Dataset health assessment for Datapilot v0.4.
 import pandas as pd
 
 from .datatype import generate_data_type_summary
+from .consistency import generate_consistency_summary
 from .duplicate import generate_duplicate_summary
 from .missing import generate_missing_value_summary
 from .models import DatasetHealth
+from .scoring import degradation_score
 
 
 def _grade_and_status(score: float) -> tuple[str, str]:
@@ -46,30 +48,23 @@ def generate_dataset_health(
     missing = generate_missing_value_summary(dataframe)
     duplicates = generate_duplicate_summary(dataframe)
     data_types = generate_data_type_summary(dataframe)
+    consistency = generate_consistency_summary(dataframe)
 
     # ------------------------------------------------------------------
     # Completeness
     # ------------------------------------------------------------------
 
-    completeness_score = max(
-        0.0,
-        min(
-            100.0,
-            100.0 - missing.missing_percentage,
-        ),
+    completeness_score = degradation_score(
+       missing.missing_percentage,
     )
 
     # ------------------------------------------------------------------
     # Duplicates
     # ------------------------------------------------------------------
 
-    duplicate_score = max(
-        0.0,
-        min(
-            100.0,
-            100.0 - duplicates.duplicate_percentage,
-        ),
-    )
+    duplicate_score = degradation_score(
+        duplicates.duplicate_percentage,
+   )
 
     # ------------------------------------------------------------------
     # Structure
@@ -140,16 +135,10 @@ def generate_dataset_health(
     # ------------------------------------------------------------------
     # Consistency
     # ------------------------------------------------------------------
-    #
-    # v0.4 currently has no objectively defined consistency metric
-    # implemented in the analysis layer.
-    #
-    # Therefore consistency is neutral until its metric specification
-    # is finalized. This prevents an unsupported penalty from being
-    # introduced into Dataset Health.
-    #
 
-    consistency_score = 100.0
+    consistency_score = degradation_score(
+        consistency.consistency_percentage,
+    )
 
     # ------------------------------------------------------------------
     # Overall score
@@ -187,6 +176,11 @@ def generate_dataset_health(
             "No duplicate rows detected."
         )
 
+    if not consistency.inconsistent_columns:
+        strengths.append(
+            "No mixed underlying value types detected."
+        )
+
     if recognized_columns == total_columns:
         strengths.append(
             "All columns have recognized data types."
@@ -223,6 +217,11 @@ def generate_dataset_health(
         0,
     )
 
+    if consistency.inconsistent_columns:
+        weaknesses.append(
+            f"{len(consistency.inconsistent_columns)} columns contain mixed underlying value types."
+        )
+
     if unrecognized_columns > 0:
         weaknesses.append(
             f"{unrecognized_columns} columns have unsupported "
@@ -260,6 +259,11 @@ def generate_dataset_health(
             "Review unsupported column data types."
         )
 
+    if consistency.inconsistent_columns:
+        recommendations.append(
+            "Review columns containing mixed underlying value types."
+        )
+
     if not recommendations:
         recommendations.append(
             "No major structural data-quality issues detected."
@@ -288,4 +292,4 @@ def generate_dataset_health(
         strengths=strengths,
         weaknesses=weaknesses,
         recommendations=recommendations,
-    )
+    )

{datapilot_kit-0.4.0 → datapilot_kit-0.4.1}/datapilot/analysis/models.py

@@ -162,4 +162,13 @@ class InsightSummary:
     """
 
     insights: list[str]
-    recommendations: list[str]
+    recommendations: list[str]
+@dataclass(slots=True)
+class ConsistencySummary:
+    """
+    Summary of objectively detectable consistency signals.
+    """
+
+    inconsistent_columns: list[str]
+    inconsistent_value_count: int
+    consistency_percentage: float

datapilot_kit-0.4.1/datapilot/analysis/scoring.py

@@ -0,0 +1,51 @@
+"""
+Deterministic scoring utilities for Datapilot.
+"""
+
+import math
+
+
+DEFAULT_DEGRADATION_EXPONENT = 1.2
+
+
+def degradation_score(
+    degradation_percentage: float,
+    exponent: float = DEFAULT_DEGRADATION_EXPONENT,
+) -> float:
+    """
+    Convert a degradation percentage into a continuous 0-100 score.
+
+    The score:
+    - is 100 when degradation is 0%
+    - reaches 0 when degradation is 100%
+    - decreases monotonically as degradation increases
+    - remains bounded between 0 and 100
+    """
+
+    if not math.isfinite(degradation_percentage):
+        raise ValueError(
+            "degradation_percentage must be finite."
+        )
+
+    if not math.isfinite(exponent) or exponent <= 0:
+        raise ValueError(
+            "exponent must be greater than 0."
+        )
+
+    degradation = max(
+        0.0,
+        min(
+            100.0,
+            degradation_percentage,
+        ),
+    ) / 100.0
+
+    score = (
+        100.0
+        * ((1.0 - degradation) ** exponent)
+    )
+
+    return round(
+        max(0.0, min(100.0, score)),
+        2,
+    )

{datapilot_kit-0.4.0 → datapilot_kit-0.4.1/datapilot_kit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapilot-kit
-Version: 0.4.0
+Version: 0.4.1
 Summary: An open-source Python library for deterministic exploratory data analysis and dataset understanding.
 Author: Avishkar D. Gopale
 License-Expression: MIT

{datapilot_kit-0.4.0 → datapilot_kit-0.4.1}/datapilot_kit.egg-info/SOURCES.txt

@@ -5,6 +5,7 @@ SPECIFICATION.md
 pyproject.toml
 datapilot/__init__.py
 datapilot/analysis/__init__.py
+datapilot/analysis/consistency.py
 datapilot/analysis/correlation.py
 datapilot/analysis/data_integrity.py
 datapilot/analysis/datatype.py
@@ -16,6 +17,7 @@ datapilot/analysis/ml_readiness.py
 datapilot/analysis/models.py
 datapilot/analysis/notebook_readiness.py
 datapilot/analysis/outliers.py
+datapilot/analysis/scoring.py
 datapilot/analysis/statistical_profile.py
 datapilot/analysis/statistics.py
 datapilot/analysis/summary.py

{datapilot_kit-0.4.0 → datapilot_kit-0.4.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datapilot-kit"
-version = "0.4.0"
+version = "0.4.1"
 description = "An open-source Python library for deterministic exploratory data analysis and dataset understanding."
 readme = "README.md"
 requires-python = ">=3.10"