noweda 0.1.0__py3-none-any.whl

noweda/__init__.py

@@ -0,0 +1,3 @@
+from .io import read
+from .accessor import NowEDAAccessor
+__all__ = ["read"]

noweda/accessor.py

@@ -0,0 +1,31 @@
+import pandas as pd
+from noweda.core.engine import AutoEDAEngine
+from noweda.plugins import default_plugins
+
+@pd.api.extensions.register_dataframe_accessor("noweda")
+class NowEDAAccessor:
+
+    def __init__(self, pandas_obj):
+        self._df = pandas_obj
+        self._report = None
+
+    def _ensure_analyzed(self):
+        if self._report is None:
+            engine = AutoEDAEngine(default_plugins())
+            self._report = engine.run_df(self._df)
+
+    def summary(self):
+        self._ensure_analyzed()
+        return self._report["results"]
+
+    def insights(self):
+        self._ensure_analyzed()
+        return self._report["insights"]
+
+    def score(self):
+        self._ensure_analyzed()
+        return self._report["scores"]
+
+    def report(self):
+        self._ensure_analyzed()
+        return self._report

noweda/cli.py

@@ -0,0 +1,29 @@
+import argparse
+import json
+from noweda import read
+from noweda.report.html import generate_html_report
+
+def main():
+    parser = argparse.ArgumentParser(description="NowEDA CLI")
+    parser.add_argument("file", help="Path to dataset")
+    parser.add_argument("--html", help="Export HTML report", default=None)
+    parser.add_argument("--json", help="Export JSON report", default=None)
+
+    args = parser.parse_args()
+
+    df = read(args.file)
+    report = df.noweda.report()
+
+    print("\n=== NOWEDA INSIGHTS ===")
+    for insight in report["insights"]:
+        print(f"- {insight}")
+
+    print("\n=== SCORES ===")
+    print(report["scores"])
+
+    if args.json:
+        with open(args.json, "w") as f:
+            json.dump(report, f, indent=2)
+
+    if args.html:
+        generate_html_report(report, args.html)

noweda/core/engine.py

@@ -0,0 +1,22 @@
+from noweda.scoring.scorer import Scorer
+from noweda.insights.generator import InsightGenerator
+
+class AutoEDAEngine:
+
+    def __init__(self, plugins):
+        self.plugins = plugins
+
+    def run_df(self, df):
+        results = {}
+
+        for plugin in self.plugins:
+            results[plugin.name] = plugin.run(df)
+
+        scores = Scorer().compute(results)
+        insights = InsightGenerator().generate(results, scores)
+
+        return {
+            "results": results,
+            "scores": scores,
+            "insights": insights
+        }

noweda/insights/generator.py

@@ -0,0 +1,151 @@
+class InsightGenerator:
+    """Translate raw plugin results into human-readable, actionable insights."""
+
+    def generate(self, results, scores):
+        insights = []
+
+        self._schema_insights(results.get("schema", {}), insights)
+        self._missing_insights(results.get("missing", {}), insights)
+        self._duplicates_insights(results.get("duplicates", {}), insights)
+        self._stats_insights(results.get("stats", {}), insights)
+        self._outlier_insights(results.get("outliers", {}), insights)
+        self._correlation_insights(results.get("correlation", {}), insights)
+        self._pii_insights(results.get("pii", {}), insights)
+        self._encoding_insights(results.get("encoding", {}), insights)
+        self._score_insights(scores, insights)
+
+        return insights
+
+    # ------------------------------------------------------------------
+    # Per-plugin insight rules
+    # ------------------------------------------------------------------
+
+    def _schema_insights(self, schema, insights):
+        id_cols = [c for c, v in schema.items() if v.get("role") == "id_candidate"]
+        cat_cols = [c for c, v in schema.items() if "categorical" in v.get("role", "")]
+        datetime_cols = [c for c, v in schema.items() if v.get("role") == "datetime"]
+
+        if id_cols:
+            insights.append(
+                f"Likely identifier column(s) detected: {', '.join(id_cols)}. "
+                "Consider excluding from modelling."
+            )
+        if cat_cols:
+            insights.append(
+                f"Column(s) with low cardinality (likely categorical): {', '.join(cat_cols)}."
+            )
+        if datetime_cols:
+            insights.append(
+                f"Datetime column(s) detected: {', '.join(datetime_cols)}. "
+                "Temporal features may be valuable."
+            )
+
+    def _missing_insights(self, missing, insights):
+        critical = [(c, v) for c, v in missing.items() if v > 0.5]
+        high = [(c, v) for c, v in missing.items() if 0.3 < v <= 0.5]
+        low = [(c, v) for c, v in missing.items() if 0 < v <= 0.3]
+
+        for col, pct in critical:
+            insights.append(
+                f"Column '{col}' is {pct:.0%} missing — consider dropping it."
+            )
+        for col, pct in high:
+            insights.append(
+                f"Column '{col}' has high missing rate ({pct:.0%}) — imputation recommended."
+            )
+        if low:
+            cols = ", ".join(f"'{c}'" for c, _ in low)
+            insights.append(f"Minor missing values in: {cols}.")
+
+    def _duplicates_insights(self, duplicates, insights):
+        dup_rows = duplicates.get("duplicate_rows", 0)
+        dup_pct = duplicates.get("duplicate_rows_pct", 0.0)
+        const_cols = duplicates.get("constant_columns", [])
+
+        if dup_rows > 0:
+            insights.append(
+                f"{dup_rows} duplicate row(s) detected ({dup_pct:.1%} of data). "
+                "De-duplication is recommended."
+            )
+        if const_cols:
+            insights.append(
+                f"Constant (zero-variance) column(s) detected: {', '.join(const_cols)}. "
+                "These carry no information and can be dropped."
+            )
+
+    def _stats_insights(self, stats, insights):
+        for col, s in stats.items():
+            skew = s.get("skewness")
+            if skew is not None and abs(skew) > 2:
+                direction = "right" if skew > 0 else "left"
+                insights.append(
+                    f"Column '{col}' is heavily {direction}-skewed (skewness={skew:.2f}). "
+                    "Log or power transform may help."
+                )
+
+    def _outlier_insights(self, outliers, insights):
+        for col, count in outliers.items():
+            if count > 0:
+                insights.append(
+                    f"Column '{col}' has {count} outlier(s) (IQR method). "
+                    "Review before modelling."
+                )
+
+    def _correlation_insights(self, correlation, insights):
+        seen = set()
+        for col1, corrs in correlation.items():
+            for col2, val in corrs.items():
+                pair = tuple(sorted([col1, col2]))
+                if col1 == col2 or pair in seen:
+                    continue
+                seen.add(pair)
+                if abs(val) > 0.9:
+                    insights.append(
+                        f"Very strong correlation ({val:.2f}) between '{col1}' and '{col2}'. "
+                        "One may be redundant."
+                    )
+                elif abs(val) > 0.7:
+                    insights.append(
+                        f"Strong correlation ({val:.2f}) between '{col1}' and '{col2}'."
+                    )
+
+    def _pii_insights(self, pii, insights):
+        for col, info in pii.items():
+            count = info.get("emails_detected", 0)
+            insights.append(
+                f"PII detected in column '{col}': {count} email address(es) found. "
+                "Mask or remove before sharing."
+            )
+
+    def _encoding_insights(self, encoding, insights):
+        for col, enc_type in encoding.items():
+            insights.append(
+                f"Column '{col}' may contain encoded data ({enc_type}). "
+                "Inspect for hidden payloads or obfuscation."
+            )
+
+    def _score_insights(self, scores, insights):
+        quality = scores.get("data_quality", 100)
+        risk = scores.get("risk", 0)
+
+        if quality >= 90:
+            insights.append("Data quality score is excellent (>=90). Dataset looks clean.")
+        elif quality >= 70:
+            insights.append(f"Data quality score is acceptable ({quality}). Minor issues present.")
+        else:
+            insights.append(
+                f"Data quality score is low ({quality}). Significant cleaning required."
+            )
+
+        if risk == 0:
+            insights.append("No risk signals detected.")
+        elif risk <= 20:
+            insights.append(f"Low risk level ({risk}). Some sensitive indicators found.")
+        elif risk <= 50:
+            insights.append(
+                f"Moderate risk level ({risk}). Review PII and encoded columns before sharing."
+            )
+        else:
+            insights.append(
+                f"High risk level ({risk}). Sensitive data likely present — handle with care."
+            )

noweda/io.py

@@ -0,0 +1,15 @@
+import pandas as pd
+
+def read(file_path, **kwargs):
+    if file_path.endswith(".csv"):
+        return pd.read_csv(file_path, **kwargs)
+    elif file_path.endswith(".xlsx"):
+        return pd.read_excel(file_path, **kwargs)
+    elif file_path.endswith(".json"):
+        return pd.read_json(file_path, **kwargs)
+    elif file_path.endswith(".xml"):
+        return pd.read_xml(file_path, **kwargs)
+    elif file_path.endswith(".html"):
+        return pd.read_html(file_path, **kwargs)[0]
+    else:
+        raise ValueError(f"Unsupported file type: {file_path}")

noweda/plugins/__init__.py

@@ -0,0 +1,21 @@
+from .missing import MissingDataPlugin
+from .stats import StatsPlugin
+from .schema import SchemaPlugin
+from .duplicates import DuplicatesPlugin
+from .correlation import CorrelationPlugin
+from .outliers import OutlierPlugin
+from .pii import PIIDetectorPlugin
+from .encoding import EncodingDetectionPlugin
+
+
+def default_plugins():
+    return [
+        SchemaPlugin(),
+        StatsPlugin(),
+        MissingDataPlugin(),
+        DuplicatesPlugin(),
+        CorrelationPlugin(),
+        OutlierPlugin(),
+        PIIDetectorPlugin(),
+        EncodingDetectionPlugin(),
+    ]

noweda/plugins/base.py

@@ -0,0 +1,5 @@
+class BasePlugin:
+    name = "base"
+
+    def run(self, df):
+        raise NotImplementedError

noweda/plugins/correlation.py

@@ -0,0 +1,10 @@
+from .base import BasePlugin
+
+class CorrelationPlugin(BasePlugin):
+    name = "correlation"
+
+    def run(self, df):
+        numeric_df = df.select_dtypes(include="number")
+        if numeric_df.empty:
+            return {}
+        return numeric_df.corr().to_dict()

noweda/plugins/duplicates.py

@@ -0,0 +1,23 @@
+from .base import BasePlugin
+
+
+class DuplicatesPlugin(BasePlugin):
+    """Detect duplicate rows and constant (zero-variance) columns."""
+
+    name = "duplicates"
+
+    def run(self, df):
+        n_rows = len(df)
+
+        duplicate_rows = int(df.duplicated().sum())
+        duplicate_pct = round(duplicate_rows / n_rows, 4) if n_rows > 0 else 0.0
+
+        constant_columns = [
+            col for col in df.columns if df[col].nunique(dropna=False) <= 1
+        ]
+
+        return {
+            "duplicate_rows": duplicate_rows,
+            "duplicate_rows_pct": duplicate_pct,
+            "constant_columns": constant_columns,
+        }

noweda/plugins/encoding.py

@@ -0,0 +1,24 @@
+import base64
+from .base import BasePlugin
+
+class EncodingDetectionPlugin(BasePlugin):
+    name = "encoding"
+
+    def is_base64(self, s):
+        try:
+            return base64.b64encode(base64.b64decode(s)) == s.encode()
+        except:
+            return False
+
+    def run(self, df):
+        results = {}
+
+        for col in df.columns:
+            if df[col].dtype == "object":
+                sample = df[col].dropna().astype(str).head(20)
+                count = sum(self.is_base64(x) for x in sample)
+
+                if count > 5:
+                    results[col] = "possible_base64"
+
+        return results

noweda/plugins/missing.py

@@ -0,0 +1,10 @@
+from .base import BasePlugin
+
+class MissingDataPlugin(BasePlugin):
+    name = "missing"
+
+    def run(self, df):
+        return {
+            col: float(df[col].isna().mean())
+            for col in df.columns
+        }

noweda/plugins/outliers.py

@@ -0,0 +1,17 @@
+from .base import BasePlugin
+
+class OutlierPlugin(BasePlugin):
+    name = "outliers"
+
+    def run(self, df):
+        result = {}
+
+        for col in df.select_dtypes(include="number").columns:
+            q1 = df[col].quantile(0.25)
+            q3 = df[col].quantile(0.75)
+            iqr = q3 - q1
+
+            outliers = df[(df[col] < q1 - 1.5*iqr) | (df[col] > q3 + 1.5*iqr)]
+            result[col] = int(len(outliers))
+
+        return result

noweda/plugins/pii.py

@@ -0,0 +1,18 @@
+import re
+from .base import BasePlugin
+
+class PIIDetectorPlugin(BasePlugin):
+    name = "pii"
+
+    EMAIL_REGEX = r"[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+"
+
+    def run(self, df):
+        findings = {}
+
+        for col in df.columns:
+            if df[col].dtype == "object":
+                matches = df[col].astype(str).str.contains(self.EMAIL_REGEX, regex=True).sum()
+                if matches > 0:
+                    findings[col] = {"emails_detected": int(matches)}
+
+        return findings

noweda/plugins/schema.py

@@ -0,0 +1,68 @@
+from .base import BasePlugin
+
+
+# Cardinality thresholds
+_ID_UNIQUENESS = 0.95   # column is likely an ID/key
+_CAT_CARDINALITY = 0.05  # column is likely categorical
+
+
+class SchemaPlugin(BasePlugin):
+    """Infer semantic column roles: id, categorical, numeric, datetime, text."""
+
+    name = "schema"
+
+    def _infer_role(self, series, n_rows):
+        dtype = series.dtype
+        n_unique = series.nunique()
+        uniqueness = n_unique / n_rows if n_rows > 0 else 0
+
+        # Datetime detection
+        if dtype.kind == "M":
+            return "datetime"
+
+        # Try to parse object columns as datetime
+        if dtype == "object":
+            sample = series.dropna().astype(str).head(20)
+            import pandas as pd
+            try:
+                pd.to_datetime(sample)
+                return "datetime"
+            except Exception:
+                pass
+
+        # Numeric
+        if dtype.kind in ("i", "u", "f"):
+            # Only flag integer-typed columns with perfect uniqueness as IDs.
+            # Float columns (salary, score) are almost never true keys.
+            # Require uniqueness == 1.0 so numeric columns with any duplicates
+            # (like age appearing twice) are NOT incorrectly flagged.
+            if dtype.kind in ("i", "u") and uniqueness == 1.0 and n_unique > 5:
+                return "id_candidate"
+            if uniqueness <= _CAT_CARDINALITY and n_unique <= 20:
+                return "categorical_numeric"
+            return "numeric"
+
+        # Object / string
+        if dtype == "object":
+            # Require near-perfect uniqueness for string ID detection
+            if uniqueness >= 0.98 and n_unique > 5:
+                return "id_candidate"
+            if uniqueness <= _CAT_CARDINALITY and n_unique <= 50:
+                return "categorical"
+            return "text"
+
+        return "unknown"
+
+    def run(self, df):
+        n_rows = len(df)
+        result = {}
+
+        for col in df.columns:
+            result[col] = {
+                "dtype": str(df[col].dtype),
+                "role": self._infer_role(df[col], n_rows),
+                "unique": int(df[col].nunique()),
+                "uniqueness_ratio": round(df[col].nunique() / n_rows, 4) if n_rows > 0 else 0,
+            }
+
+        return result

noweda/plugins/stats.py

@@ -0,0 +1,40 @@
+from .base import BasePlugin
+
+
+class StatsPlugin(BasePlugin):
+    """Compute descriptive statistics for numeric and categorical columns."""
+
+    name = "stats"
+
+    def run(self, df):
+        result = {}
+
+        for col in df.columns:
+            series = df[col]
+            col_stats = {
+                "dtype": str(series.dtype),
+                "count": int(series.count()),
+                "missing": int(series.isna().sum()),
+                "unique": int(series.nunique()),
+            }
+
+            if series.dtype.kind in ("i", "u", "f"):
+                col_stats.update({
+                    "mean": float(series.mean()),
+                    "median": float(series.median()),
+                    "std": float(series.std()),
+                    "min": float(series.min()),
+                    "max": float(series.max()),
+                    "q25": float(series.quantile(0.25)),
+                    "q75": float(series.quantile(0.75)),
+                    "skewness": float(series.skew()),
+                })
+            elif series.dtype == "object" or str(series.dtype) == "category":
+                top = series.value_counts()
+                if not top.empty:
+                    col_stats["top_value"] = str(top.index[0])
+                    col_stats["top_freq"] = int(top.iloc[0])
+
+            result[col] = col_stats
+
+        return result

noweda/report/html.py

@@ -0,0 +1,283 @@
+import json
+from datetime import datetime
+
+
+def generate_html_report(report, output_path):
+    scores = report.get("scores", {})
+    insights = report.get("insights", [])
+    results = report.get("results", {})
+
+    quality = scores.get("data_quality", 0)
+    risk = scores.get("risk", 0)
+    readiness = scores.get("model_readiness", 0)
+
+    quality_color = _score_color(quality, invert=False)
+    risk_color = _score_color(risk, invert=True, max_val=100)
+    readiness_color = _score_color(readiness, invert=False)
+
+    insights_html = "".join(
+        f'<li class="insight-item">{_escape(i)}</li>' for i in insights
+    )
+
+    schema_html = _schema_table(results.get("schema", {}))
+    missing_html = _missing_table(results.get("missing", {}))
+    duplicates_html = _duplicates_section(results.get("duplicates", {}))
+    outliers_html = _outliers_table(results.get("outliers", {}))
+    pii_html = _pii_table(results.get("pii", {}))
+    encoding_html = _encoding_table(results.get("encoding", {}))
+
+    generated_at = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+    html = f"""<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8" />
+<meta name="viewport" content="width=device-width, initial-scale=1.0" />
+<title>NowEDA Report</title>
+<style>
+  :root {{
+    --bg: #0f1117;
+    --surface: #1a1d27;
+    --surface2: #22263a;
+    --border: #2e3250;
+    --text: #e2e8f0;
+    --muted: #8892b0;
+    --accent: #7c6af7;
+    --green: #4ade80;
+    --yellow: #facc15;
+    --red: #f87171;
+  }}
+  * {{ box-sizing: border-box; margin: 0; padding: 0; }}
+  body {{ background: var(--bg); color: var(--text); font-family: 'Segoe UI', system-ui, sans-serif; font-size: 14px; }}
+  .container {{ max-width: 1100px; margin: 0 auto; padding: 32px 24px; }}
+  .header {{ margin-bottom: 36px; border-bottom: 1px solid var(--border); padding-bottom: 20px; }}
+  .header h1 {{ font-size: 28px; font-weight: 700; color: var(--accent); letter-spacing: -0.5px; }}
+  .header .meta {{ color: var(--muted); font-size: 12px; margin-top: 6px; }}
+  .scores-grid {{ display: grid; grid-template-columns: repeat(3, 1fr); gap: 16px; margin-bottom: 36px; }}
+  .score-card {{ background: var(--surface); border: 1px solid var(--border); border-radius: 12px; padding: 20px; text-align: center; }}
+  .score-card .label {{ color: var(--muted); font-size: 11px; text-transform: uppercase; letter-spacing: 1px; margin-bottom: 8px; }}
+  .score-card .value {{ font-size: 40px; font-weight: 800; line-height: 1; }}
+  .score-card .sub {{ font-size: 11px; color: var(--muted); margin-top: 4px; }}
+  .section {{ background: var(--surface); border: 1px solid var(--border); border-radius: 12px; padding: 24px; margin-bottom: 24px; }}
+  .section h2 {{ font-size: 16px; font-weight: 600; margin-bottom: 16px; color: var(--text); border-bottom: 1px solid var(--border); padding-bottom: 10px; }}
+  .insight-list {{ list-style: none; display: flex; flex-direction: column; gap: 8px; }}
+  .insight-item {{ background: var(--surface2); border-left: 3px solid var(--accent); border-radius: 4px; padding: 10px 14px; font-size: 13px; line-height: 1.5; }}
+  table {{ width: 100%; border-collapse: collapse; font-size: 13px; }}
+  th {{ text-align: left; color: var(--muted); font-weight: 600; font-size: 11px; text-transform: uppercase; letter-spacing: 0.5px; padding: 8px 12px; border-bottom: 1px solid var(--border); }}
+  td {{ padding: 8px 12px; border-bottom: 1px solid var(--border); color: var(--text); }}
+  tr:last-child td {{ border-bottom: none; }}
+  tr:hover td {{ background: var(--surface2); }}
+  .tag {{ display: inline-block; padding: 2px 8px; border-radius: 4px; font-size: 11px; font-weight: 600; text-transform: uppercase; }}
+  .tag-id {{ background: #312e81; color: #a5b4fc; }}
+  .tag-cat {{ background: #164e63; color: #67e8f9; }}
+  .tag-num {{ background: #14532d; color: #86efac; }}
+  .tag-text {{ background: #44403c; color: #d6d3d1; }}
+  .tag-dt {{ background: #4a1d96; color: #c4b5fd; }}
+  .tag-unk {{ background: #2d2d2d; color: #9ca3af; }}
+  .pct-bar {{ background: var(--surface2); border-radius: 4px; height: 6px; width: 100px; display: inline-block; vertical-align: middle; margin-left: 8px; }}
+  .pct-fill {{ height: 100%; border-radius: 4px; }}
+  .empty {{ color: var(--muted); font-style: italic; font-size: 13px; }}
+  .footer {{ text-align: center; color: var(--muted); font-size: 11px; margin-top: 40px; padding-top: 20px; border-top: 1px solid var(--border); }}
+</style>
+</head>
+<body>
+<div class="container">
+
+  <div class="header">
+    <h1>NowEDA Report</h1>
+    <div class="meta">Generated: {generated_at}</div>
+  </div>
+
+  <div class="scores-grid">
+    <div class="score-card">
+      <div class="label">Data Quality</div>
+      <div class="value" style="color:{quality_color}">{quality}</div>
+      <div class="sub">out of 100</div>
+    </div>
+    <div class="score-card">
+      <div class="label">Risk Level</div>
+      <div class="value" style="color:{risk_color}">{risk}</div>
+      <div class="sub">higher = more risk</div>
+    </div>
+    <div class="score-card">
+      <div class="label">Model Readiness</div>
+      <div class="value" style="color:{readiness_color}">{readiness}</div>
+      <div class="sub">out of 100</div>
+    </div>
+  </div>
+
+  <div class="section">
+    <h2>Insights</h2>
+    {"<ul class='insight-list'>" + insights_html + "</ul>" if insights else "<p class='empty'>No insights generated.</p>"}
+  </div>
+
+  {schema_html}
+  {missing_html}
+  {duplicates_html}
+  {outliers_html}
+  {pii_html}
+  {encoding_html}
+
+  <div class="footer">
+    Built with <strong>NowEDA</strong> — Automated EDA Framework
+  </div>
+
+</div>
+</body>
+</html>"""
+
+    with open(output_path, "w", encoding="utf-8") as f:
+        f.write(html)
+
+
+# ------------------------------------------------------------------
+# Helpers
+# ------------------------------------------------------------------
+
+def _escape(s):
+    return (
+        str(s)
+        .replace("&", "&amp;")
+        .replace("<", "&lt;")
+        .replace(">", "&gt;")
+        .replace('"', "&quot;")
+    )
+
+
+def _score_color(val, invert=False, max_val=100):
+    """Return a CSS color (green/yellow/red) based on value."""
+    ratio = val / max_val if max_val else 0
+    if invert:
+        ratio = 1 - min(ratio, 1)
+    if ratio >= 0.75:
+        return "var(--green)"
+    elif ratio >= 0.4:
+        return "var(--yellow)"
+    return "var(--red)"
+
+
+def _role_tag(role):
+    mapping = {
+        "id_candidate": ("ID", "tag-id"),
+        "categorical": ("Categorical", "tag-cat"),
+        "categorical_numeric": ("Cat. Numeric", "tag-cat"),
+        "numeric": ("Numeric", "tag-num"),
+        "text": ("Text", "tag-text"),
+        "datetime": ("Datetime", "tag-dt"),
+        "unknown": ("Unknown", "tag-unk"),
+    }
+    label, css = mapping.get(role, (role, "tag-unk"))
+    return f'<span class="tag {css}">{label}</span>'
+
+
+def _schema_table(schema):
+    if not schema:
+        return ""
+    rows = ""
+    for col, info in schema.items():
+        rows += (
+            f"<tr>"
+            f"<td>{_escape(col)}</td>"
+            f"<td>{_escape(info.get('dtype', ''))}</td>"
+            f"<td>{_role_tag(info.get('role', 'unknown'))}</td>"
+            f"<td>{info.get('unique', '')}</td>"
+            f"<td>{info.get('uniqueness_ratio', ''):.2%}</td>"
+            f"</tr>"
+        )
+    return f"""<div class="section">
+  <h2>Column Schema</h2>
+  <table>
+    <thead><tr><th>Column</th><th>Dtype</th><th>Role</th><th>Unique Values</th><th>Uniqueness</th></tr></thead>
+    <tbody>{rows}</tbody>
+  </table>
+</div>"""
+
+
+def _missing_table(missing):
+    if not missing:
+        return ""
+    rows = ""
+    for col, pct in sorted(missing.items(), key=lambda x: -x[1]):
+        pct_int = int(pct * 100)
+        bar_color = "#f87171" if pct > 0.3 else ("#facc15" if pct > 0 else "#4ade80")
+        bar = (
+            f'<span class="pct-bar"><span class="pct-fill" '
+            f'style="width:{pct_int}%;background:{bar_color}"></span></span>'
+        )
+        rows += f"<tr><td>{_escape(col)}</td><td>{pct:.1%} {bar}</td></tr>"
+    return f"""<div class="section">
+  <h2>Missing Values</h2>
+  <table>
+    <thead><tr><th>Column</th><th>Missing %</th></tr></thead>
+    <tbody>{rows}</tbody>
+  </table>
+</div>"""
+
+
+def _duplicates_section(duplicates):
+    if not duplicates:
+        return ""
+    dup_rows = duplicates.get("duplicate_rows", 0)
+    dup_pct = duplicates.get("duplicate_rows_pct", 0.0)
+    const_cols = duplicates.get("constant_columns", [])
+    const_str = ", ".join(const_cols) if const_cols else "None"
+    return f"""<div class="section">
+  <h2>Duplicates &amp; Constants</h2>
+  <table>
+    <thead><tr><th>Metric</th><th>Value</th></tr></thead>
+    <tbody>
+      <tr><td>Duplicate rows</td><td>{dup_rows} ({dup_pct:.1%})</td></tr>
+      <tr><td>Constant columns</td><td>{_escape(const_str)}</td></tr>
+    </tbody>
+  </table>
+</div>"""
+
+
+def _outliers_table(outliers):
+    if not outliers:
+        return ""
+    rows = "".join(
+        f"<tr><td>{_escape(col)}</td><td>{count}</td></tr>"
+        for col, count in sorted(outliers.items(), key=lambda x: -x[1])
+        if count > 0
+    )
+    if not rows:
+        return ""
+    return f"""<div class="section">
+  <h2>Outliers (IQR Method)</h2>
+  <table>
+    <thead><tr><th>Column</th><th>Outlier Count</th></tr></thead>
+    <tbody>{rows}</tbody>
+  </table>
+</div>"""
+
+
+def _pii_table(pii):
+    if not pii:
+        return ""
+    rows = ""
+    for col, info in pii.items():
+        rows += f"<tr><td>{_escape(col)}</td><td>{_escape(str(info))}</td></tr>"
+    return f"""<div class="section" style="border-color:#f87171">
+  <h2>PII Detection</h2>
+  <table>
+    <thead><tr><th>Column</th><th>Finding</th></tr></thead>
+    <tbody>{rows}</tbody>
+  </table>
+</div>"""
+
+
+def _encoding_table(encoding):
+    if not encoding:
+        return ""
+    rows = "".join(
+        f"<tr><td>{_escape(col)}</td><td>{_escape(enc_type)}</td></tr>"
+        for col, enc_type in encoding.items()
+    )
+    return f"""<div class="section" style="border-color:#facc15">
+  <h2>Encoding Detection</h2>
+  <table>
+    <thead><tr><th>Column</th><th>Detected Encoding</th></tr></thead>
+    <tbody>{rows}</tbody>
+  </table>
+</div>"""

noweda/scoring/scorer.py

@@ -0,0 +1,86 @@
+class Scorer:
+    """
+    Compute data quality and risk scores from plugin results.
+
+    Scores:
+        data_quality : 0-100  (higher = cleaner)
+        risk         : 0+     (higher = more sensitive/risky)
+        model_readiness : 0-100  (higher = more ready for ML)
+    """
+
+    def compute(self, results):
+        scores = {
+            "data_quality": 100,
+            "risk": 0,
+            "model_readiness": 100,
+        }
+
+        self._penalise_missing(results.get("missing", {}), scores)
+        self._penalise_duplicates(results.get("duplicates", {}), scores)
+        self._penalise_outliers(results.get("outliers", {}), scores)
+        self._penalise_skew(results.get("stats", {}), scores)
+        self._add_pii_risk(results.get("pii", {}), scores)
+        self._add_encoding_risk(results.get("encoding", {}), scores)
+        self._penalise_schema(results.get("schema", {}), scores)
+
+        # Clamp quality and readiness to [0, 100]
+        scores["data_quality"] = max(0, min(100, scores["data_quality"]))
+        scores["model_readiness"] = max(0, min(100, scores["model_readiness"]))
+
+        return scores
+
+    # ------------------------------------------------------------------
+
+    def _penalise_missing(self, missing, scores):
+        for pct in missing.values():
+            if pct > 0.5:
+                scores["data_quality"] -= 10
+                scores["model_readiness"] -= 15
+            elif pct > 0.3:
+                scores["data_quality"] -= 5
+                scores["model_readiness"] -= 8
+            elif pct > 0:
+                scores["data_quality"] -= 2
+                scores["model_readiness"] -= 3
+
+    def _penalise_duplicates(self, duplicates, scores):
+        dup_pct = duplicates.get("duplicate_rows_pct", 0.0)
+        const_cols = duplicates.get("constant_columns", [])
+
+        if dup_pct > 0.1:
+            scores["data_quality"] -= 10
+        elif dup_pct > 0:
+            scores["data_quality"] -= 3
+
+        scores["data_quality"] -= len(const_cols) * 3
+        scores["model_readiness"] -= len(const_cols) * 5
+
+    def _penalise_outliers(self, outliers, scores):
+        total = sum(outliers.values())
+        if total > 50:
+            scores["data_quality"] -= 10
+            scores["model_readiness"] -= 10
+        elif total > 10:
+            scores["data_quality"] -= 5
+            scores["model_readiness"] -= 5
+
+    def _penalise_skew(self, stats, scores):
+        heavy_skew_cols = [
+            col for col, s in stats.items()
+            if s.get("skewness") is not None and abs(s["skewness"]) > 2
+        ]
+        scores["model_readiness"] -= len(heavy_skew_cols) * 5
+
+    def _add_pii_risk(self, pii, scores):
+        scores["risk"] += len(pii) * 15
+
+    def _add_encoding_risk(self, encoding, scores):
+        scores["risk"] += len(encoding) * 10
+
+    def _penalise_schema(self, schema, scores):
+        # All-text or all-unknown columns reduce model readiness
+        untyped = sum(
+            1 for v in schema.values()
+            if v.get("role") in ("text", "unknown")
+        )
+        scores["model_readiness"] -= untyped * 3

noweda-0.1.0.dist-info/METADATA

@@ -0,0 +1,248 @@
+Metadata-Version: 2.4
+Name: noweda
+Version: 0.1.0
+Summary: Automated EDA with insights, scoring, and security-aware detection — built as a native pandas extension.
+Author-email: Daniel Peng <danielpeng95@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/codewithdaniel1/NowEDA
+Project-URL: Repository, https://github.com/codewithdaniel1/NowEDA
+Project-URL: Bug Tracker, https://github.com/codewithdaniel1/NowEDA/issues
+Keywords: eda,data-analysis,pandas,data-quality,pii-detection,automated-eda,profiling
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas>=1.3
+Requires-Dist: numpy>=1.21
+Requires-Dist: openpyxl>=3.0
+Dynamic: license-file
+
+# NowEDA
+
+**Automated Exploratory Data Analysis — built as a native pandas extension.**
+
+NowEDA is a lightweight, modular Python framework that turns any dataset into instant insight. Load any file, call `df.noweda.*`, and get a full EDA report — including data quality scoring, PII detection, outlier analysis, and human-readable insights — with zero boilerplate.
+
+---
+
+## Features
+
+| Feature | Description |
+|---|---|
+| Universal ingestion | CSV, Excel, JSON, XML, HTML |
+| Native pandas accessor | `df.noweda.*` — feels like pandas |
+| Plugin architecture | Every analysis is a swappable plugin |
+| Schema inference | Auto-detects IDs, categoricals, datetimes, text |
+| Data quality scoring | 0–100 quality + model-readiness score |
+| Risk scoring | PII and encoding risk level |
+| PII detection | Email addresses + extensible patterns |
+| Encoding detection | Base64 and obfuscation signals |
+| Outlier detection | IQR-based, per numeric column |
+| Duplicate detection | Exact row duplicates + constant columns |
+| Actionable insights | Human-readable text, not just numbers |
+| HTML report | Dark-themed, stakeholder-ready export |
+| CLI | One-liner from the terminal |
+
+---
+
+## Installation
+
+```bash
+cd NowEDA
+pip install -e .
+```
+
+> Requires Python 3.8+ and pandas 1.3+.
+
+---
+
+## Quick Start
+
+```python
+import noweda
+
+df = noweda.read("data.csv")
+
+# Still a regular pandas DataFrame — nothing changes
+print(df.head())
+print(df.describe())
+
+# NowEDA layer
+print(df.noweda.insights())   # human-readable insight list
+print(df.noweda.score())      # quality, risk, model_readiness
+print(df.noweda.summary())    # raw plugin results
+report = df.noweda.report()   # full structured dict
+```
+
+### All supported formats
+
+```python
+df = noweda.read("data.csv")
+df = noweda.read("data.xlsx", sheet_name="Sheet1")
+df = noweda.read("data.json")
+df = noweda.read("data.xml")
+df = noweda.read("data.html")
+```
+
+Any `**kwargs` are forwarded to the underlying pandas reader.
+
+---
+
+## CLI
+
+```bash
+# Print insights and scores to the terminal
+noweda data.csv
+
+# Export a dark-themed HTML report
+noweda data.csv --html report.html
+
+# Export a JSON report
+noweda data.csv --json report.json
+
+# Both at once
+noweda data.csv --html report.html --json report.json
+```
+
+---
+
+## Score Breakdown
+
+| Score | Range | Meaning |
+|---|---|---|
+| `data_quality` | 0–100 | Penalised for missing values, duplicates, constants, outliers |
+| `model_readiness` | 0–100 | Penalised for skew, untyped columns, high missingness |
+| `risk` | 0+ | Added per PII column (+15) and encoded column (+10) |
+
+---
+
+## Plugin System
+
+Every analysis step is an independent plugin. You can swap, extend, or disable plugins.
+
+```python
+from noweda.core.engine import AutoEDAEngine
+from noweda.plugins.missing import MissingDataPlugin
+from noweda.plugins.pii import PIIDetectorPlugin
+
+# Run only the plugins you want
+engine = AutoEDAEngine([MissingDataPlugin(), PIIDetectorPlugin()])
+report = engine.run_df(df)
+```
+
+### Built-in plugins
+
+| Plugin | Name key | What it detects |
+|---|---|---|
+| `SchemaPlugin` | `schema` | Column roles: id, categorical, numeric, datetime, text |
+| `StatsPlugin` | `stats` | Descriptive stats: mean, median, std, skewness, etc. |
+| `MissingDataPlugin` | `missing` | Per-column missing rate |
+| `DuplicatesPlugin` | `duplicates` | Duplicate rows, constant columns |
+| `CorrelationPlugin` | `correlation` | Pearson correlation matrix (numeric columns) |
+| `OutlierPlugin` | `outliers` | IQR-based outlier counts per column |
+| `PIIDetectorPlugin` | `pii` | Email addresses (extensible to SSN, phone, etc.) |
+| `EncodingDetectionPlugin` | `encoding` | Base64-encoded strings |
+
+### Writing a custom plugin
+
+```python
+from noweda.plugins.base import BasePlugin
+
+class MyPlugin(BasePlugin):
+    name = "my_check"
+
+    def run(self, df):
+        # return any JSON-serialisable dict
+        return {"total_rows": len(df)}
+```
+
+---
+
+## HTML Report
+
+```bash
+noweda examples/sample.csv --html report.html
+```
+
+The report includes:
+
+- Score cards (quality, risk, model readiness)
+- Actionable insights list
+- Column schema table with inferred roles
+- Missing value bars
+- Duplicate and constant column summary
+- Outlier counts
+- PII findings (highlighted)
+- Encoding signals (highlighted)
+
+---
+
+## Running Tests
+
+```bash
+pip install pytest
+python -m pytest tests/ -v
+```
+
+---
+
+## Project Structure
+
+```
+NowEDA/
+├── noweda/
+│   ├── __init__.py          # exposes noweda.read()
+│   ├── io.py                # file ingestion (all formats)
+│   ├── accessor.py          # df.noweda.* pandas accessor
+│   ├── core/
+│   │   └── engine.py        # orchestrates plugins → scorer → insights
+│   ├── plugins/
+│   │   ├── base.py
+│   │   ├── schema.py
+│   │   ├── stats.py
+│   │   ├── missing.py
+│   │   ├── duplicates.py
+│   │   ├── correlation.py
+│   │   ├── outliers.py
+│   │   ├── pii.py
+│   │   └── encoding.py
+│   ├── scoring/
+│   │   └── scorer.py
+│   ├── insights/
+│   │   └── generator.py
+│   ├── report/
+│   │   └── html.py
+│   └── cli.py
+├── examples/
+│   └── sample.csv
+├── tests/
+│   └── test_basic.py
+└── pyproject.toml
+```
+
+---
+
+## Roadmap
+
+- [ ] Visualization layer (histograms, correlation heatmap)
+- [ ] Dataset fingerprinting / hash-based change detection
+- [ ] Additional PII patterns (phone, SSN, credit card)
+- [ ] Streaming / chunked ingestion for large files
+- [ ] PyPI publish
+- [ ] Web dashboard UI
+
+---
+
+## Author
+
+**Daniel Peng** — [danielpeng@osiris.cyber.nyu.edu](mailto:)

noweda-0.1.0.dist-info/RECORD

@@ -0,0 +1,28 @@
+noweda/__init__.py,sha256=SfZISfrn1z9pCXoXIhKySrUjdUQyoRP3msU0qRa44Lg,77
+noweda/accessor.py,sha256=zra5VAyWEfrjqd0kvP5ic15HzenJLY-i7cHqRZI9lL8,821
+noweda/cli.py,sha256=p6MEIJXwlKXn-l7sBCwPvakVBjrwfbUTGyNkQsuG_6I,813
+noweda/io.py,sha256=SMYi1pFcxO-JedqpeVMcq-Y08td5sMqKffHIkgie7h8,559
+noweda/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+noweda/core/engine.py,sha256=ZxDO71rL2RUsaal4_iZEvPvy7h44W4gEmXWUAx8LLu8,549
+noweda/insights/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+noweda/insights/generator.py,sha256=3sc4YUuxapOIeonUGxq_BZoV7jhRzwSRPscooj-SQgY,6336
+noweda/plugins/__init__.py,sha256=M2grrY-qz8Q8btXdvCLIqOYOd5_9a8DGeh7bnx2a3qo,570
+noweda/plugins/base.py,sha256=Iu-v-j8sm_hKiSIalCoBD1Uq5btS96i5Ed3aeOqy8-s,94
+noweda/plugins/correlation.py,sha256=36sfsDHAoq_it-md6ySxDbjYVTQHDlYCPd98XMpBnwg,266
+noweda/plugins/duplicates.py,sha256=7yjvYLjYPEJEQ5UFdgkYbtj3ul4uFaloZjS4vg_jFU4,634
+noweda/plugins/encoding.py,sha256=BlN0kn02ZRBx3w6DI35yN4RrI3hrLfk57PPljntaGtw,616
+noweda/plugins/missing.py,sha256=gcPgk5TSkpRi_0xABi8BQsAxHevUdPUI91HxbFRg0L0,219
+noweda/plugins/outliers.py,sha256=DRmqFyimIBMHLbH_aXKH3nv3Xp18gHd9GFDd6vA9oRQ,447
+noweda/plugins/pii.py,sha256=w3UYxeM7nnjEVJ4yHxCkaDl8ubodh62wTckz05mGlMw,505
+noweda/plugins/schema.py,sha256=GwKNCgFEijZKbC6FuPuzTIV9Z5FQec6PUQ6SuRwPQ0A,2277
+noweda/plugins/stats.py,sha256=8vRWxQvf5gxvFVdq_laplt06uDVZO36Qy2VTk-VLrpo,1361
+noweda/report/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+noweda/report/html.py,sha256=9dkQjDcgnJz2MaXw7DiQRLuBR2GU_pwHtJ1WYntIQvY,10192
+noweda/scoring/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+noweda/scoring/scorer.py,sha256=gG5ttXEsroYvEAsmqP-MbuAV-JLfb3dqXs3sDkmfvD0,3103
+noweda-0.1.0.dist-info/licenses/LICENSE,sha256=7zCIaXWjgEjqbSjHZfGkaVjwpZzrnC8PExoh-HArwog,1068
+noweda-0.1.0.dist-info/METADATA,sha256=UlIcU3LVuLuFhzlXAt9lMWdnz1N71nLKoblXv-JsXrA,7024
+noweda-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+noweda-0.1.0.dist-info/entry_points.txt,sha256=bPHLmGelOac4dnTCaMjzLQqdn-N3ZDXzA-3iceAXn-Q,43
+noweda-0.1.0.dist-info/top_level.txt,sha256=z79WDaLX_ynq-UdeEYwwSgHLArVUj--W5hueoZF8pig,7
+noweda-0.1.0.dist-info/RECORD,,

noweda-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

noweda-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+noweda = noweda.cli:main

noweda-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Daniel Peng
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

noweda-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+noweda