datapilot-kit 0.3.0rc1__py3-none-any.whl

datapilot/cli/__init__.py

@@ -0,0 +1,3 @@
+"""
+Command-line interface for Datapilot.
+"""

datapilot/core/__init__.py

@@ -0,0 +1,3 @@
+"""
+Core modules for Datapilot.
+"""

datapilot/core/loader.py

@@ -0,0 +1,72 @@
+"""
+Dataset loading utilities for Datapilot.
+
+This module is responsible for loading supported dataset formats
+and converting them into a pandas DataFrame.
+"""
+
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+CSV_EXTENSIONS = {".csv"}
+EXCEL_EXTENSIONS = {".xlsx", ".xls"}
+
+
+def load_dataset(data: Any) -> pd.DataFrame:
+    """
+    Load a supported dataset into a pandas DataFrame.
+    """
+
+    if isinstance(data, pd.DataFrame):
+        return _load_dataframe(data)
+
+    if isinstance(data, (str, Path)):
+        return _load_file(Path(data))
+
+    raise TypeError(
+        f"Unsupported input type: {type(data).__name__}"
+    )
+
+
+def _load_dataframe(dataframe: pd.DataFrame) -> pd.DataFrame:
+    """
+    Return a copy of the provided DataFrame.
+    """
+
+    return dataframe.copy(deep=True)
+
+
+def _load_file(path: Path) -> pd.DataFrame:
+    """
+    Load a dataset from a supported file.
+    """
+
+    suffix = path.suffix.lower()
+
+    if suffix in CSV_EXTENSIONS:
+        return _load_csv(path)
+
+    if suffix in EXCEL_EXTENSIONS:
+        return _load_excel(path)
+
+    raise ValueError(
+        f"Unsupported file format: '{suffix}'"
+    )
+
+
+def _load_csv(path: Path) -> pd.DataFrame:
+    """
+    Load a CSV file.
+    """
+
+    return pd.read_csv(path)
+
+
+def _load_excel(path: Path) -> pd.DataFrame:
+    """
+    Load an Excel file.
+    """
+
+    return pd.read_excel(path)

datapilot/core/report.py

@@ -0,0 +1,104 @@
+"""
+Report object for Datapilot.
+
+The Report class is the primary interface returned by the
+public `analyze()` function.
+"""
+
+import pandas as pd
+
+from ..analysis.correlation import (
+    generate_correlation_summary,
+)
+from ..analysis.datatype import generate_data_type_summary
+from ..analysis.duplicate import generate_duplicate_summary
+from ..analysis.health import generate_dataset_health
+from ..analysis.insights import (
+    generate_insight_summary,
+)
+from ..analysis.missing import generate_missing_value_summary
+from ..analysis.models import (
+    CorrelationSummary,
+    DatasetHealth,
+    DatasetSummary,
+    DataTypeSummary,
+    DuplicateSummary,
+    InsightSummary,
+    MissingValueSummary,
+    OutlierSummary,
+    StatisticsSummary,
+)
+from ..analysis.outliers import generate_outlier_summary
+from ..analysis.statistics import generate_statistics_summary
+from ..analysis.summary import generate_summary
+
+
+class Report:
+    """
+    Represents the analysis results of a dataset.
+    """
+
+    def __init__(self, dataframe: pd.DataFrame) -> None:
+        """
+        Initialize a Report object.
+
+        Parameters
+        ----------
+        dataframe : pandas.DataFrame
+            Dataset to analyze.
+        """
+        self._df = dataframe
+
+    def summary(self) -> DatasetSummary:
+        """
+        Return a summary of the dataset.
+        """
+        return generate_summary(self._df)
+
+    def missing_values(self) -> MissingValueSummary:
+        """
+        Return missing value statistics for the dataset.
+        """
+        return generate_missing_value_summary(self._df)
+
+    def duplicates(self) -> DuplicateSummary:
+        """
+        Return duplicate row statistics for the dataset.
+        """
+        return generate_duplicate_summary(self._df)
+
+    def data_types(self) -> DataTypeSummary:
+        """
+        Return data type statistics for the dataset.
+        """
+        return generate_data_type_summary(self._df)
+
+    def dataset_health(self) -> DatasetHealth:
+        """
+        Return the overall health assessment of the dataset.
+        """
+        return generate_dataset_health(self._df)
+
+    def statistics(self) -> StatisticsSummary:
+        """
+        Return statistical summary.
+        """
+        return generate_statistics_summary(self._df)
+
+    def outliers(self) -> OutlierSummary:
+        """
+        Return outlier statistics.
+        """
+        return generate_outlier_summary(self._df)
+
+    def correlation(self) -> CorrelationSummary:
+        """
+        Return correlation analysis.
+        """
+        return generate_correlation_summary(self._df)
+
+    def insights(self) -> InsightSummary:
+        """
+        Return generated dataset insights.
+        """
+        return generate_insight_summary(self._df)

datapilot/interpretation/__init__.py

@@ -0,0 +1,3 @@
+"""
+Interpretation modules for Datapilot.
+"""

datapilot/reporting/__init__.py

@@ -0,0 +1,3 @@
+"""
+Reporting utilities for Datapilot.
+"""

datapilot/reporting/fragments.py

@@ -0,0 +1,329 @@
+"""
+HTML fragment builders for Datapilot reports.
+"""
+
+from datapilot.core.report import Report
+
+
+def build_column_name_chips(
+    report: Report,
+) -> str:
+    """
+    Build HTML chips for dataset column names.
+    """
+
+    summary = report.summary()
+
+    chips: list[str] = []
+
+    for column in summary.column_names:
+        chips.append(
+            (
+                '<span class="column-chip">'
+                f"{column}"
+                "</span>"
+            )
+        )
+
+    return "\n".join(chips)
+
+
+def build_health_headline(
+    report: Report,
+) -> str:
+    """
+    Build health headline.
+    """
+
+    health = report.dataset_health()
+
+    return (
+        f"Overall dataset quality is "
+        f"{health.status.lower()}."
+    )
+
+
+def build_health_summary(
+    report: Report,
+) -> str:
+    """
+    Build health summary.
+    """
+
+    health = report.dataset_health()
+
+    return (
+        f"The dataset received a score of "
+        f"{health.score}/100 with grade "
+        f"{health.grade}."
+    )
+
+
+def build_html_list(
+    items: list[str],
+) -> str:
+    """
+    Convert a list of strings into HTML list items.
+    """
+
+    if not items:
+        return "<li>None</li>"
+
+    return "\n".join(
+        f"<li>{item}</li>"
+        for item in items
+    )
+
+
+def build_missing_table(
+    report: Report,
+) -> str:
+    """
+    Build missing values table.
+    """
+
+    missing = report.missing_values()
+
+    rows: list[str] = []
+
+    if not missing.columns_with_missing:
+        return (
+            "<p>No missing values were detected.</p>"
+        )
+
+    rows.append(
+        """
+<table class="report-table">
+    <thead>
+        <tr>
+            <th>Column</th>
+            <th>Missing Values</th>
+        </tr>
+    </thead>
+    <tbody>
+"""
+    )
+
+    for column, count in (
+        missing.columns_with_missing.items()
+    ):
+        rows.append(
+            f"""
+<tr>
+    <td>{column}</td>
+    <td>{count}</td>
+</tr>
+"""
+        )
+
+    rows.append(
+        """
+    </tbody>
+</table>
+"""
+    )
+
+    return "\n".join(rows)
+
+def build_statistics_table(
+    report: Report,
+) -> str:
+    """
+    Build statistics table.
+    """
+
+    statistics = report.statistics()
+
+    rows: list[str] = []
+
+    if not statistics.column_statistics:
+        return (
+            "<p>No numeric columns were detected.</p>"
+        )
+
+    rows.append(
+        """
+<table class="report-table">
+    <thead>
+        <tr>
+            <th>Column</th>
+            <th>Mean</th>
+            <th>Median</th>
+            <th>Std</th>
+            <th>Min</th>
+            <th>Max</th>
+        </tr>
+    </thead>
+    <tbody>
+"""
+    )
+
+    for (
+        column,
+        values,
+    ) in statistics.column_statistics.items():
+        rows.append(
+            f"""
+<tr>
+    <td>{column}</td>
+    <td>{values["mean"]:.2f}</td>
+    <td>{values["median"]:.2f}</td>
+    <td>{values["std"]:.2f}</td>
+    <td>{values["min"]:.2f}</td>
+    <td>{values["max"]:.2f}</td>
+</tr>
+"""
+        )
+
+    rows.append(
+        """
+    </tbody>
+</table>
+"""
+    )
+
+    return "\n".join(rows)
+
+def build_outlier_cards(
+    report: Report,
+) -> str:
+    """
+    Build HTML cards for outlier analysis.
+    """
+
+    outliers = report.outliers()
+
+    if not outliers.columns_with_outliers:
+        return (
+            "<p>No outliers were detected.</p>"
+        )
+
+    cards: list[str] = []
+
+    for column, count in (
+        outliers.columns_with_outliers.items()
+    ):
+        cards.append(
+            f"""
+<div class="outlier-card">
+    <h4>{column}</h4>
+    <p>{count} outliers detected</p>
+</div>
+"""
+        )
+
+    return "\n".join(cards)
+
+def build_correlation_table(
+    report: Report,
+) -> str:
+    """
+    Build correlation table.
+    """
+
+    correlation = report.correlation()
+
+    rows: list[str] = []
+
+    if (
+        not correlation.strong_positive_pairs
+        and not correlation.strong_negative_pairs
+    ):
+        return (
+            "<p>No strong correlations were detected.</p>"
+        )
+
+    rows.append(
+        """
+<table class="report-table">
+    <thead>
+        <tr>
+            <th>Relationship</th>
+            <th>Type</th>
+            <th>Correlation</th>
+        </tr>
+    </thead>
+    <tbody>
+"""
+    )
+
+    for pair, value in (
+        correlation.strong_positive_pairs.items()
+    ):
+        rows.append(
+            f"""
+<tr>
+    <td>{pair}</td>
+    <td>Positive</td>
+    <td>{value:.2f}</td>
+</tr>
+"""
+        )
+
+    for pair, value in (
+        correlation.strong_negative_pairs.items()
+    ):
+        rows.append(
+            f"""
+<tr>
+    <td>{pair}</td>
+    <td>Negative</td>
+    <td>{value:.2f}</td>
+</tr>
+"""
+        )
+
+    rows.append(
+        """
+    </tbody>
+</table>
+"""
+    )
+
+    return "\n".join(rows)
+
+def build_insights(
+    report: Report,
+) -> str:
+    """
+    Build insights HTML.
+    """
+
+    insight_summary = report.insights()
+
+    rows: list[str] = []
+
+    rows.append("<ul>")
+
+    for insight in insight_summary.insights:
+        rows.append(
+            f"<li>{insight}</li>"
+        )
+
+    rows.append("</ul>")
+
+    return "\n".join(rows)
+
+def build_insight_recommendations(
+    report: Report,
+) -> str:
+    """
+    Build recommendation HTML.
+    """
+
+    insight_summary = report.insights()
+
+    rows: list[str] = []
+
+    rows.append("<ul>")
+
+    for recommendation in (
+        insight_summary.recommendations
+    ):
+        rows.append(
+            f"<li>{recommendation}</li>"
+        )
+
+    rows.append("</ul>")
+
+    return "\n".join(rows)

datapilot/reporting/html.py

@@ -0,0 +1,51 @@
+"""
+HTML report generation for Datapilot.
+"""
+
+import shutil
+from pathlib import Path
+
+from datapilot.core.report import Report
+
+from .renderer import HTMLRenderer
+
+
+def generate_html_report(
+    report: Report,
+    output_path: str | Path,
+) -> None:
+    """
+    Generate a Datapilot HTML report.
+    """
+
+    output_path = Path(output_path)
+
+    project_root = (
+        Path(__file__).parent.parent
+    )
+
+    template = (
+        project_root
+        / "templates"
+        / "report.html"
+    )
+
+    css_source = (
+        project_root
+        / "assets"
+        / "style.css"
+    )
+
+    renderer = HTMLRenderer(
+        template_path=template,
+    )
+
+    renderer.render(
+        report=report,
+        output_path=output_path,
+    )
+
+    shutil.copy2(
+        css_source,
+        output_path.parent / "style.css",
+    )

datapilot/reporting/placeholders.py

@@ -0,0 +1,133 @@
+"""
+Placeholder generation for Datapilot HTML reports.
+"""
+
+from datetime import datetime
+
+from datapilot.core.report import Report
+
+from .fragments import (
+    build_column_name_chips,
+    build_correlation_table,
+    build_health_headline,
+    build_health_summary,
+    build_html_list,
+    build_insight_recommendations,
+    build_insights,
+    build_missing_table,
+    build_outlier_cards,
+    build_statistics_table,
+)
+
+DATAPILOT_VERSION = "0.2.0"
+
+
+def build_placeholders(
+    report: Report,
+) -> dict[str, str]:
+    """
+    Build placeholder dictionary for the HTML template.
+    """
+
+    summary = report.summary()
+    health = report.dataset_health()
+    missing = report.missing_values()
+    duplicates = report.duplicates()
+
+    health_class = (
+        "kpi-card--good"
+        if health.score >= 90
+        else "kpi-card--warn"
+        if health.score >= 75
+        else "kpi-card--bad"
+    )
+
+    return {
+        "{{DATASET_NAME}}": "Dataset",
+        "{{REPORT_GENERATED_AT}}": datetime.now().strftime(
+            "%d %b %Y %H:%M",
+        ),
+        "{{DATAPILOT_VERSION}}": DATAPILOT_VERSION,
+        "{{REPORT_SUMMARY}}": (
+            "Automated Data Quality Assessment Report"
+        ),
+
+        "{{ROWS}}": str(summary.rows),
+        "{{COLUMNS}}": str(summary.columns),
+        "{{MEMORY_USAGE}}": (
+            f"{summary.memory_usage_mb:.2f} MB"
+        ),
+        "{{FILE_FORMAT}}": "CSV",
+
+        "{{COLUMN_NAME_CHIPS}}": (
+            build_column_name_chips(report)
+        ),
+
+        "{{HEALTH_SCORE}}": str(health.score),
+        "{{GRADE}}": health.grade,
+        "{{HEALTH_STATUS}}": health.status,
+
+        "{{HEALTH_HEADLINE}}": (
+            build_health_headline(report)
+        ),
+        "{{HEALTH_SUMMARY}}": (
+            build_health_summary(report)
+        ),
+
+        "{{STRENGTHS_LIST}}": (
+            build_html_list(
+                health.strengths,
+            )
+        ),
+        "{{WEAKNESSES_LIST}}": (
+            build_html_list(
+                health.weaknesses,
+            )
+        ),
+        "{{RECOMMENDATIONS_LIST}}": (
+            build_html_list(
+                health.recommendations,
+            )
+        ),
+
+        "{{HEALTH_SCORE_SEVERITY_CLASS}}": health_class,
+        "{{HEALTH_SCORE_SEVERITY_CLASS_GAUGE}}": health_class,
+
+        "{{MISSING_VALUES_COUNT}}": str(
+            missing.total_missing,
+        ),
+        "{{MISSING_VALUES_PERCENT}}": (
+            f"{missing.missing_percentage:.1f}"
+        ),
+
+        "{{MISSING_TABLE}}": (
+            build_missing_table(report)
+        ),
+
+        "{{STATISTICS_TABLE}}": (
+            build_statistics_table(report)
+        ),
+
+        "{{OUTLIER_CARDS}}": (
+            build_outlier_cards(report)
+        ),
+
+        "{{CORRELATION_TABLE}}": (
+            build_correlation_table(report)
+        ),
+
+        "{{INSIGHTS}}": (
+            build_insights(report)
+        ),
+
+        "{{RECOMMENDATIONS}}": (
+            build_insight_recommendations(report)
+        ),
+
+        "{{DUPLICATE_ROWS_COUNT}}": str(
+            duplicates.total_duplicates,
+        ),
+        "{{DUPLICATE_ROWS_PERCENT}}": (
+            f"{duplicates.duplicate_percentage:.1f}"
+        ),
+    }