dqdetect 0.1.0__py3-none-any.whl

dqdetect/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

dqdetect/__main__.py

@@ -0,0 +1,5 @@
+from dqdetect.cli import main
+
+
+if __name__ == "__main__":
+    main()

dqdetect/checks.py

@@ -0,0 +1,69 @@
+import pandas as pd
+from pandas.api.types import is_numeric_dtype
+
+
+DATE_NAME_HINTS = (
+    "date",
+    "time",
+    "timestamp",
+    "created",
+    "updated",
+    "dob",
+)
+
+
+def is_date_like_column(column_name):
+    name = str(column_name).lower()
+    return any(hint in name for hint in DATE_NAME_HINTS)
+
+
+def mixed_numeric_text_count(series):
+    if is_numeric_dtype(series):
+        return 0
+
+    values = series.dropna().astype(str).str.strip()
+    values = values[values.ne("")]
+
+    if values.empty:
+        return 0
+
+    parsed = pd.to_numeric(values, errors="coerce")
+    numeric_count = int(parsed.notna().sum())
+    text_count = int(parsed.isna().sum())
+
+    if numeric_count == 0 or text_count == 0:
+        return 0
+
+    return min(numeric_count, text_count)
+
+
+def invalid_date_count(series, column_name):
+    if not is_date_like_column(column_name):
+        return 0
+
+    values = series.dropna()
+    if values.empty:
+        return 0
+
+    parsed = pd.to_datetime(values, errors="coerce")
+    return int(parsed.isna().sum())
+
+
+def iqr_outlier_count(series):
+    if not is_numeric_dtype(series):
+        return 0
+
+    values = series.dropna()
+    if len(values) < 4:
+        return 0
+
+    q1 = values.quantile(0.25)
+    q3 = values.quantile(0.75)
+    iqr = q3 - q1
+
+    if pd.isna(iqr) or iqr == 0:
+        return 0
+
+    lower = q1 - 1.5 * iqr
+    upper = q3 + 1.5 * iqr
+    return int(((values < lower) | (values > upper)).sum())

dqdetect/cli.py

@@ -0,0 +1,106 @@
+import argparse
+from pathlib import Path
+
+from dqdetect.io import load_data
+from dqdetect.profiler import profile_dataframe
+from dqdetect.report import write_reports
+
+
+def _issue_line(count, singular_subject, plural_subject, issue):
+    subject = singular_subject if count == 1 else plural_subject
+    verb = "contains" if count == 1 else "contain"
+    return f"- {count} {subject} {verb} {issue}"
+
+
+def build_parser():
+    parser = argparse.ArgumentParser(
+        prog="dqdetect",
+        description="Profile a CSV or Excel file for common data-quality issues.",
+    )
+    parser.add_argument("file", help="Path to a CSV or Excel file")
+    parser.add_argument(
+        "--sheet",
+        help="Excel sheet name or index. Ignored for CSV files.",
+    )
+    parser.add_argument(
+        "--output",
+        default="reports",
+        help="Directory for generated reports (default: reports)",
+    )
+    parser.add_argument(
+        "--format",
+        choices=["md", "html", "both"],
+        default="both",
+        help="Report format (default: both)",
+    )
+    return parser
+
+
+def main(argv=None):
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    source = Path(args.file)
+
+    try:
+        dataframe = load_data(source, sheet=args.sheet)
+    except (FileNotFoundError, ValueError) as exc:
+        parser.error(str(exc))
+
+    profile = profile_dataframe(dataframe, source_name=source.name)
+    summary = profile["summary"]
+    written = write_reports(
+        profile,
+        output_dir=args.output,
+        stem=source.stem,
+        report_format=args.format,
+    )
+
+    print("Data Quality Detective")
+    print(f"File: {source}")
+    print()
+    print(f"Rows: {summary['rows']:,}")
+    print(f"Columns: {summary['columns']:,}")
+    print(f"Duplicate rows: {summary['duplicate_rows']:,}")
+    print()
+    print("Issues found")
+    print(
+        _issue_line(
+            summary["columns_with_missing"],
+            "column",
+            "columns",
+            "missing values",
+        )
+    )
+    print(
+        _issue_line(
+            summary["mixed_numeric_text_columns"],
+            "column",
+            "columns",
+            "mixed numeric/text values",
+        )
+    )
+    print(
+        _issue_line(
+            summary["date_columns_with_invalid_values"],
+            "date-like column",
+            "date-like columns",
+            "invalid values",
+        )
+    )
+    print(
+        _issue_line(
+            summary["numeric_columns_with_outliers"],
+            "numeric column",
+            "numeric columns",
+            "potential IQR outliers",
+        )
+    )
+    print()
+    print("Reports written to:")
+    for path in written:
+        print(f"  {path}")
+
+
+if __name__ == "__main__":
+    main()

dqdetect/io.py

@@ -0,0 +1,35 @@
+from pathlib import Path
+
+import pandas as pd
+
+
+SUPPORTED_SUFFIXES = {".csv", ".xlsx"}
+
+
+def _normalize_sheet(sheet):
+    if sheet is None:
+        return 0
+
+    if isinstance(sheet, str) and sheet.isdigit():
+        return int(sheet)
+
+    return sheet
+
+
+def load_data(path, sheet=None):
+    source = Path(path)
+
+    if not source.exists():
+        raise FileNotFoundError(f"File not found: {source}")
+
+    suffix = source.suffix.lower()
+    if suffix not in SUPPORTED_SUFFIXES:
+        supported = ", ".join(sorted(SUPPORTED_SUFFIXES))
+        raise ValueError(
+            f"Unsupported file type '{suffix}'. Supported types: {supported}"
+        )
+
+    if suffix == ".csv":
+        return pd.read_csv(source)
+
+    return pd.read_excel(source, sheet_name=_normalize_sheet(sheet))

dqdetect/profiler.py

@@ -0,0 +1,86 @@
+from pathlib import Path
+
+import pandas as pd
+
+from dqdetect.checks import (
+    invalid_date_count,
+    iqr_outlier_count,
+    mixed_numeric_text_count,
+)
+
+
+def _sample_values(series, limit=3):
+    values = series.dropna().astype(str).drop_duplicates().head(limit)
+    return [value[:60] for value in values]
+
+
+def profile_dataframe(df, source_name="dataset"):
+    row_count, column_count = df.shape
+    duplicate_rows = int(df.duplicated().sum())
+
+    columns = []
+
+    for column in df.columns:
+        series = df[column]
+        missing_count = int(series.isna().sum())
+        unique_count = int(series.nunique(dropna=True))
+        mixed_count = mixed_numeric_text_count(series)
+        invalid_dates = invalid_date_count(series, column)
+        outliers = iqr_outlier_count(series)
+        is_constant = unique_count <= 1
+
+        issues = []
+        if missing_count:
+            issues.append("missing values")
+        if is_constant:
+            issues.append("constant column")
+        if mixed_count:
+            issues.append("mixed numeric/text")
+        if invalid_dates:
+            issues.append("invalid date values")
+        if outliers:
+            issues.append("potential outliers")
+
+        columns.append(
+            {
+                "column": str(column),
+                "dtype": str(series.dtype),
+                "missing_count": missing_count,
+                "missing_pct": round(
+                    (missing_count / row_count * 100) if row_count else 0.0,
+                    2,
+                ),
+                "unique_count": unique_count,
+                "constant": is_constant,
+                "mixed_numeric_text_count": mixed_count,
+                "invalid_date_count": invalid_dates,
+                "outlier_count": outliers,
+                "sample_values": _sample_values(series),
+                "issues": issues,
+            }
+        )
+
+    summary = {
+        "source": Path(source_name).name,
+        "rows": row_count,
+        "columns": column_count,
+        "duplicate_rows": duplicate_rows,
+        "columns_with_missing": sum(
+            item["missing_count"] > 0 for item in columns
+        ),
+        "constant_columns": sum(item["constant"] for item in columns),
+        "mixed_numeric_text_columns": sum(
+            item["mixed_numeric_text_count"] > 0 for item in columns
+        ),
+        "date_columns_with_invalid_values": sum(
+            item["invalid_date_count"] > 0 for item in columns
+        ),
+        "numeric_columns_with_outliers": sum(
+            item["outlier_count"] > 0 for item in columns
+        ),
+    }
+
+    return {
+        "summary": summary,
+        "columns": columns,
+    }

dqdetect/report.py

@@ -0,0 +1,187 @@
+from datetime import datetime, timezone
+from html import escape
+from pathlib import Path
+
+
+def _recommendations(profile):
+    summary = profile["summary"]
+    recommendations = []
+
+    if summary["duplicate_rows"]:
+        recommendations.append(
+            "Review duplicate rows before aggregating or counting records."
+        )
+    if summary["columns_with_missing"]:
+        recommendations.append(
+            "Review missing values by column before deciding whether to impute, exclude, or preserve them."
+        )
+    if summary["mixed_numeric_text_columns"]:
+        recommendations.append(
+            "Keep mixed numeric/text source values separate from any cleaned numeric field instead of coercing text to zero."
+        )
+    if summary["date_columns_with_invalid_values"]:
+        recommendations.append(
+            "Inspect invalid date values before calculating durations or time-based KPIs."
+        )
+    if summary["numeric_columns_with_outliers"]:
+        recommendations.append(
+            "Investigate outliers before removing them; unusual values may be valid business events."
+        )
+    if summary["constant_columns"]:
+        recommendations.append(
+            "Constant columns may not add analytical value and can often be excluded from modeling."
+        )
+
+    if not recommendations:
+        recommendations.append(
+            "No issues were flagged by the current rules. Domain-specific validation may still be needed."
+        )
+
+    return recommendations
+
+
+def render_markdown(profile):
+    summary = profile["summary"]
+    generated = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
+
+    lines = [
+        "# Data Quality Report",
+        "",
+        f"**Source:** {summary['source']}",
+        f"**Generated:** {generated}",
+        "",
+        "## Dataset summary",
+        "",
+        "| Metric | Value |",
+        "|---|---:|",
+        f"| Rows | {summary['rows']:,} |",
+        f"| Columns | {summary['columns']:,} |",
+        f"| Duplicate rows | {summary['duplicate_rows']:,} |",
+        f"| Columns with missing values | {summary['columns_with_missing']} |",
+        f"| Constant columns | {summary['constant_columns']} |",
+        f"| Mixed numeric/text columns | {summary['mixed_numeric_text_columns']} |",
+        f"| Date columns with invalid values | {summary['date_columns_with_invalid_values']} |",
+        f"| Numeric columns with potential outliers | {summary['numeric_columns_with_outliers']} |",
+        "",
+        "## Column profile",
+        "",
+        "| Column | Type | Missing | Unique | Mixed | Invalid dates | Outliers | Issues |",
+        "|---|---|---:|---:|---:|---:|---:|---|",
+    ]
+
+    for item in profile["columns"]:
+        issues = ", ".join(item["issues"]) if item["issues"] else "none"
+        lines.append(
+            "| {column} | {dtype} | {missing_count} ({missing_pct:.2f}%) | "
+            "{unique_count} | {mixed_numeric_text_count} | {invalid_date_count} | "
+            "{outlier_count} | {issues_text} |".format(**item, issues_text=issues)
+        )
+
+    lines.extend(["", "## Recommendations", ""])
+    for recommendation in _recommendations(profile):
+        lines.append(f"- {recommendation}")
+
+    lines.extend(
+        [
+            "",
+            "## Notes",
+            "",
+            "The checks in this report are screening rules, not automatic cleaning instructions. Business context should decide what gets changed.",
+            "",
+        ]
+    )
+
+    return "\n".join(lines)
+
+
+def render_html(profile):
+    summary = profile["summary"]
+    generated = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
+
+    rows = []
+    for item in profile["columns"]:
+        issues = ", ".join(item["issues"]) if item["issues"] else "none"
+        rows.append(
+            "<tr>"
+            f"<td>{escape(item['column'])}</td>"
+            f"<td>{escape(item['dtype'])}</td>"
+            f"<td>{item['missing_count']} ({item['missing_pct']:.2f}%)</td>"
+            f"<td>{item['unique_count']}</td>"
+            f"<td>{item['mixed_numeric_text_count']}</td>"
+            f"<td>{item['invalid_date_count']}</td>"
+            f"<td>{item['outlier_count']}</td>"
+            f"<td>{escape(issues)}</td>"
+            "</tr>"
+        )
+
+    recommendation_items = "".join(
+        f"<li>{escape(item)}</li>" for item in _recommendations(profile)
+    )
+
+    return f"""<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Data Quality Report</title>
+<style>
+body {{ font-family: Arial, sans-serif; max-width: 1100px; margin: 40px auto; padding: 0 24px; color: #202124; }}
+h1, h2 {{ color: #172554; }}
+.cards {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); gap: 12px; margin: 24px 0; }}
+.card {{ border: 1px solid #d9dee8; border-radius: 10px; padding: 16px; }}
+.value {{ font-size: 1.5rem; font-weight: 700; margin-top: 6px; }}
+table {{ border-collapse: collapse; width: 100%; font-size: 0.9rem; }}
+th, td {{ border-bottom: 1px solid #e5e7eb; padding: 10px; text-align: left; vertical-align: top; }}
+th {{ background: #f8fafc; }}
+.note {{ background: #f8fafc; border-left: 4px solid #64748b; padding: 12px 16px; margin-top: 24px; }}
+</style>
+</head>
+<body>
+<h1>Data Quality Report</h1>
+<p><strong>Source:</strong> {escape(summary['source'])}<br>
+<strong>Generated:</strong> {generated}</p>
+
+<div class="cards">
+  <div class="card">Rows<div class="value">{summary['rows']:,}</div></div>
+  <div class="card">Columns<div class="value">{summary['columns']}</div></div>
+  <div class="card">Duplicate rows<div class="value">{summary['duplicate_rows']}</div></div>
+  <div class="card">Columns with missing<div class="value">{summary['columns_with_missing']}</div></div>
+</div>
+
+<h2>Column profile</h2>
+<table>
+<thead>
+<tr><th>Column</th><th>Type</th><th>Missing</th><th>Unique</th><th>Mixed</th><th>Invalid dates</th><th>Outliers</th><th>Issues</th></tr>
+</thead>
+<tbody>
+{''.join(rows)}
+</tbody>
+</table>
+
+<h2>Recommendations</h2>
+<ul>{recommendation_items}</ul>
+
+<div class="note">
+These checks are screening rules, not automatic cleaning instructions. Business context should decide what gets changed.
+</div>
+</body>
+</html>
+"""
+
+
+def write_reports(profile, output_dir, stem, report_format="both"):
+    output = Path(output_dir)
+    output.mkdir(parents=True, exist_ok=True)
+    written = []
+
+    if report_format in {"md", "both"}:
+        markdown_path = output / f"{stem}_report.md"
+        markdown_path.write_text(render_markdown(profile), encoding="utf-8")
+        written.append(markdown_path)
+
+    if report_format in {"html", "both"}:
+        html_path = output / f"{stem}_report.html"
+        html_path.write_text(render_html(profile), encoding="utf-8")
+        written.append(html_path)
+
+    return written

dqdetect-0.1.0.dist-info/METADATA

@@ -0,0 +1,153 @@
+Metadata-Version: 2.4
+Name: dqdetect
+Version: 0.1.0
+Summary: Command-line data quality profiler for CSV and Excel files
+Author: Shahed
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/Shahedr/data-quality-detective
+Project-URL: Repository, https://github.com/Shahedr/data-quality-detective
+Project-URL: Issues, https://github.com/Shahedr/data-quality-detective/issues
+Keywords: data-quality,data-profiling,csv,excel,analytics,pandas
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas>=2.2
+Requires-Dist: openpyxl>=3.1
+Provides-Extra: dev
+Requires-Dist: build>=1.2; extra == "dev"
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: twine>=6.0; extra == "dev"
+Dynamic: license-file
+
+# Data Quality Detective
+
+[![tests](https://github.com/Shahedr/data-quality-detective/actions/workflows/tests.yml/badge.svg)](https://github.com/Shahedr/data-quality-detective/actions/workflows/tests.yml) ![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue) ![MIT License](https://img.shields.io/badge/license-MIT-green)
+
+A small command-line tool for quickly profiling CSV and Excel files before analysis.
+
+I built this around a problem I keep running into in analytics work: a dataset can look usable at first, but the real issues only show up after checking missing values, duplicate rows, mixed numeric/text fields, date parsing, and unusual numeric values.
+
+**dqdetect** puts those checks into one repeatable command and writes a report you can review or share.
+
+## Quick start
+
+~~~bash
+git clone https://github.com/Shahedr/data-quality-detective.git
+cd data-quality-detective
+
+python -m venv .venv
+# macOS/Linux
+source .venv/bin/activate
+# Windows
+# .venv/Scripts/activate
+
+pip install -e .
+dqdetect examples/messy_orders.csv
+~~~
+
+By default the command writes both Markdown and HTML reports to the **reports/** folder.
+
+## What it checks
+
+- dataset shape and column types
+- duplicate rows
+- missing values by column
+- constant columns
+- mixed numeric/text values
+- invalid values in date-like columns
+- IQR-based outlier counts for numeric columns
+
+The tool does **not** automatically delete or "fix" anything. The report is meant to help an analyst decide what deserves review.
+
+## Example
+
+~~~text
+Data Quality Detective
+File: examples/messy_orders.csv
+
+Rows: 12
+Columns: 8
+Duplicate rows: 1
+
+Reports written to:
+  reports/messy_orders_report.md
+  reports/messy_orders_report.html
+~~~
+
+See [examples/messy_orders_report.md](examples/messy_orders_report.md) for a sample report.
+
+## CLI
+
+~~~bash
+dqdetect path/to/file.csv
+dqdetect workbook.xlsx --sheet Orders\ndqdetect workbook.xlsx --sheet 0
+dqdetect data.csv --format html
+dqdetect data.csv --output audit_reports
+~~~
+
+Supported formats:
+
+- CSV
+- Excel (.xlsx)
+
+## Why these checks?
+
+This first version focuses on issues that can change an analysis if they are handled carelessly.
+
+For example, a freight-cost field containing both dollar values and text such as "included elsewhere" should not simply be converted to zero. Likewise, an extreme numeric value may be a data-entry problem or a legitimate observation. The tool flags these cases instead of making the decision for you.
+
+## Project structure
+
+~~~text
+data-quality-detective/
+├── dqdetect/
+│   ├── __init__.py
+│   ├── checks.py
+│   ├── cli.py
+│   ├── io.py
+│   ├── profiler.py
+│   └── report.py
+├── examples/
+│   ├── messy_orders.csv
+│   └── messy_orders_report.md
+├── tests/
+├── .github/workflows/tests.yml
+├── CONTRIBUTING.md
+├── LICENSE
+└── pyproject.toml
+~~~
+
+## Development
+
+~~~bash
+pip install -e ".[dev]"
+pytest
+~~~
+
+## Roadmap
+
+Things I would like to add next:
+
+- user-configurable thresholds
+- JSON output for automation
+- schema rules for expected columns and types
+- PostgreSQL table profiling
+- comparison between two versions of a dataset
+- richer HTML charts
+- optional GitHub Action for automated data-quality checks
+
+I am keeping the first release intentionally small so the checks are understandable and easy to extend.
+
+## License
+
+MIT

dqdetect-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+dqdetect/__init__.py,sha256=kUR5RAFc7HCeiqdlX36dZOHkUI5wI6V_43RpEcD8b-0,22
+dqdetect/__main__.py,sha256=sSpaYFy6kdpeNlcrd4cKpUPtg9H_obwQI2dBMcn0aoQ,70
+dqdetect/checks.py,sha256=TJiRvliE4SxxxKIeYFf375mW4KEskGX4UEKLyKJfszU,1443
+dqdetect/cli.py,sha256=7JouCNS37dYGvXnf2J6FyBCc6q0U5weTv8rAawJB-SI,2744
+dqdetect/io.py,sha256=ZcFPmL29Edf0U1wrrJTEn_RWjr6CHhIEs7Gb4Cewhk0,782
+dqdetect/profiler.py,sha256=yQVjzGAWEyNXia8WFgNxXppmK0RWrMxdD4xGWPUmWSM,2669
+dqdetect/report.py,sha256=RQD6eHCOfyyrZZuVk48hCdlutqC_cGemZRyjPaKlCpQ,6845
+dqdetect-0.1.0.dist-info/licenses/LICENSE,sha256=wji_CJxt9mbB6XDfopRCAmFqY1X-p_AKNlriT9YNy_I,1063
+dqdetect-0.1.0.dist-info/METADATA,sha256=8ZEIBRjjdfDEKgs3lojhSYPGCo16qI4SD2jDbD8cGPk,4651
+dqdetect-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+dqdetect-0.1.0.dist-info/entry_points.txt,sha256=ZF_jCESrxtDUUw6z7mKyVicpaANoCcCV4EF19EtyCQc,47
+dqdetect-0.1.0.dist-info/top_level.txt,sha256=YekEoq6Y2Ho9r7uDXPXNrhJYkvmLYGmMHx5ssSg6oo0,9
+dqdetect-0.1.0.dist-info/RECORD,,

dqdetect-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

dqdetect-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+dqdetect = dqdetect.cli:main

dqdetect-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Shahed
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dqdetect-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+dqdetect