csvguard 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- csvguard/__init__.py +12 -0
- csvguard/cleaner.py +97 -0
- csvguard/cli.py +87 -0
- csvguard/profiler.py +119 -0
- csvguard/reporter.py +147 -0
- csvguard-0.1.0.dist-info/METADATA +122 -0
- csvguard-0.1.0.dist-info/RECORD +10 -0
- csvguard-0.1.0.dist-info/WHEEL +5 -0
- csvguard-0.1.0.dist-info/entry_points.txt +2 -0
- csvguard-0.1.0.dist-info/top_level.txt +1 -0
csvguard/__init__.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""
|
|
2
|
+
csvguard: The Fast, Terminal-First Data Quality & Cleaning CLI for Python
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
__version__ = "0.1.0"
|
|
6
|
+
__author__ = "Sumit"
|
|
7
|
+
__license__ = "MIT"
|
|
8
|
+
|
|
9
|
+
from .profiler import audit
|
|
10
|
+
from .cleaner import clean
|
|
11
|
+
|
|
12
|
+
__all__ = ["audit", "clean", "__version__"]
|
csvguard/cleaner.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
import pandas as pd
|
|
4
|
+
import numpy as np
|
|
5
|
+
|
|
6
|
+
def to_snake_case(text):
|
|
7
|
+
text = str(text).strip()
|
|
8
|
+
text = re.sub(r'[^\w\s-]', '', text)
|
|
9
|
+
text = re.sub(r'[-\s]+', '_', text)
|
|
10
|
+
return text.lower()
|
|
11
|
+
|
|
12
|
+
def clean(
|
|
13
|
+
file_path,
|
|
14
|
+
output_path=None,
|
|
15
|
+
standardize_headers=True,
|
|
16
|
+
drop_duplicates=True,
|
|
17
|
+
impute_numeric="median",
|
|
18
|
+
impute_categorical="Unknown",
|
|
19
|
+
clip_outliers=False
|
|
20
|
+
):
|
|
21
|
+
if not os.path.exists(file_path):
|
|
22
|
+
raise FileNotFoundError(f"File not found: {file_path}")
|
|
23
|
+
|
|
24
|
+
df = pd.read_csv(file_path)
|
|
25
|
+
orig_rows, orig_cols = df.shape
|
|
26
|
+
changes = []
|
|
27
|
+
|
|
28
|
+
# 1. Standardize Headers
|
|
29
|
+
if standardize_headers:
|
|
30
|
+
old_cols = list(df.columns)
|
|
31
|
+
new_cols = [to_snake_case(c) for c in old_cols]
|
|
32
|
+
renamed_count = sum(1 for o, n in zip(old_cols, new_cols) if o != n)
|
|
33
|
+
df.columns = new_cols
|
|
34
|
+
if renamed_count > 0:
|
|
35
|
+
changes.append(f"Standardized {renamed_count} column header(s) to snake_case")
|
|
36
|
+
|
|
37
|
+
# 2. Drop Duplicates
|
|
38
|
+
if drop_duplicates:
|
|
39
|
+
dups = df.duplicated().sum()
|
|
40
|
+
if dups > 0:
|
|
41
|
+
df = df.drop_duplicates()
|
|
42
|
+
changes.append(f"Removed {dups} duplicate row(s)")
|
|
43
|
+
|
|
44
|
+
# 3. Impute Numeric Columns
|
|
45
|
+
if impute_numeric in ["median", "mean"]:
|
|
46
|
+
num_imputed = 0
|
|
47
|
+
for col in df.select_dtypes(include=[np.number]).columns:
|
|
48
|
+
missing = df[col].isna().sum()
|
|
49
|
+
if missing > 0:
|
|
50
|
+
val = df[col].median() if impute_numeric == "median" else df[col].mean()
|
|
51
|
+
df[col] = df[col].fillna(round(val, 2))
|
|
52
|
+
num_imputed += missing
|
|
53
|
+
if num_imputed > 0:
|
|
54
|
+
changes.append(f"Imputed {num_imputed} missing numeric values using {impute_numeric}")
|
|
55
|
+
|
|
56
|
+
# 4. Impute Categorical Columns
|
|
57
|
+
if impute_categorical:
|
|
58
|
+
cat_imputed = 0
|
|
59
|
+
for col in df.select_dtypes(include=['object', 'category']).columns:
|
|
60
|
+
missing = df[col].isna().sum()
|
|
61
|
+
if missing > 0:
|
|
62
|
+
df[col] = df[col].fillna(impute_categorical)
|
|
63
|
+
cat_imputed += missing
|
|
64
|
+
if cat_imputed > 0:
|
|
65
|
+
changes.append(f"Imputed {cat_imputed} missing text values with '{impute_categorical}'")
|
|
66
|
+
|
|
67
|
+
# 5. Clip Outliers (Optional)
|
|
68
|
+
if clip_outliers:
|
|
69
|
+
outliers_clipped = 0
|
|
70
|
+
for col in df.select_dtypes(include=[np.number]).columns:
|
|
71
|
+
if df[col].dropna().shape[0] > 4:
|
|
72
|
+
q1 = df[col].quantile(0.25)
|
|
73
|
+
q3 = df[col].quantile(0.75)
|
|
74
|
+
iqr = q3 - q1
|
|
75
|
+
lower = q1 - 1.5 * iqr
|
|
76
|
+
upper = q3 + 1.5 * iqr
|
|
77
|
+
|
|
78
|
+
mask = (df[col] < lower) | (df[col] > upper)
|
|
79
|
+
outliers_clipped += int(mask.sum())
|
|
80
|
+
df[col] = df[col].clip(lower, upper)
|
|
81
|
+
if outliers_clipped > 0:
|
|
82
|
+
changes.append(f"Clipped {outliers_clipped} extreme outlier(s) using IQR fences")
|
|
83
|
+
|
|
84
|
+
# Export
|
|
85
|
+
if not output_path:
|
|
86
|
+
base, ext = os.path.splitext(file_path)
|
|
87
|
+
output_path = f"{base}_cleaned{ext}"
|
|
88
|
+
|
|
89
|
+
df.to_csv(output_path, index=False)
|
|
90
|
+
|
|
91
|
+
return {
|
|
92
|
+
"output_path": output_path,
|
|
93
|
+
"original_rows": orig_rows,
|
|
94
|
+
"final_rows": df.shape[0],
|
|
95
|
+
"columns": df.shape[1],
|
|
96
|
+
"changes": changes
|
|
97
|
+
}
|
csvguard/cli.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import argparse
|
|
3
|
+
|
|
4
|
+
# Ensure UTF-8 output encoding across Windows terminals
|
|
5
|
+
if sys.stdout.encoding != 'utf-8':
|
|
6
|
+
try:
|
|
7
|
+
sys.stdout.reconfigure(encoding='utf-8', errors='replace')
|
|
8
|
+
sys.stderr.reconfigure(encoding='utf-8', errors='replace')
|
|
9
|
+
except Exception:
|
|
10
|
+
pass
|
|
11
|
+
|
|
12
|
+
from rich.console import Console
|
|
13
|
+
from .profiler import audit
|
|
14
|
+
from .cleaner import clean
|
|
15
|
+
from .reporter import render_terminal_audit, export_markdown_report
|
|
16
|
+
from . import __version__
|
|
17
|
+
|
|
18
|
+
console = Console(force_terminal=True, legacy_windows=False)
|
|
19
|
+
|
|
20
|
+
def main():
|
|
21
|
+
parser = argparse.ArgumentParser(
|
|
22
|
+
prog="csvguard",
|
|
23
|
+
description="csvguard: The Fast, Terminal-First Data Quality & Cleaning CLI"
|
|
24
|
+
)
|
|
25
|
+
parser.add_argument("-v", "--version", action="version", version=f"csvguard {__version__}")
|
|
26
|
+
|
|
27
|
+
subparsers = parser.add_subparsers(dest="command", help="Available sub-commands")
|
|
28
|
+
|
|
29
|
+
# 1. audit sub-command
|
|
30
|
+
audit_parser = subparsers.add_parser("audit", help="Audit a CSV file and display an interactive terminal health report")
|
|
31
|
+
audit_parser.add_argument("file", help="Path to the CSV file to audit")
|
|
32
|
+
|
|
33
|
+
# 2. clean sub-command
|
|
34
|
+
clean_parser = subparsers.add_parser("clean", help="Clean, sanitize, and impute issues in a CSV file")
|
|
35
|
+
clean_parser.add_argument("file", help="Path to the CSV file to clean")
|
|
36
|
+
clean_parser.add_argument("-o", "--output", help="Output path for the cleaned CSV file")
|
|
37
|
+
clean_parser.add_argument("--auto", action="store_true", help="Apply standard best-practice cleaning (drop dups, snake_case headers, median imputation)")
|
|
38
|
+
clean_parser.add_argument("--no-duplicates", action="store_true", help="Do not drop duplicates")
|
|
39
|
+
clean_parser.add_argument("--impute-num", choices=["median", "mean"], default="median", help="Method for missing numeric imputation")
|
|
40
|
+
clean_parser.add_argument("--impute-cat", default="Unknown", help="Value to fill missing categorical cells with")
|
|
41
|
+
clean_parser.add_argument("--clip-outliers", action="store_true", help="Clip extreme numeric outliers using IQR fences")
|
|
42
|
+
|
|
43
|
+
# 3. report sub-command
|
|
44
|
+
report_parser = subparsers.add_parser("report", help="Generate and export a Markdown health report")
|
|
45
|
+
report_parser.add_argument("file", help="Path to the CSV file to inspect")
|
|
46
|
+
report_parser.add_argument("-o", "--output", default="csvguard_report.md", help="Output path for Markdown report (default: csvguard_report.md)")
|
|
47
|
+
|
|
48
|
+
args = parser.parse_args()
|
|
49
|
+
|
|
50
|
+
if not args.command:
|
|
51
|
+
parser.print_help()
|
|
52
|
+
sys.exit(0)
|
|
53
|
+
|
|
54
|
+
try:
|
|
55
|
+
if args.command == "audit":
|
|
56
|
+
profile = audit(args.file)
|
|
57
|
+
render_terminal_audit(profile)
|
|
58
|
+
|
|
59
|
+
elif args.command == "clean":
|
|
60
|
+
result = clean(
|
|
61
|
+
file_path=args.file,
|
|
62
|
+
output_path=args.output,
|
|
63
|
+
standardize_headers=True,
|
|
64
|
+
drop_duplicates=not args.no_duplicates,
|
|
65
|
+
impute_numeric=args.impute_num,
|
|
66
|
+
impute_categorical=args.impute_cat,
|
|
67
|
+
clip_outliers=args.clip_outliers or args.auto
|
|
68
|
+
)
|
|
69
|
+
console.print("\n[bold green]Cleaning Successfully Completed![/bold green]")
|
|
70
|
+
console.print(f"[bold]Saved cleaned file to:[/bold] [cyan]{result['output_path']}[/cyan]")
|
|
71
|
+
console.print(f"[bold]Rows:[/bold] {result['original_rows']} -> [green]{result['final_rows']}[/green] | [bold]Columns:[/bold] {result['columns']}\n")
|
|
72
|
+
console.print("[bold]Modifications applied:[/bold]")
|
|
73
|
+
for ch in result["changes"]:
|
|
74
|
+
console.print(f" * [green]{ch}[/green]")
|
|
75
|
+
console.print("")
|
|
76
|
+
|
|
77
|
+
elif args.command == "report":
|
|
78
|
+
profile = audit(args.file)
|
|
79
|
+
out_path = export_markdown_report(profile, args.output)
|
|
80
|
+
console.print(f"\n[bold green]Audit Report Exported Successfully:[/bold green] [cyan]{out_path}[/cyan]\n")
|
|
81
|
+
|
|
82
|
+
except Exception as e:
|
|
83
|
+
console.print(f"\n[bold red]Error:[/bold red] {str(e)}\n")
|
|
84
|
+
sys.exit(1)
|
|
85
|
+
|
|
86
|
+
if __name__ == "__main__":
|
|
87
|
+
main()
|
csvguard/profiler.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
def detect_delimiter(file_path):
|
|
6
|
+
with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
|
|
7
|
+
first_line = f.readline()
|
|
8
|
+
if ';' in first_line and ',' not in first_line:
|
|
9
|
+
return ';'
|
|
10
|
+
elif '\t' in first_line:
|
|
11
|
+
return '\t'
|
|
12
|
+
elif '|' in first_line:
|
|
13
|
+
return '|'
|
|
14
|
+
return ','
|
|
15
|
+
|
|
16
|
+
def audit(file_path):
|
|
17
|
+
if not os.path.exists(file_path):
|
|
18
|
+
raise FileNotFoundError(f"File not found: {file_path}")
|
|
19
|
+
|
|
20
|
+
delimiter = detect_delimiter(file_path)
|
|
21
|
+
try:
|
|
22
|
+
df = pd.read_csv(file_path, sep=delimiter, encoding='utf-8')
|
|
23
|
+
except UnicodeDecodeError:
|
|
24
|
+
df = pd.read_csv(file_path, sep=delimiter, encoding='latin1')
|
|
25
|
+
|
|
26
|
+
num_rows, num_cols = df.shape
|
|
27
|
+
file_size_kb = round(os.path.getsize(file_path) / 1024, 2)
|
|
28
|
+
|
|
29
|
+
# 1. Duplicates
|
|
30
|
+
dup_count = int(df.duplicated().sum())
|
|
31
|
+
dup_pct = round((dup_count / num_rows) * 100, 2) if num_rows > 0 else 0.0
|
|
32
|
+
|
|
33
|
+
# 2. Column Metrics & Anomalies
|
|
34
|
+
col_profiles = []
|
|
35
|
+
total_missing_cells = 0
|
|
36
|
+
total_outliers = 0
|
|
37
|
+
header_issues = 0
|
|
38
|
+
|
|
39
|
+
for col in df.columns:
|
|
40
|
+
col_clean = str(col).strip()
|
|
41
|
+
if col != col_clean or ' ' in col_clean:
|
|
42
|
+
header_issues += 1
|
|
43
|
+
|
|
44
|
+
series = df[col]
|
|
45
|
+
missing_count = int(series.isna().sum())
|
|
46
|
+
total_missing_cells += missing_count
|
|
47
|
+
missing_pct = round((missing_count / num_rows) * 100, 2) if num_rows > 0 else 0.0
|
|
48
|
+
unique_count = int(series.nunique())
|
|
49
|
+
|
|
50
|
+
# Outlier detection for numeric columns (IQR Method)
|
|
51
|
+
outlier_count = 0
|
|
52
|
+
is_numeric = pd.api.types.is_numeric_dtype(series)
|
|
53
|
+
col_type = "Numeric" if is_numeric else "Text / Categorical"
|
|
54
|
+
|
|
55
|
+
if is_numeric and series.dropna().shape[0] > 4:
|
|
56
|
+
clean_series = series.dropna()
|
|
57
|
+
q1 = clean_series.quantile(0.25)
|
|
58
|
+
q3 = clean_series.quantile(0.75)
|
|
59
|
+
iqr = q3 - q1
|
|
60
|
+
lower_bound = q1 - 1.5 * iqr
|
|
61
|
+
upper_bound = q3 + 1.5 * iqr
|
|
62
|
+
outlier_mask = (clean_series < lower_bound) | (clean_series > upper_bound)
|
|
63
|
+
outlier_count = int(outlier_mask.sum())
|
|
64
|
+
total_outliers += outlier_count
|
|
65
|
+
|
|
66
|
+
status = "Healthy"
|
|
67
|
+
if missing_pct > 20.0 or outlier_count > (0.05 * num_rows):
|
|
68
|
+
status = "Warning"
|
|
69
|
+
elif missing_pct > 50.0:
|
|
70
|
+
status = "Critical"
|
|
71
|
+
|
|
72
|
+
col_profiles.append({
|
|
73
|
+
"name": str(col),
|
|
74
|
+
"type": col_type,
|
|
75
|
+
"missing_count": missing_count,
|
|
76
|
+
"missing_pct": missing_pct,
|
|
77
|
+
"unique_count": unique_count,
|
|
78
|
+
"outlier_count": outlier_count,
|
|
79
|
+
"status": status
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
# 3. Data Health Score Calculation (0 to 100)
|
|
83
|
+
total_cells = max(1, num_rows * num_cols)
|
|
84
|
+
missing_ratio = total_missing_cells / total_cells
|
|
85
|
+
|
|
86
|
+
score = 100.0
|
|
87
|
+
score -= (missing_ratio * 100.0 * 0.45) # Up to 45 pts penalty for missing data
|
|
88
|
+
score -= min(25.0, dup_pct * 1.5) # Up to 25 pts penalty for duplicates
|
|
89
|
+
score -= min(15.0, (total_outliers / total_cells) * 100.0 * 2.0) # Up to 15 pts for outliers
|
|
90
|
+
score -= min(15.0, header_issues * 2.5) # Up to 15 pts for bad headers
|
|
91
|
+
|
|
92
|
+
final_score = int(max(10, min(100, round(score))))
|
|
93
|
+
|
|
94
|
+
if final_score >= 90:
|
|
95
|
+
grade = "A (Excellent)"
|
|
96
|
+
elif final_score >= 80:
|
|
97
|
+
grade = "B (Good, Minor Issues)"
|
|
98
|
+
elif final_score >= 70:
|
|
99
|
+
grade = "C (Fair, Needs Attention)"
|
|
100
|
+
elif final_score >= 60:
|
|
101
|
+
grade = "D (Poor Quality)"
|
|
102
|
+
else:
|
|
103
|
+
grade = "F (Critical Action Needed)"
|
|
104
|
+
|
|
105
|
+
return {
|
|
106
|
+
"file_name": os.path.basename(file_path),
|
|
107
|
+
"file_path": file_path,
|
|
108
|
+
"file_size_kb": file_size_kb,
|
|
109
|
+
"rows": num_rows,
|
|
110
|
+
"cols": num_cols,
|
|
111
|
+
"duplicates": dup_count,
|
|
112
|
+
"dup_pct": dup_pct,
|
|
113
|
+
"total_missing": total_missing_cells,
|
|
114
|
+
"total_outliers": total_outliers,
|
|
115
|
+
"header_issues": header_issues,
|
|
116
|
+
"health_score": final_score,
|
|
117
|
+
"grade": grade,
|
|
118
|
+
"columns": col_profiles
|
|
119
|
+
}
|
csvguard/reporter.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
# Ensure UTF-8 output encoding across Windows terminals
|
|
5
|
+
if sys.stdout.encoding != 'utf-8':
|
|
6
|
+
try:
|
|
7
|
+
sys.stdout.reconfigure(encoding='utf-8', errors='replace')
|
|
8
|
+
sys.stderr.reconfigure(encoding='utf-8', errors='replace')
|
|
9
|
+
except Exception:
|
|
10
|
+
pass
|
|
11
|
+
|
|
12
|
+
from rich.console import Console
|
|
13
|
+
from rich.table import Table
|
|
14
|
+
from rich.panel import Panel
|
|
15
|
+
from rich import box
|
|
16
|
+
|
|
17
|
+
console = Console(force_terminal=True, legacy_windows=False)
|
|
18
|
+
|
|
19
|
+
def render_terminal_audit(profile):
|
|
20
|
+
score = profile["health_score"]
|
|
21
|
+
grade = profile["grade"]
|
|
22
|
+
|
|
23
|
+
if score >= 85:
|
|
24
|
+
score_badge = f"[bold white on green] {score}/100 [/bold white on green] {grade}"
|
|
25
|
+
elif score >= 70:
|
|
26
|
+
score_badge = f"[bold black on yellow] {score}/100 [/bold black on yellow] {grade}"
|
|
27
|
+
else:
|
|
28
|
+
score_badge = f"[bold white on red] {score}/100 [/bold white on red] {grade}"
|
|
29
|
+
|
|
30
|
+
# Top Banner
|
|
31
|
+
console.print("\n")
|
|
32
|
+
console.print(Panel(
|
|
33
|
+
f"[bold white][SHIELD] csvguard Audit:[/bold white] [cyan]{profile['file_name']}[/cyan] ({profile['file_size_kb']} KB)\n"
|
|
34
|
+
f"Data Health Score: {score_badge}",
|
|
35
|
+
box=box.ROUNDED,
|
|
36
|
+
border_style="blue"
|
|
37
|
+
))
|
|
38
|
+
|
|
39
|
+
# Summary Metrics Table
|
|
40
|
+
summary_table = Table(box=box.SIMPLE_HEAD, show_header=True, header_style="bold cyan")
|
|
41
|
+
summary_table.add_column("Rows", justify="center")
|
|
42
|
+
summary_table.add_column("Columns", justify="center")
|
|
43
|
+
summary_table.add_column("Duplicate Rows", justify="center")
|
|
44
|
+
summary_table.add_column("Missing Cells", justify="center")
|
|
45
|
+
summary_table.add_column("Outliers (IQR)", justify="center")
|
|
46
|
+
summary_table.add_column("Header Issues", justify="center")
|
|
47
|
+
|
|
48
|
+
dup_str = f"[red]{profile['duplicates']}[/red]" if profile['duplicates'] > 0 else "[green]0[/green]"
|
|
49
|
+
miss_str = f"[red]{profile['total_missing']}[/red]" if profile['total_missing'] > 0 else "[green]0[/green]"
|
|
50
|
+
out_str = f"[yellow]{profile['total_outliers']}[/yellow]" if profile['total_outliers'] > 0 else "[green]0[/green]"
|
|
51
|
+
head_str = f"[yellow]{profile['header_issues']}[/yellow]" if profile['header_issues'] > 0 else "[green]0[/green]"
|
|
52
|
+
|
|
53
|
+
summary_table.add_row(
|
|
54
|
+
str(profile['rows']),
|
|
55
|
+
str(profile['cols']),
|
|
56
|
+
dup_str,
|
|
57
|
+
miss_str,
|
|
58
|
+
out_str,
|
|
59
|
+
head_str
|
|
60
|
+
)
|
|
61
|
+
console.print(summary_table)
|
|
62
|
+
|
|
63
|
+
# Detailed Column Breakdown Table
|
|
64
|
+
col_table = Table(
|
|
65
|
+
title="Column-by-Column Deep Health Inspection",
|
|
66
|
+
box=box.ROUNDED,
|
|
67
|
+
border_style="dim",
|
|
68
|
+
header_style="bold blue"
|
|
69
|
+
)
|
|
70
|
+
col_table.add_column("#", justify="right", style="dim")
|
|
71
|
+
col_table.add_column("Column Name", justify="left", style="bold")
|
|
72
|
+
col_table.add_column("Inferred Type", justify="center")
|
|
73
|
+
col_table.add_column("Missing (%)", justify="right")
|
|
74
|
+
col_table.add_column("Uniques", justify="right")
|
|
75
|
+
col_table.add_column("Outliers", justify="right")
|
|
76
|
+
col_table.add_column("Health Status", justify="center")
|
|
77
|
+
|
|
78
|
+
for idx, c in enumerate(profile["columns"], start=1):
|
|
79
|
+
if c["missing_pct"] == 0:
|
|
80
|
+
m_text = "[green]0% (0)[/green]"
|
|
81
|
+
elif c["missing_pct"] < 20:
|
|
82
|
+
m_text = f"[yellow]{c['missing_pct']}% ({c['missing_count']})[/yellow]"
|
|
83
|
+
else:
|
|
84
|
+
m_text = f"[bold red]{c['missing_pct']}% ({c['missing_count']})[/bold red]"
|
|
85
|
+
|
|
86
|
+
o_text = f"[yellow]{c['outlier_count']}[/yellow]" if c['outlier_count'] > 0 else "[green]0[/green]"
|
|
87
|
+
|
|
88
|
+
if c["status"] == "Healthy":
|
|
89
|
+
status_text = "[green][OK] Healthy[/green]"
|
|
90
|
+
elif c["status"] == "Warning":
|
|
91
|
+
status_text = "[yellow][WARN] Warning[/yellow]"
|
|
92
|
+
else:
|
|
93
|
+
status_text = "[bold red][CRIT] Critical[/bold red]"
|
|
94
|
+
|
|
95
|
+
col_table.add_row(
|
|
96
|
+
str(idx),
|
|
97
|
+
c["name"],
|
|
98
|
+
c["type"],
|
|
99
|
+
m_text,
|
|
100
|
+
str(c["unique_count"]),
|
|
101
|
+
o_text,
|
|
102
|
+
status_text
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
console.print(col_table)
|
|
106
|
+
|
|
107
|
+
if score < 90:
|
|
108
|
+
console.print("\n[bold yellow]Recommended Action:[/bold yellow] Run [bold cyan]csvguard clean <filename> --auto[/bold cyan] to automatically fix duplicates, sanitize headers, and impute missing values!\n")
|
|
109
|
+
else:
|
|
110
|
+
console.print("\n[bold green]Great news![/bold green] Your dataset meets high quality standards.\n")
|
|
111
|
+
|
|
112
|
+
def export_markdown_report(profile, output_file="csvguard_report.md"):
|
|
113
|
+
md = f"""# csvguard Data Quality Report
|
|
114
|
+
|
|
115
|
+
- **File Analyzed:** `{profile['file_name']}`
|
|
116
|
+
- **File Size:** {profile['file_size_kb']} KB
|
|
117
|
+
- **Health Score:** **{profile['health_score']}/100** ({profile['grade']})
|
|
118
|
+
- **Total Records:** {profile['rows']} rows, {profile['cols']} columns
|
|
119
|
+
|
|
120
|
+
---
|
|
121
|
+
|
|
122
|
+
## Summary Metrics
|
|
123
|
+
|
|
124
|
+
| Metric | Measured Value |
|
|
125
|
+
|:---|:---:|
|
|
126
|
+
| **Total Rows** | {profile['rows']} |
|
|
127
|
+
| **Total Columns** | {profile['cols']} |
|
|
128
|
+
| **Duplicate Rows** | {profile['duplicates']} ({profile['dup_pct']}%) |
|
|
129
|
+
| **Total Missing Cells** | {profile['total_missing']} |
|
|
130
|
+
| **Statistical Outliers (IQR)** | {profile['total_outliers']} |
|
|
131
|
+
| **Header Formatting Issues** | {profile['header_issues']} |
|
|
132
|
+
|
|
133
|
+
---
|
|
134
|
+
|
|
135
|
+
## Column Health Matrix
|
|
136
|
+
|
|
137
|
+
| Column Name | Inferred Type | Missing Count | Missing (%) | Unique Values | Outliers | Status |
|
|
138
|
+
|:---|:---|:---:|:---:|:---:|:---:|:---:|
|
|
139
|
+
"""
|
|
140
|
+
for c in profile["columns"]:
|
|
141
|
+
md += f"| `{c['name']}` | {c['type']} | {c['missing_count']} | {c['missing_pct']}% | {c['unique_count']} | {c['outlier_count']} | {c['status']} |\n"
|
|
142
|
+
|
|
143
|
+
md += "\n---\n*Generated autonomously by [csvguard](https://github.com/your-username/csvguard).*"
|
|
144
|
+
|
|
145
|
+
with open(output_file, "w", encoding="utf-8") as f:
|
|
146
|
+
f.write(md)
|
|
147
|
+
return output_file
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: csvguard
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: The Fast, Terminal-First Data Quality & Cleaning CLI
|
|
5
|
+
Home-page: https://github.com/your-username/csvguard
|
|
6
|
+
Author: Sumit
|
|
7
|
+
Author-email: Sumit <sumit.developer@example.com>
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Requires-Python: >=3.8
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
Requires-Dist: pandas>=1.3.0
|
|
15
|
+
Requires-Dist: numpy>=1.20.0
|
|
16
|
+
Requires-Dist: rich>=12.0.0
|
|
17
|
+
Dynamic: author
|
|
18
|
+
Dynamic: home-page
|
|
19
|
+
Dynamic: requires-python
|
|
20
|
+
|
|
21
|
+
# ๐ก๏ธ csvguard
|
|
22
|
+
|
|
23
|
+
> **The Fast, Terminal-First Data Quality Profiler & Cleaning CLI for Python.**
|
|
24
|
+
|
|
25
|
+
[](https://pypi.org/)
|
|
26
|
+
[](https://www.python.org/)
|
|
27
|
+
[](https://opensource.org/licenses/MIT)
|
|
28
|
+
[](https://github.com/psf/black)
|
|
29
|
+
|
|
30
|
+
Stop writing 40 lines of boilerplate Pandas code just to inspect missing values, bad headers, duplicate rows, and statistical outliers in messy CSV files.
|
|
31
|
+
|
|
32
|
+
`csvguard` gives you a **CIBIL-style Data Health Score (0-100)**, an interactive terminal dashboard, and one-command automated data cleaning.
|
|
33
|
+
|
|
34
|
+
---
|
|
35
|
+
|
|
36
|
+
## โจ Features
|
|
37
|
+
|
|
38
|
+
- ๐ฉบ **Instant Health Score (0-100):** Comprehensive weighted evaluation of missing data ratios, duplicate rows, IQR outliers, and header hygiene.
|
|
39
|
+
- ๐จ **Rich Terminal Dashboard:** Color-coded tables, status indicators, and progress spinners directly inside your shell.
|
|
40
|
+
- ๐งน **Automated Autonomous Cleaning:** Drop duplicates, sanitize column headers to `snake_case`, and impute missing numerical/categorical values with a single command.
|
|
41
|
+
- ๐ **Statistical Outlier Detection:** Identifies extreme values using Tukey's Interquartile Range (IQR) fences.
|
|
42
|
+
- ๐ **Markdown & CI/CD Export:** Generate markdown audit reports suitable for GitHub PRs and automated data validation pipelines.
|
|
43
|
+
- โก **Dual Interface:** Use as a standalone Command-Line Tool (`csvguard`) or as a Python library (`import csvguard`).
|
|
44
|
+
|
|
45
|
+
---
|
|
46
|
+
|
|
47
|
+
## ๐ Installation
|
|
48
|
+
|
|
49
|
+
### Via PyPI (Recommended):
|
|
50
|
+
```bash
|
|
51
|
+
pip install csvguard
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
### From Source (Local Development):
|
|
55
|
+
```bash
|
|
56
|
+
git clone https://github.com/your-username/csvguard.git
|
|
57
|
+
cd csvguard
|
|
58
|
+
pip install -e .
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
---
|
|
62
|
+
|
|
63
|
+
## ๐ป CLI Usage
|
|
64
|
+
|
|
65
|
+
### 1. Audit a CSV File (Health Checkup):
|
|
66
|
+
```bash
|
|
67
|
+
csvguard audit data.csv
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### 2. Auto-Clean and Sanitize Data:
|
|
71
|
+
```bash
|
|
72
|
+
csvguard clean messy.csv --auto -o cleaned.csv
|
|
73
|
+
```
|
|
74
|
+
This automatically:
|
|
75
|
+
- Sanitizes headers (` Annual Income ` โ `annual_income`)
|
|
76
|
+
- Removes exact duplicate records
|
|
77
|
+
- Imputes missing numerical values with column medians
|
|
78
|
+
- Fills missing text fields with `'Unknown'`
|
|
79
|
+
|
|
80
|
+
### 3. Generate a Markdown Documentation Report:
|
|
81
|
+
```bash
|
|
82
|
+
csvguard report data.csv -o DATA_QUALITY_REPORT.md
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
---
|
|
86
|
+
|
|
87
|
+
## ๐ Python Library Usage
|
|
88
|
+
|
|
89
|
+
You can also import `csvguard` directly in your machine learning scripts or Jupyter Notebooks:
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
import csvguard as cg
|
|
93
|
+
|
|
94
|
+
# 1. Audit dataset
|
|
95
|
+
profile = cg.audit("samples/messy_sample.csv")
|
|
96
|
+
print(f"Health Score: {profile['health_score']}/100 ({profile['grade']})")
|
|
97
|
+
print(f"Duplicates: {profile['duplicates']}")
|
|
98
|
+
|
|
99
|
+
# 2. Clean dataset programmatically
|
|
100
|
+
res = cg.clean("samples/messy_sample.csv", output_path="clean.csv", impute_numeric="median")
|
|
101
|
+
print(f"Cleaned dataset saved to: {res['output_path']}")
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
---
|
|
105
|
+
|
|
106
|
+
## ๐ฆ How to Publish to PyPI (For Maintainers)
|
|
107
|
+
|
|
108
|
+
1. Build the distribution package:
|
|
109
|
+
```bash
|
|
110
|
+
python -m pip install --upgrade build twine
|
|
111
|
+
python -m build
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
2. Upload to PyPI:
|
|
115
|
+
```bash
|
|
116
|
+
python -m twine upload dist/*
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
---
|
|
120
|
+
|
|
121
|
+
## ๐ License
|
|
122
|
+
Distributed under the [MIT License](LICENSE).
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
csvguard/__init__.py,sha256=qucuUOWUVN9ASrDTKt6BAl183jOX1azJFMsIfwKYYxc,247
|
|
2
|
+
csvguard/cleaner.py,sha256=-UONqM2VvfhLCyZHFu-gZOZlI9mmB9Ejsh4soQJnF2g,3350
|
|
3
|
+
csvguard/cli.py,sha256=7yofHbsCXoCez_CuE0FBPOg8Zhp_dJVcLkd4aQedsII,4116
|
|
4
|
+
csvguard/profiler.py,sha256=3Vtz27Urv33u61Znt9dnutS4GNogQ1J9plorO9KtT7g,4200
|
|
5
|
+
csvguard/reporter.py,sha256=mqbRkD_vddoD0a3x-K9NLboUMayxXIbNNoUX7DETYIQ,5696
|
|
6
|
+
csvguard-0.1.0.dist-info/METADATA,sha256=V1KqkRplqjSTRUW8z8JVJC1vA35OUdsKVKx9W3bmjWA,3964
|
|
7
|
+
csvguard-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
8
|
+
csvguard-0.1.0.dist-info/entry_points.txt,sha256=GYDRDnlY7XDIie39t0wxPyIlnAWxPX-F2Jrt65_OJ5s,47
|
|
9
|
+
csvguard-0.1.0.dist-info/top_level.txt,sha256=MJ8hyTMoKNsjuCdEd8YU2jOvKSi11D082DUOUW9qlFo,9
|
|
10
|
+
csvguard-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
csvguard
|