datadoc-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datadoc/__init__.py +0 -0
- datadoc/cli/__init__.py +0 -0
- datadoc/cli/app.py +539 -0
- datadoc/core/__init__.py +0 -0
- datadoc/core/engine.py +142 -0
- datadoc/plugins/__init__.py +0 -0
- datadoc/plugins/base.py +70 -0
- datadoc/plugins/datetime_feat.py +102 -0
- datadoc/plugins/encoders.py +73 -0
- datadoc/plugins/missing_values.py +74 -0
- datadoc/plugins/outliers.py +93 -0
- datadoc/plugins/scaling.py +103 -0
- datadoc_cli-0.1.0.dist-info/METADATA +254 -0
- datadoc_cli-0.1.0.dist-info/RECORD +17 -0
- datadoc_cli-0.1.0.dist-info/WHEEL +4 -0
- datadoc_cli-0.1.0.dist-info/entry_points.txt +2 -0
- datadoc_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
datadoc/__init__.py
ADDED
|
File without changes
|
datadoc/cli/__init__.py
ADDED
|
File without changes
|
datadoc/cli/app.py
ADDED
|
@@ -0,0 +1,539 @@
|
|
|
1
|
+
import typer
|
|
2
|
+
from rich.console import Console
|
|
3
|
+
from rich.table import Table
|
|
4
|
+
from rich.panel import Panel
|
|
5
|
+
from rich.text import Text
|
|
6
|
+
from rich import box
|
|
7
|
+
import os
|
|
8
|
+
import time
|
|
9
|
+
|
|
10
|
+
from datadoc.core.engine import DATADOC
|
|
11
|
+
|
|
12
|
+
app = typer.Typer(
|
|
13
|
+
help="DATADOC: The Open Source Operating System for Dataset Engineering.",
|
|
14
|
+
add_completion=False,
|
|
15
|
+
no_args_is_help=True,
|
|
16
|
+
)
|
|
17
|
+
console = Console()
|
|
18
|
+
|
|
19
|
+
VERSION = "0.1.0"
|
|
20
|
+
|
|
21
|
+
BANNER = r"""
|
|
22
|
+
____ _ _____ _ ____ ___ ____
|
|
23
|
+
| _ \ / \|_ _|/ \ | _ \ / _ \ / ___|
|
|
24
|
+
| | | |/ _ \ | | / _ \ | | | | | | | |
|
|
25
|
+
| |_| / ___ \| |/ ___ \| |_| | |_| | |___
|
|
26
|
+
|____/_/ \_\_/_/ \_\____/ \___/ \____|
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def print_banner():
|
|
31
|
+
banner_text = Text(BANNER, style="bold cyan")
|
|
32
|
+
panel = Panel(
|
|
33
|
+
banner_text,
|
|
34
|
+
subtitle=f"[dim]v{VERSION} - The Open Source OS for Dataset Engineering[/dim]",
|
|
35
|
+
border_style="bright_blue",
|
|
36
|
+
padding=(0, 2),
|
|
37
|
+
)
|
|
38
|
+
console.print(panel)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def print_step(icon: str, message: str, style: str = "bold white"):
|
|
42
|
+
console.print(f" {icon} [{style}]{message}[/]")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def load_dataset(file_path: str) -> DATADOC:
|
|
46
|
+
if not os.path.exists(file_path):
|
|
47
|
+
console.print(f"\n [bold red][X] File not found:[/] {file_path}")
|
|
48
|
+
raise typer.Exit(code=1)
|
|
49
|
+
|
|
50
|
+
print_step("[>>]", f"Loading [cyan]{file_path}[/cyan]...")
|
|
51
|
+
try:
|
|
52
|
+
doc = DATADOC(file_path)
|
|
53
|
+
except Exception as e:
|
|
54
|
+
console.print(f"\n [bold red][X] Failed to read dataset:[/] {e}")
|
|
55
|
+
raise typer.Exit(code=1)
|
|
56
|
+
|
|
57
|
+
rows, cols = doc.df.height, doc.df.width
|
|
58
|
+
print_step("[OK]", f"Loaded [green]{rows:,}[/green] rows x [green]{cols}[/green] columns", "bold green")
|
|
59
|
+
return doc
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# ──────────────────────────────────────────────────────────────
|
|
63
|
+
# COMMAND: analyze
|
|
64
|
+
# ──────────────────────────────────────────────────────────────
|
|
65
|
+
@app.command()
|
|
66
|
+
def analyze(file_path: str):
|
|
67
|
+
"""
|
|
68
|
+
Scans the dataset and returns a rich health report.
|
|
69
|
+
"""
|
|
70
|
+
print_banner()
|
|
71
|
+
doc = load_dataset(file_path)
|
|
72
|
+
|
|
73
|
+
with console.status("[bold cyan]Running health analysis...", spinner="dots"):
|
|
74
|
+
report = doc.analyze()
|
|
75
|
+
print_step("[OK]", "Analysis complete.", "bold green")
|
|
76
|
+
|
|
77
|
+
console.print()
|
|
78
|
+
table = Table(
|
|
79
|
+
title="[bold]Health Report[/bold]",
|
|
80
|
+
box=box.ROUNDED,
|
|
81
|
+
title_style="bold cyan",
|
|
82
|
+
header_style="bold bright_white",
|
|
83
|
+
border_style="bright_blue",
|
|
84
|
+
show_lines=True,
|
|
85
|
+
)
|
|
86
|
+
table.add_column("Metric", justify="left", style="white", min_width=25)
|
|
87
|
+
table.add_column("Value", justify="center", style="bold", min_width=10)
|
|
88
|
+
table.add_column("Status", justify="center", min_width=15)
|
|
89
|
+
|
|
90
|
+
table.add_row("Rows", f"{report['rows']:,}", "[green]--[/green]")
|
|
91
|
+
table.add_row("Columns", str(report["cols"]), "[green]--[/green]")
|
|
92
|
+
|
|
93
|
+
for p_name, p_stats in report["plugins"].items():
|
|
94
|
+
if p_name == "MissingValuePlugin":
|
|
95
|
+
total = p_stats["total_missing"]
|
|
96
|
+
status = "[green][OK] Clean[/green]" if total == 0 else f"[red][!!] {total} found[/red]"
|
|
97
|
+
table.add_row("Missing Values", str(total), status)
|
|
98
|
+
elif p_name == "OutlierPlugin":
|
|
99
|
+
count = len(p_stats.get("outlier_columns", []))
|
|
100
|
+
status = "[green][OK] Clean[/green]" if count == 0 else f"[yellow][!!] {count} cols[/yellow]"
|
|
101
|
+
table.add_row("Outlier Columns", str(count), status)
|
|
102
|
+
elif p_name == "DatetimePlugin":
|
|
103
|
+
count = len(p_stats.get("datetime_columns", []))
|
|
104
|
+
if count > 0:
|
|
105
|
+
cols_list = ", ".join(p_stats["datetime_columns"])
|
|
106
|
+
table.add_row("Datetime Columns", str(count), f"[cyan]{cols_list}[/cyan]")
|
|
107
|
+
else:
|
|
108
|
+
table.add_row("Datetime Columns", "0", "[dim]None[/dim]")
|
|
109
|
+
elif p_name == "CategoricalEncoderPlugin":
|
|
110
|
+
count = len(p_stats.get("categorical_columns", []))
|
|
111
|
+
if count > 0:
|
|
112
|
+
cols_list = ", ".join(p_stats["categorical_columns"])
|
|
113
|
+
table.add_row("Categorical Columns", str(count), f"[cyan]{cols_list}[/cyan]")
|
|
114
|
+
else:
|
|
115
|
+
table.add_row("Categorical Columns", "0", "[dim]None[/dim]")
|
|
116
|
+
elif p_name == "ScalingPlugin":
|
|
117
|
+
has_issue = p_stats.get("has_scale_issues", False)
|
|
118
|
+
ratio = p_stats.get("scale_ratio", 0)
|
|
119
|
+
if has_issue:
|
|
120
|
+
table.add_row("Scale Mismatch", f"{ratio}x", f"[yellow][!!] Needs scaling[/yellow]")
|
|
121
|
+
else:
|
|
122
|
+
table.add_row("Scale Mismatch", "--", "[green][OK] Balanced[/green]")
|
|
123
|
+
|
|
124
|
+
console.print(table)
|
|
125
|
+
console.print()
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
# ──────────────────────────────────────────────────────────────
|
|
129
|
+
# COMMAND: recommend
|
|
130
|
+
# ──────────────────────────────────────────────────────────────
|
|
131
|
+
@app.command()
|
|
132
|
+
def recommend(file_path: str):
|
|
133
|
+
"""
|
|
134
|
+
Outputs a list of suggested engineering steps without applying them.
|
|
135
|
+
"""
|
|
136
|
+
print_banner()
|
|
137
|
+
doc = load_dataset(file_path)
|
|
138
|
+
|
|
139
|
+
with console.status("[bold cyan]Generating recommendations...", spinner="dots"):
|
|
140
|
+
recommendations = doc.recommend()
|
|
141
|
+
print_step("[OK]", "Recommendations ready.", "bold green")
|
|
142
|
+
|
|
143
|
+
console.print()
|
|
144
|
+
if not recommendations:
|
|
145
|
+
console.print(Panel(
|
|
146
|
+
"[bold green][OK] Dataset looks perfectly healthy! No recommendations.[/bold green]",
|
|
147
|
+
border_style="green",
|
|
148
|
+
))
|
|
149
|
+
return
|
|
150
|
+
|
|
151
|
+
rec_table = Table(
|
|
152
|
+
title="[bold]Recommended Actions[/bold]",
|
|
153
|
+
box=box.ROUNDED,
|
|
154
|
+
title_style="bold yellow",
|
|
155
|
+
header_style="bold bright_white",
|
|
156
|
+
border_style="yellow",
|
|
157
|
+
show_lines=True,
|
|
158
|
+
)
|
|
159
|
+
rec_table.add_column("#", justify="center", style="bold yellow", width=4)
|
|
160
|
+
rec_table.add_column("Recommendation", style="white")
|
|
161
|
+
|
|
162
|
+
for i, rec in enumerate(recommendations, 1):
|
|
163
|
+
rec_table.add_row(str(i), rec)
|
|
164
|
+
|
|
165
|
+
console.print(rec_table)
|
|
166
|
+
console.print()
|
|
167
|
+
console.print(" [dim]Run [bold cyan]datadoc engineer <file>[/bold cyan] to apply these automatically.[/dim]\n")
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
# ──────────────────────────────────────────────────────────────
|
|
171
|
+
# COMMAND: engineer
|
|
172
|
+
# ──────────────────────────────────────────────────────────────
|
|
173
|
+
@app.command()
|
|
174
|
+
def engineer(file_path: str):
|
|
175
|
+
"""
|
|
176
|
+
Automatically applies best-practice pipelines.
|
|
177
|
+
"""
|
|
178
|
+
print_banner()
|
|
179
|
+
doc = load_dataset(file_path)
|
|
180
|
+
|
|
181
|
+
console.print()
|
|
182
|
+
with console.status("[bold cyan]Running Rule Engine...", spinner="dots") as status:
|
|
183
|
+
def on_progress(plugin_name, p_status, details):
|
|
184
|
+
if p_status == "running":
|
|
185
|
+
status.update(f"[bold cyan]Running [yellow]{plugin_name}[/yellow]...[/bold cyan]")
|
|
186
|
+
time.sleep(0.5) # Add a small delay to make the UI update visible to the user
|
|
187
|
+
elif p_status == "applied":
|
|
188
|
+
print_step("[>>]", f"Applied [bold cyan]{plugin_name}[/bold cyan]", "bold white")
|
|
189
|
+
for detail in details:
|
|
190
|
+
console.print(f" [dim]-> {detail}[/dim]")
|
|
191
|
+
elif p_status == "skipped":
|
|
192
|
+
print_step("[--]", f"Skipped [dim]{plugin_name}[/dim] (not needed)", "dim white")
|
|
193
|
+
|
|
194
|
+
clean_df = doc.engineer(progress_callback=on_progress)
|
|
195
|
+
|
|
196
|
+
output_path = f"clean_{os.path.basename(file_path)}"
|
|
197
|
+
clean_df.write_csv(output_path)
|
|
198
|
+
|
|
199
|
+
console.print()
|
|
200
|
+
console.print(Panel(
|
|
201
|
+
f"[bold green][OK] Success![/bold green]\n\n"
|
|
202
|
+
f" Input: [cyan]{file_path}[/cyan] ({doc.df.height} rows x {doc.df.width} cols)\n"
|
|
203
|
+
f" Output: [cyan]{output_path}[/cyan] ({clean_df.height} rows x {clean_df.width} cols)\n\n"
|
|
204
|
+
f" Applied: {', '.join(doc._applied_plugins) or 'None'}\n"
|
|
205
|
+
f" Skipped: {', '.join(doc._skipped_plugins) or 'None'}",
|
|
206
|
+
title="[bold]Engineering Complete[/bold]",
|
|
207
|
+
border_style="green",
|
|
208
|
+
))
|
|
209
|
+
console.print()
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
# ──────────────────────────────────────────────────────────────
|
|
213
|
+
# COMMAND: compare
|
|
214
|
+
# ──────────────────────────────────────────────────────────────
|
|
215
|
+
@app.command()
|
|
216
|
+
def compare(file_path: str):
|
|
217
|
+
"""
|
|
218
|
+
Shows a diff-like comparison between the raw and engineered dataset.
|
|
219
|
+
"""
|
|
220
|
+
print_banner()
|
|
221
|
+
doc = load_dataset(file_path)
|
|
222
|
+
|
|
223
|
+
with console.status("[bold cyan]Engineering dataset for comparison...", spinner="dots"):
|
|
224
|
+
clean_df = doc.engineer()
|
|
225
|
+
diff = doc.compare(clean_df)
|
|
226
|
+
|
|
227
|
+
console.print()
|
|
228
|
+
table = Table(
|
|
229
|
+
title="[bold]Before vs After Comparison[/bold]",
|
|
230
|
+
box=box.ROUNDED,
|
|
231
|
+
title_style="bold magenta",
|
|
232
|
+
header_style="bold bright_white",
|
|
233
|
+
border_style="magenta",
|
|
234
|
+
show_lines=True,
|
|
235
|
+
)
|
|
236
|
+
table.add_column("Metric", style="white", min_width=20)
|
|
237
|
+
table.add_column("Before", justify="center", style="red", min_width=15)
|
|
238
|
+
table.add_column("After", justify="center", style="green", min_width=15)
|
|
239
|
+
|
|
240
|
+
orig_r, orig_c = diff["original_shape"]
|
|
241
|
+
clean_r, clean_c = diff["clean_shape"]
|
|
242
|
+
table.add_row("Rows", str(orig_r), str(clean_r))
|
|
243
|
+
table.add_row("Columns", str(orig_c), str(clean_c))
|
|
244
|
+
table.add_row("Missing Values", str(diff["original_missing"]), str(diff["clean_missing"]))
|
|
245
|
+
table.add_row(
|
|
246
|
+
"Columns Added",
|
|
247
|
+
"--",
|
|
248
|
+
f"+{diff['cols_added']}" if diff["cols_added"] > 0 else "0"
|
|
249
|
+
)
|
|
250
|
+
|
|
251
|
+
# Data type breakdown
|
|
252
|
+
for dtype, count in diff["original_dtypes"].items():
|
|
253
|
+
dtype_str = str(dtype)
|
|
254
|
+
clean_count = diff["clean_dtypes"].get(dtype, 0)
|
|
255
|
+
table.add_row(f"dtype: {dtype_str}", str(count), str(clean_count))
|
|
256
|
+
|
|
257
|
+
# Check for new dtypes in clean that weren't in original
|
|
258
|
+
for dtype, count in diff["clean_dtypes"].items():
|
|
259
|
+
if dtype not in diff["original_dtypes"]:
|
|
260
|
+
table.add_row(f"dtype: {str(dtype)}", "0", str(count))
|
|
261
|
+
|
|
262
|
+
console.print(table)
|
|
263
|
+
|
|
264
|
+
# Show applied/skipped plugins
|
|
265
|
+
console.print()
|
|
266
|
+
if hasattr(doc, '_applied_plugins'):
|
|
267
|
+
print_step("[OK]", f"Applied: {', '.join(doc._applied_plugins)}", "bold green")
|
|
268
|
+
print_step("[--]", f"Skipped: {', '.join(doc._skipped_plugins)}", "dim")
|
|
269
|
+
console.print()
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
# ──────────────────────────────────────────────────────────────
|
|
273
|
+
# COMMAND: visualize
|
|
274
|
+
# ──────────────────────────────────────────────────────────────
|
|
275
|
+
@app.command()
|
|
276
|
+
def visualize(file_path: str):
|
|
277
|
+
"""
|
|
278
|
+
Generates a massive, interactive terminal dashboard comparing the before and after states.
|
|
279
|
+
"""
|
|
280
|
+
import plotext as plt
|
|
281
|
+
import numpy as np
|
|
282
|
+
import sys
|
|
283
|
+
import polars as pl
|
|
284
|
+
|
|
285
|
+
if sys.platform == "win32":
|
|
286
|
+
sys.stdout.reconfigure(encoding="utf-8")
|
|
287
|
+
|
|
288
|
+
print_banner()
|
|
289
|
+
doc = load_dataset(file_path)
|
|
290
|
+
|
|
291
|
+
with console.status("[bold cyan]Generating terminal dashboard...", spinner="dots"):
|
|
292
|
+
clean_df = doc.engineer()
|
|
293
|
+
|
|
294
|
+
orig_missing = {c: doc.df[c].null_count() for c in doc.df.columns}
|
|
295
|
+
clean_missing = {c: clean_df[c].null_count() for c in clean_df.columns}
|
|
296
|
+
cols_with_missing = [c for c, count in orig_missing.items() if count > 0]
|
|
297
|
+
|
|
298
|
+
# Missing values bar chart
|
|
299
|
+
if cols_with_missing:
|
|
300
|
+
plt.clf()
|
|
301
|
+
plt.theme("dark")
|
|
302
|
+
orig_vals = [orig_missing[c] for c in cols_with_missing]
|
|
303
|
+
clean_vals = [clean_missing.get(c, 0) for c in cols_with_missing]
|
|
304
|
+
|
|
305
|
+
plt.multiple_bar(cols_with_missing, [orig_vals, clean_vals], labels=["Before", "After"])
|
|
306
|
+
plt.title("Missing Values Resolution")
|
|
307
|
+
plt.plotsize(100, 20)
|
|
308
|
+
missing_plot = plt.build()
|
|
309
|
+
else:
|
|
310
|
+
missing_plot = " [dim]No missing values found in the original dataset.[/dim]"
|
|
311
|
+
|
|
312
|
+
# Numerical distributions
|
|
313
|
+
num_cols = [c for c in doc.df.columns if doc.df[c].dtype in pl.NUMERIC_DTYPES]
|
|
314
|
+
dist_plots = []
|
|
315
|
+
|
|
316
|
+
for col in num_cols:
|
|
317
|
+
if col not in clean_df.columns:
|
|
318
|
+
continue
|
|
319
|
+
|
|
320
|
+
orig_data = doc.df[col].drop_nulls()
|
|
321
|
+
clean_data = clean_df[col].drop_nulls()
|
|
322
|
+
|
|
323
|
+
if len(orig_data) == 0 or len(clean_data) == 0:
|
|
324
|
+
continue
|
|
325
|
+
|
|
326
|
+
plt.clf()
|
|
327
|
+
plt.theme("dark")
|
|
328
|
+
|
|
329
|
+
plt.hist(orig_data.to_list(), bins=20, label="Before", color="red")
|
|
330
|
+
plt.hist(clean_data.to_list(), bins=20, label="After", color="green")
|
|
331
|
+
plt.title(f"Distribution: {col}")
|
|
332
|
+
plt.plotsize(100, 20)
|
|
333
|
+
dist_plots.append(plt.build())
|
|
334
|
+
|
|
335
|
+
# Render directly to terminal
|
|
336
|
+
console.print()
|
|
337
|
+
console.print(Panel(
|
|
338
|
+
Text.from_ansi(missing_plot) if "No missing" not in missing_plot else missing_plot,
|
|
339
|
+
title="[bold yellow]Missing Values Breakdown[/bold yellow]",
|
|
340
|
+
border_style="yellow"
|
|
341
|
+
))
|
|
342
|
+
|
|
343
|
+
for i, d_plot in enumerate(dist_plots):
|
|
344
|
+
console.print(Panel(
|
|
345
|
+
Text.from_ansi(d_plot),
|
|
346
|
+
title=f"[bold cyan]Numerical Distribution: {num_cols[i]}[/bold cyan]",
|
|
347
|
+
border_style="cyan"
|
|
348
|
+
))
|
|
349
|
+
|
|
350
|
+
console.print(Panel(
|
|
351
|
+
f"[bold green]Terminal dashboard generated successfully![/bold green]\n\n"
|
|
352
|
+
f" Applied Plugins: {', '.join(doc._applied_plugins) or 'None'}",
|
|
353
|
+
title="[bold]Summary[/bold]",
|
|
354
|
+
border_style="green",
|
|
355
|
+
))
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
# ──────────────────────────────────────────────────────────────
|
|
359
|
+
# COMMAND: pipeline
|
|
360
|
+
# ──────────────────────────────────────────────────────────────
|
|
361
|
+
@app.command()
|
|
362
|
+
def pipeline(file_path: str):
|
|
363
|
+
"""
|
|
364
|
+
Exports the generated pipeline as a standalone .py script.
|
|
365
|
+
"""
|
|
366
|
+
print_banner()
|
|
367
|
+
doc = load_dataset(file_path)
|
|
368
|
+
|
|
369
|
+
with console.status("[bold cyan]Generating Python pipeline...", spinner="dots"):
|
|
370
|
+
script = doc.pipeline()
|
|
371
|
+
print_step("[OK]", "Pipeline generated.", "bold green")
|
|
372
|
+
|
|
373
|
+
output_path = f"pipeline_{os.path.basename(file_path).split('.')[0]}.py"
|
|
374
|
+
|
|
375
|
+
with open(output_path, "w") as f:
|
|
376
|
+
f.write(script)
|
|
377
|
+
|
|
378
|
+
console.print()
|
|
379
|
+
console.print(Panel(
|
|
380
|
+
f"[bold green][OK] Pipeline generated![/bold green]\n\n"
|
|
381
|
+
f" Saved to: [bold cyan]{output_path}[/bold cyan]\n\n"
|
|
382
|
+
f" [dim]Run it with:[/dim] [bold white]python {output_path}[/bold white]",
|
|
383
|
+
title="[bold]Pipeline Export[/bold]",
|
|
384
|
+
border_style="cyan",
|
|
385
|
+
))
|
|
386
|
+
console.print()
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
# ──────────────────────────────────────────────────────────────
|
|
390
|
+
# COMMAND: report
|
|
391
|
+
# ──────────────────────────────────────────────────────────────
|
|
392
|
+
@app.command()
|
|
393
|
+
def report(file_path: str):
|
|
394
|
+
"""
|
|
395
|
+
Generates a Markdown report summarizing the dataset and recommendations.
|
|
396
|
+
"""
|
|
397
|
+
print_banner()
|
|
398
|
+
doc = load_dataset(file_path)
|
|
399
|
+
|
|
400
|
+
with console.status("[bold cyan]Generating report...", spinner="dots"):
|
|
401
|
+
report_data = doc.analyze()
|
|
402
|
+
recommendations = doc.recommend()
|
|
403
|
+
plugin_info = doc.list_plugins()
|
|
404
|
+
|
|
405
|
+
md_lines = [
|
|
406
|
+
f"# DATADOC Report: {os.path.basename(file_path)}",
|
|
407
|
+
"",
|
|
408
|
+
"---",
|
|
409
|
+
"",
|
|
410
|
+
"## Dataset Overview",
|
|
411
|
+
"",
|
|
412
|
+
f"| Metric | Value |",
|
|
413
|
+
f"|--------|-------|",
|
|
414
|
+
f"| File | `{file_path}` |",
|
|
415
|
+
f"| Rows | {report_data['rows']:,} |",
|
|
416
|
+
f"| Columns | {report_data['cols']} |",
|
|
417
|
+
"",
|
|
418
|
+
]
|
|
419
|
+
|
|
420
|
+
# Plugin analysis details
|
|
421
|
+
md_lines.append("## Health Analysis")
|
|
422
|
+
md_lines.append("")
|
|
423
|
+
for p_name, p_stats in report_data["plugins"].items():
|
|
424
|
+
md_lines.append(f"### {p_name}")
|
|
425
|
+
md_lines.append("")
|
|
426
|
+
for key, val in p_stats.items():
|
|
427
|
+
md_lines.append(f"- **{key}**: `{val}`")
|
|
428
|
+
md_lines.append("")
|
|
429
|
+
|
|
430
|
+
# Recommendations
|
|
431
|
+
md_lines.append("## Recommendations")
|
|
432
|
+
md_lines.append("")
|
|
433
|
+
if recommendations:
|
|
434
|
+
for i, rec in enumerate(recommendations, 1):
|
|
435
|
+
md_lines.append(f"{i}. {rec}")
|
|
436
|
+
else:
|
|
437
|
+
md_lines.append("No issues found. Dataset is healthy!")
|
|
438
|
+
md_lines.append("")
|
|
439
|
+
|
|
440
|
+
# Plugin registry
|
|
441
|
+
md_lines.append("## Plugin Registry")
|
|
442
|
+
md_lines.append("")
|
|
443
|
+
md_lines.append("| Plugin | Version | Priority | Will Trigger |")
|
|
444
|
+
md_lines.append("|--------|---------|----------|-------------|")
|
|
445
|
+
for p in plugin_info:
|
|
446
|
+
trigger = "Yes" if p["will_trigger"] else "No"
|
|
447
|
+
md_lines.append(f"| {p['name']} | {p['version']} | {p['priority']} | {trigger} |")
|
|
448
|
+
md_lines.append("")
|
|
449
|
+
md_lines.append("---")
|
|
450
|
+
md_lines.append(f"*Generated by DATADOC v{VERSION}*")
|
|
451
|
+
md_lines.append("")
|
|
452
|
+
|
|
453
|
+
output_path = f"report_{os.path.basename(file_path).split('.')[0]}.md"
|
|
454
|
+
with open(output_path, "w") as f:
|
|
455
|
+
f.write("\n".join(md_lines))
|
|
456
|
+
|
|
457
|
+
console.print()
|
|
458
|
+
console.print(Panel(
|
|
459
|
+
f"[bold green][OK] Report generated![/bold green]\n\n"
|
|
460
|
+
f" Saved to: [bold cyan]{output_path}[/bold cyan]",
|
|
461
|
+
title="[bold]Report Export[/bold]",
|
|
462
|
+
border_style="cyan",
|
|
463
|
+
))
|
|
464
|
+
console.print()
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
# ──────────────────────────────────────────────────────────────
|
|
468
|
+
# COMMAND: plugin
|
|
469
|
+
# ──────────────────────────────────────────────────────────────
|
|
470
|
+
@app.command(name="plugin")
|
|
471
|
+
def plugin_list():
|
|
472
|
+
"""
|
|
473
|
+
Lists all registered plugins and their status.
|
|
474
|
+
"""
|
|
475
|
+
print_banner()
|
|
476
|
+
|
|
477
|
+
# Create a temporary DATADOC instance with a minimal df just to list plugins
|
|
478
|
+
# We don't need a real file for this
|
|
479
|
+
import io
|
|
480
|
+
dummy_csv = b"a,b\n1,2\n"
|
|
481
|
+
import polars as pl
|
|
482
|
+
dummy_df = pl.read_csv(dummy_csv)
|
|
483
|
+
|
|
484
|
+
from datadoc.plugins.missing_values import MissingValuePlugin
|
|
485
|
+
from datadoc.plugins.outliers import OutlierPlugin
|
|
486
|
+
from datadoc.plugins.datetime_feat import DatetimePlugin
|
|
487
|
+
from datadoc.plugins.encoders import CategoricalEncoderPlugin
|
|
488
|
+
from datadoc.plugins.scaling import ScalingPlugin
|
|
489
|
+
|
|
490
|
+
plugins = sorted([
|
|
491
|
+
MissingValuePlugin(),
|
|
492
|
+
OutlierPlugin(),
|
|
493
|
+
DatetimePlugin(),
|
|
494
|
+
CategoricalEncoderPlugin(),
|
|
495
|
+
ScalingPlugin(),
|
|
496
|
+
], key=lambda p: p.priority)
|
|
497
|
+
|
|
498
|
+
console.print()
|
|
499
|
+
table = Table(
|
|
500
|
+
title="[bold]Registered Plugins[/bold]",
|
|
501
|
+
box=box.ROUNDED,
|
|
502
|
+
title_style="bold cyan",
|
|
503
|
+
header_style="bold bright_white",
|
|
504
|
+
border_style="bright_blue",
|
|
505
|
+
show_lines=True,
|
|
506
|
+
)
|
|
507
|
+
table.add_column("Priority", justify="center", style="yellow", width=10)
|
|
508
|
+
table.add_column("Plugin", style="bold white", min_width=25)
|
|
509
|
+
table.add_column("Version", justify="center", style="cyan", width=10)
|
|
510
|
+
table.add_column("Description", style="dim white")
|
|
511
|
+
|
|
512
|
+
for p in plugins:
|
|
513
|
+
table.add_row(str(p.priority), p.name, p.version, p.description)
|
|
514
|
+
|
|
515
|
+
console.print(table)
|
|
516
|
+
|
|
517
|
+
# Plugin explanations
|
|
518
|
+
console.print()
|
|
519
|
+
console.print(Panel(
|
|
520
|
+
"\n".join([f" [bold cyan]{p.name}[/bold cyan]: {p.explain()}" for p in plugins]),
|
|
521
|
+
title="[bold]Plugin Explanations[/bold]",
|
|
522
|
+
border_style="dim",
|
|
523
|
+
))
|
|
524
|
+
console.print()
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
# ──────────────────────────────────────────────────────────────
|
|
528
|
+
# COMMAND: version
|
|
529
|
+
# ──────────────────────────────────────────────────────────────
|
|
530
|
+
@app.command()
|
|
531
|
+
def version():
|
|
532
|
+
"""
|
|
533
|
+
Displays the current DATADOC version.
|
|
534
|
+
"""
|
|
535
|
+
print_banner()
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
if __name__ == "__main__":
|
|
539
|
+
app()
|
datadoc/core/__init__.py
ADDED
|
File without changes
|
datadoc/core/engine.py
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import polars as pl
|
|
2
|
+
from typing import Dict, Any, List
|
|
3
|
+
from collections import Counter
|
|
4
|
+
from datadoc.plugins.base import BasePlugin
|
|
5
|
+
from datadoc.plugins.missing_values import MissingValuePlugin
|
|
6
|
+
from datadoc.plugins.outliers import OutlierPlugin
|
|
7
|
+
from datadoc.plugins.datetime_feat import DatetimePlugin
|
|
8
|
+
from datadoc.plugins.encoders import CategoricalEncoderPlugin
|
|
9
|
+
from datadoc.plugins.scaling import ScalingPlugin
|
|
10
|
+
|
|
11
|
+
class DATADOC:
|
|
12
|
+
def __init__(self, file_path: str):
|
|
13
|
+
self.file_path = file_path
|
|
14
|
+
self.df = pl.read_csv(file_path, infer_schema_length=10000)
|
|
15
|
+
self._original_df = self.df.clone()
|
|
16
|
+
|
|
17
|
+
# Plugins sorted by priority (lower = runs first)
|
|
18
|
+
self.plugins: List[BasePlugin] = sorted([
|
|
19
|
+
MissingValuePlugin(),
|
|
20
|
+
OutlierPlugin(),
|
|
21
|
+
DatetimePlugin(),
|
|
22
|
+
CategoricalEncoderPlugin(),
|
|
23
|
+
ScalingPlugin(),
|
|
24
|
+
], key=lambda p: p.priority)
|
|
25
|
+
|
|
26
|
+
def analyze(self) -> Dict[str, Any]:
|
|
27
|
+
"""Runs the analyze step across all plugins."""
|
|
28
|
+
dtype_counts = Counter([str(dt) for dt in self.df.dtypes])
|
|
29
|
+
report = {
|
|
30
|
+
"rows": self.df.height,
|
|
31
|
+
"cols": self.df.width,
|
|
32
|
+
"dtypes": dict(dtype_counts),
|
|
33
|
+
"plugins": {}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
for plugin in self.plugins:
|
|
37
|
+
report["plugins"][plugin.name] = plugin.analyze(self.df)
|
|
38
|
+
|
|
39
|
+
return report
|
|
40
|
+
|
|
41
|
+
def recommend(self) -> list[str]:
|
|
42
|
+
"""Runs analysis and aggregates recommendations from all plugins."""
|
|
43
|
+
recommendations = []
|
|
44
|
+
for plugin in self.plugins:
|
|
45
|
+
analysis = plugin.analyze(self.df)
|
|
46
|
+
recs = plugin.recommend(analysis)
|
|
47
|
+
recommendations.extend(recs)
|
|
48
|
+
return recommendations
|
|
49
|
+
|
|
50
|
+
def pipeline(self) -> str:
|
|
51
|
+
"""Generates the full Python script representing the pipeline."""
|
|
52
|
+
lines = [
|
|
53
|
+
"import polars as pl",
|
|
54
|
+
"import numpy as np",
|
|
55
|
+
"",
|
|
56
|
+
"",
|
|
57
|
+
f"def load_and_clean_data(file_path: str = '{self.file_path}') -> pl.DataFrame:",
|
|
58
|
+
" df = pl.read_csv(file_path, infer_schema_length=10000)",
|
|
59
|
+
"",
|
|
60
|
+
]
|
|
61
|
+
|
|
62
|
+
for plugin in self.plugins:
|
|
63
|
+
analysis = plugin.analyze(self.df)
|
|
64
|
+
should_apply = any(bool(v) for k, v in analysis.items() if k.startswith('has_'))
|
|
65
|
+
if should_apply:
|
|
66
|
+
code_snippet = plugin.generate_code(analysis)
|
|
67
|
+
if code_snippet:
|
|
68
|
+
# Indent each line of the code snippet
|
|
69
|
+
for line in code_snippet.split("\n"):
|
|
70
|
+
lines.append(f" {line}")
|
|
71
|
+
lines.append("")
|
|
72
|
+
|
|
73
|
+
lines.append(" return df")
|
|
74
|
+
lines.append("")
|
|
75
|
+
lines.append("")
|
|
76
|
+
lines.append("if __name__ == '__main__':")
|
|
77
|
+
lines.append(" clean_df = load_and_clean_data()")
|
|
78
|
+
lines.append(" print(f'Successfully processed {clean_df.height} rows x {clean_df.width} columns!')")
|
|
79
|
+
lines.append("")
|
|
80
|
+
|
|
81
|
+
return "\n".join(lines)
|
|
82
|
+
|
|
83
|
+
def engineer(self, progress_callback=None) -> pl.DataFrame:
|
|
84
|
+
"""Automatically triggers plugins that are needed."""
|
|
85
|
+
df_transformed = self.df.clone()
|
|
86
|
+
|
|
87
|
+
self._applied_plugins = []
|
|
88
|
+
self._skipped_plugins = []
|
|
89
|
+
|
|
90
|
+
for plugin in self.plugins:
|
|
91
|
+
if progress_callback:
|
|
92
|
+
progress_callback(plugin.name, "running", [])
|
|
93
|
+
|
|
94
|
+
analysis = plugin.analyze(df_transformed)
|
|
95
|
+
should_apply = any(bool(v) for k, v in analysis.items() if k.startswith('has_'))
|
|
96
|
+
if should_apply:
|
|
97
|
+
details = plugin.recommend(analysis)
|
|
98
|
+
df_transformed = plugin.apply(df_transformed)
|
|
99
|
+
self._applied_plugins.append(plugin.name)
|
|
100
|
+
if progress_callback:
|
|
101
|
+
progress_callback(plugin.name, "applied", details)
|
|
102
|
+
else:
|
|
103
|
+
self._skipped_plugins.append(plugin.name)
|
|
104
|
+
if progress_callback:
|
|
105
|
+
progress_callback(plugin.name, "skipped", [])
|
|
106
|
+
|
|
107
|
+
return df_transformed
|
|
108
|
+
|
|
109
|
+
def compare(self, clean_df: pl.DataFrame) -> Dict[str, Any]:
|
|
110
|
+
"""Compare original and engineered dataframes."""
|
|
111
|
+
original = self.df
|
|
112
|
+
|
|
113
|
+
orig_missing = sum(original[c].null_count() for c in original.columns)
|
|
114
|
+
clean_missing = sum(clean_df[c].null_count() for c in clean_df.columns)
|
|
115
|
+
|
|
116
|
+
result = {
|
|
117
|
+
"original_shape": original.shape,
|
|
118
|
+
"clean_shape": clean_df.shape,
|
|
119
|
+
"rows_changed": original.height != clean_df.height,
|
|
120
|
+
"cols_added": clean_df.width - original.width,
|
|
121
|
+
"original_missing": orig_missing,
|
|
122
|
+
"clean_missing": clean_missing,
|
|
123
|
+
"original_dtypes": dict(Counter([str(dt) for dt in original.dtypes])),
|
|
124
|
+
"clean_dtypes": dict(Counter([str(dt) for dt in clean_df.dtypes])),
|
|
125
|
+
}
|
|
126
|
+
return result
|
|
127
|
+
|
|
128
|
+
def list_plugins(self) -> List[Dict[str, Any]]:
|
|
129
|
+
"""Returns info about all registered plugins."""
|
|
130
|
+
info = []
|
|
131
|
+
for plugin in self.plugins:
|
|
132
|
+
analysis = plugin.analyze(self.df)
|
|
133
|
+
has_work = any(bool(v) for k, v in analysis.items() if k.startswith('has_'))
|
|
134
|
+
info.append({
|
|
135
|
+
"name": plugin.name,
|
|
136
|
+
"version": plugin.version,
|
|
137
|
+
"description": plugin.description,
|
|
138
|
+
"priority": plugin.priority,
|
|
139
|
+
"will_trigger": has_work,
|
|
140
|
+
"dependencies": plugin.dependencies,
|
|
141
|
+
})
|
|
142
|
+
return info
|
|
File without changes
|