datadoc-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
datadoc/__init__.py ADDED
File without changes
File without changes
datadoc/cli/app.py ADDED
@@ -0,0 +1,539 @@
1
+ import typer
2
+ from rich.console import Console
3
+ from rich.table import Table
4
+ from rich.panel import Panel
5
+ from rich.text import Text
6
+ from rich import box
7
+ import os
8
+ import time
9
+
10
+ from datadoc.core.engine import DATADOC
11
+
12
+ app = typer.Typer(
13
+ help="DATADOC: The Open Source Operating System for Dataset Engineering.",
14
+ add_completion=False,
15
+ no_args_is_help=True,
16
+ )
17
+ console = Console()
18
+
19
+ VERSION = "0.1.0"
20
+
21
+ BANNER = r"""
22
+ ____ _ _____ _ ____ ___ ____
23
+ | _ \ / \|_ _|/ \ | _ \ / _ \ / ___|
24
+ | | | |/ _ \ | | / _ \ | | | | | | | |
25
+ | |_| / ___ \| |/ ___ \| |_| | |_| | |___
26
+ |____/_/ \_\_/_/ \_\____/ \___/ \____|
27
+ """
28
+
29
+
30
+ def print_banner():
31
+ banner_text = Text(BANNER, style="bold cyan")
32
+ panel = Panel(
33
+ banner_text,
34
+ subtitle=f"[dim]v{VERSION} - The Open Source OS for Dataset Engineering[/dim]",
35
+ border_style="bright_blue",
36
+ padding=(0, 2),
37
+ )
38
+ console.print(panel)
39
+
40
+
41
+ def print_step(icon: str, message: str, style: str = "bold white"):
42
+ console.print(f" {icon} [{style}]{message}[/]")
43
+
44
+
45
+ def load_dataset(file_path: str) -> DATADOC:
46
+ if not os.path.exists(file_path):
47
+ console.print(f"\n [bold red][X] File not found:[/] {file_path}")
48
+ raise typer.Exit(code=1)
49
+
50
+ print_step("[>>]", f"Loading [cyan]{file_path}[/cyan]...")
51
+ try:
52
+ doc = DATADOC(file_path)
53
+ except Exception as e:
54
+ console.print(f"\n [bold red][X] Failed to read dataset:[/] {e}")
55
+ raise typer.Exit(code=1)
56
+
57
+ rows, cols = doc.df.height, doc.df.width
58
+ print_step("[OK]", f"Loaded [green]{rows:,}[/green] rows x [green]{cols}[/green] columns", "bold green")
59
+ return doc
60
+
61
+
62
+ # ──────────────────────────────────────────────────────────────
63
+ # COMMAND: analyze
64
+ # ──────────────────────────────────────────────────────────────
65
+ @app.command()
66
+ def analyze(file_path: str):
67
+ """
68
+ Scans the dataset and returns a rich health report.
69
+ """
70
+ print_banner()
71
+ doc = load_dataset(file_path)
72
+
73
+ with console.status("[bold cyan]Running health analysis...", spinner="dots"):
74
+ report = doc.analyze()
75
+ print_step("[OK]", "Analysis complete.", "bold green")
76
+
77
+ console.print()
78
+ table = Table(
79
+ title="[bold]Health Report[/bold]",
80
+ box=box.ROUNDED,
81
+ title_style="bold cyan",
82
+ header_style="bold bright_white",
83
+ border_style="bright_blue",
84
+ show_lines=True,
85
+ )
86
+ table.add_column("Metric", justify="left", style="white", min_width=25)
87
+ table.add_column("Value", justify="center", style="bold", min_width=10)
88
+ table.add_column("Status", justify="center", min_width=15)
89
+
90
+ table.add_row("Rows", f"{report['rows']:,}", "[green]--[/green]")
91
+ table.add_row("Columns", str(report["cols"]), "[green]--[/green]")
92
+
93
+ for p_name, p_stats in report["plugins"].items():
94
+ if p_name == "MissingValuePlugin":
95
+ total = p_stats["total_missing"]
96
+ status = "[green][OK] Clean[/green]" if total == 0 else f"[red][!!] {total} found[/red]"
97
+ table.add_row("Missing Values", str(total), status)
98
+ elif p_name == "OutlierPlugin":
99
+ count = len(p_stats.get("outlier_columns", []))
100
+ status = "[green][OK] Clean[/green]" if count == 0 else f"[yellow][!!] {count} cols[/yellow]"
101
+ table.add_row("Outlier Columns", str(count), status)
102
+ elif p_name == "DatetimePlugin":
103
+ count = len(p_stats.get("datetime_columns", []))
104
+ if count > 0:
105
+ cols_list = ", ".join(p_stats["datetime_columns"])
106
+ table.add_row("Datetime Columns", str(count), f"[cyan]{cols_list}[/cyan]")
107
+ else:
108
+ table.add_row("Datetime Columns", "0", "[dim]None[/dim]")
109
+ elif p_name == "CategoricalEncoderPlugin":
110
+ count = len(p_stats.get("categorical_columns", []))
111
+ if count > 0:
112
+ cols_list = ", ".join(p_stats["categorical_columns"])
113
+ table.add_row("Categorical Columns", str(count), f"[cyan]{cols_list}[/cyan]")
114
+ else:
115
+ table.add_row("Categorical Columns", "0", "[dim]None[/dim]")
116
+ elif p_name == "ScalingPlugin":
117
+ has_issue = p_stats.get("has_scale_issues", False)
118
+ ratio = p_stats.get("scale_ratio", 0)
119
+ if has_issue:
120
+ table.add_row("Scale Mismatch", f"{ratio}x", f"[yellow][!!] Needs scaling[/yellow]")
121
+ else:
122
+ table.add_row("Scale Mismatch", "--", "[green][OK] Balanced[/green]")
123
+
124
+ console.print(table)
125
+ console.print()
126
+
127
+
128
+ # ──────────────────────────────────────────────────────────────
129
+ # COMMAND: recommend
130
+ # ──────────────────────────────────────────────────────────────
131
+ @app.command()
132
+ def recommend(file_path: str):
133
+ """
134
+ Outputs a list of suggested engineering steps without applying them.
135
+ """
136
+ print_banner()
137
+ doc = load_dataset(file_path)
138
+
139
+ with console.status("[bold cyan]Generating recommendations...", spinner="dots"):
140
+ recommendations = doc.recommend()
141
+ print_step("[OK]", "Recommendations ready.", "bold green")
142
+
143
+ console.print()
144
+ if not recommendations:
145
+ console.print(Panel(
146
+ "[bold green][OK] Dataset looks perfectly healthy! No recommendations.[/bold green]",
147
+ border_style="green",
148
+ ))
149
+ return
150
+
151
+ rec_table = Table(
152
+ title="[bold]Recommended Actions[/bold]",
153
+ box=box.ROUNDED,
154
+ title_style="bold yellow",
155
+ header_style="bold bright_white",
156
+ border_style="yellow",
157
+ show_lines=True,
158
+ )
159
+ rec_table.add_column("#", justify="center", style="bold yellow", width=4)
160
+ rec_table.add_column("Recommendation", style="white")
161
+
162
+ for i, rec in enumerate(recommendations, 1):
163
+ rec_table.add_row(str(i), rec)
164
+
165
+ console.print(rec_table)
166
+ console.print()
167
+ console.print(" [dim]Run [bold cyan]datadoc engineer <file>[/bold cyan] to apply these automatically.[/dim]\n")
168
+
169
+
170
+ # ──────────────────────────────────────────────────────────────
171
+ # COMMAND: engineer
172
+ # ──────────────────────────────────────────────────────────────
173
+ @app.command()
174
+ def engineer(file_path: str):
175
+ """
176
+ Automatically applies best-practice pipelines.
177
+ """
178
+ print_banner()
179
+ doc = load_dataset(file_path)
180
+
181
+ console.print()
182
+ with console.status("[bold cyan]Running Rule Engine...", spinner="dots") as status:
183
+ def on_progress(plugin_name, p_status, details):
184
+ if p_status == "running":
185
+ status.update(f"[bold cyan]Running [yellow]{plugin_name}[/yellow]...[/bold cyan]")
186
+ time.sleep(0.5) # Add a small delay to make the UI update visible to the user
187
+ elif p_status == "applied":
188
+ print_step("[>>]", f"Applied [bold cyan]{plugin_name}[/bold cyan]", "bold white")
189
+ for detail in details:
190
+ console.print(f" [dim]-> {detail}[/dim]")
191
+ elif p_status == "skipped":
192
+ print_step("[--]", f"Skipped [dim]{plugin_name}[/dim] (not needed)", "dim white")
193
+
194
+ clean_df = doc.engineer(progress_callback=on_progress)
195
+
196
+ output_path = f"clean_{os.path.basename(file_path)}"
197
+ clean_df.write_csv(output_path)
198
+
199
+ console.print()
200
+ console.print(Panel(
201
+ f"[bold green][OK] Success![/bold green]\n\n"
202
+ f" Input: [cyan]{file_path}[/cyan] ({doc.df.height} rows x {doc.df.width} cols)\n"
203
+ f" Output: [cyan]{output_path}[/cyan] ({clean_df.height} rows x {clean_df.width} cols)\n\n"
204
+ f" Applied: {', '.join(doc._applied_plugins) or 'None'}\n"
205
+ f" Skipped: {', '.join(doc._skipped_plugins) or 'None'}",
206
+ title="[bold]Engineering Complete[/bold]",
207
+ border_style="green",
208
+ ))
209
+ console.print()
210
+
211
+
212
+ # ──────────────────────────────────────────────────────────────
213
+ # COMMAND: compare
214
+ # ──────────────────────────────────────────────────────────────
215
+ @app.command()
216
+ def compare(file_path: str):
217
+ """
218
+ Shows a diff-like comparison between the raw and engineered dataset.
219
+ """
220
+ print_banner()
221
+ doc = load_dataset(file_path)
222
+
223
+ with console.status("[bold cyan]Engineering dataset for comparison...", spinner="dots"):
224
+ clean_df = doc.engineer()
225
+ diff = doc.compare(clean_df)
226
+
227
+ console.print()
228
+ table = Table(
229
+ title="[bold]Before vs After Comparison[/bold]",
230
+ box=box.ROUNDED,
231
+ title_style="bold magenta",
232
+ header_style="bold bright_white",
233
+ border_style="magenta",
234
+ show_lines=True,
235
+ )
236
+ table.add_column("Metric", style="white", min_width=20)
237
+ table.add_column("Before", justify="center", style="red", min_width=15)
238
+ table.add_column("After", justify="center", style="green", min_width=15)
239
+
240
+ orig_r, orig_c = diff["original_shape"]
241
+ clean_r, clean_c = diff["clean_shape"]
242
+ table.add_row("Rows", str(orig_r), str(clean_r))
243
+ table.add_row("Columns", str(orig_c), str(clean_c))
244
+ table.add_row("Missing Values", str(diff["original_missing"]), str(diff["clean_missing"]))
245
+ table.add_row(
246
+ "Columns Added",
247
+ "--",
248
+ f"+{diff['cols_added']}" if diff["cols_added"] > 0 else "0"
249
+ )
250
+
251
+ # Data type breakdown
252
+ for dtype, count in diff["original_dtypes"].items():
253
+ dtype_str = str(dtype)
254
+ clean_count = diff["clean_dtypes"].get(dtype, 0)
255
+ table.add_row(f"dtype: {dtype_str}", str(count), str(clean_count))
256
+
257
+ # Check for new dtypes in clean that weren't in original
258
+ for dtype, count in diff["clean_dtypes"].items():
259
+ if dtype not in diff["original_dtypes"]:
260
+ table.add_row(f"dtype: {str(dtype)}", "0", str(count))
261
+
262
+ console.print(table)
263
+
264
+ # Show applied/skipped plugins
265
+ console.print()
266
+ if hasattr(doc, '_applied_plugins'):
267
+ print_step("[OK]", f"Applied: {', '.join(doc._applied_plugins)}", "bold green")
268
+ print_step("[--]", f"Skipped: {', '.join(doc._skipped_plugins)}", "dim")
269
+ console.print()
270
+
271
+
272
+ # ──────────────────────────────────────────────────────────────
273
+ # COMMAND: visualize
274
+ # ──────────────────────────────────────────────────────────────
275
+ @app.command()
276
+ def visualize(file_path: str):
277
+ """
278
+ Generates a massive, interactive terminal dashboard comparing the before and after states.
279
+ """
280
+ import plotext as plt
281
+ import numpy as np
282
+ import sys
283
+ import polars as pl
284
+
285
+ if sys.platform == "win32":
286
+ sys.stdout.reconfigure(encoding="utf-8")
287
+
288
+ print_banner()
289
+ doc = load_dataset(file_path)
290
+
291
+ with console.status("[bold cyan]Generating terminal dashboard...", spinner="dots"):
292
+ clean_df = doc.engineer()
293
+
294
+ orig_missing = {c: doc.df[c].null_count() for c in doc.df.columns}
295
+ clean_missing = {c: clean_df[c].null_count() for c in clean_df.columns}
296
+ cols_with_missing = [c for c, count in orig_missing.items() if count > 0]
297
+
298
+ # Missing values bar chart
299
+ if cols_with_missing:
300
+ plt.clf()
301
+ plt.theme("dark")
302
+ orig_vals = [orig_missing[c] for c in cols_with_missing]
303
+ clean_vals = [clean_missing.get(c, 0) for c in cols_with_missing]
304
+
305
+ plt.multiple_bar(cols_with_missing, [orig_vals, clean_vals], labels=["Before", "After"])
306
+ plt.title("Missing Values Resolution")
307
+ plt.plotsize(100, 20)
308
+ missing_plot = plt.build()
309
+ else:
310
+ missing_plot = " [dim]No missing values found in the original dataset.[/dim]"
311
+
312
+ # Numerical distributions
313
+ num_cols = [c for c in doc.df.columns if doc.df[c].dtype in pl.NUMERIC_DTYPES]
314
+ dist_plots = []
315
+
316
+ for col in num_cols:
317
+ if col not in clean_df.columns:
318
+ continue
319
+
320
+ orig_data = doc.df[col].drop_nulls()
321
+ clean_data = clean_df[col].drop_nulls()
322
+
323
+ if len(orig_data) == 0 or len(clean_data) == 0:
324
+ continue
325
+
326
+ plt.clf()
327
+ plt.theme("dark")
328
+
329
+ plt.hist(orig_data.to_list(), bins=20, label="Before", color="red")
330
+ plt.hist(clean_data.to_list(), bins=20, label="After", color="green")
331
+ plt.title(f"Distribution: {col}")
332
+ plt.plotsize(100, 20)
333
+ dist_plots.append(plt.build())
334
+
335
+ # Render directly to terminal
336
+ console.print()
337
+ console.print(Panel(
338
+ Text.from_ansi(missing_plot) if "No missing" not in missing_plot else missing_plot,
339
+ title="[bold yellow]Missing Values Breakdown[/bold yellow]",
340
+ border_style="yellow"
341
+ ))
342
+
343
+ for i, d_plot in enumerate(dist_plots):
344
+ console.print(Panel(
345
+ Text.from_ansi(d_plot),
346
+ title=f"[bold cyan]Numerical Distribution: {num_cols[i]}[/bold cyan]",
347
+ border_style="cyan"
348
+ ))
349
+
350
+ console.print(Panel(
351
+ f"[bold green]Terminal dashboard generated successfully![/bold green]\n\n"
352
+ f" Applied Plugins: {', '.join(doc._applied_plugins) or 'None'}",
353
+ title="[bold]Summary[/bold]",
354
+ border_style="green",
355
+ ))
356
+
357
+
358
+ # ──────────────────────────────────────────────────────────────
359
+ # COMMAND: pipeline
360
+ # ──────────────────────────────────────────────────────────────
361
+ @app.command()
362
+ def pipeline(file_path: str):
363
+ """
364
+ Exports the generated pipeline as a standalone .py script.
365
+ """
366
+ print_banner()
367
+ doc = load_dataset(file_path)
368
+
369
+ with console.status("[bold cyan]Generating Python pipeline...", spinner="dots"):
370
+ script = doc.pipeline()
371
+ print_step("[OK]", "Pipeline generated.", "bold green")
372
+
373
+ output_path = f"pipeline_{os.path.basename(file_path).split('.')[0]}.py"
374
+
375
+ with open(output_path, "w") as f:
376
+ f.write(script)
377
+
378
+ console.print()
379
+ console.print(Panel(
380
+ f"[bold green][OK] Pipeline generated![/bold green]\n\n"
381
+ f" Saved to: [bold cyan]{output_path}[/bold cyan]\n\n"
382
+ f" [dim]Run it with:[/dim] [bold white]python {output_path}[/bold white]",
383
+ title="[bold]Pipeline Export[/bold]",
384
+ border_style="cyan",
385
+ ))
386
+ console.print()
387
+
388
+
389
+ # ──────────────────────────────────────────────────────────────
390
+ # COMMAND: report
391
+ # ──────────────────────────────────────────────────────────────
392
+ @app.command()
393
+ def report(file_path: str):
394
+ """
395
+ Generates a Markdown report summarizing the dataset and recommendations.
396
+ """
397
+ print_banner()
398
+ doc = load_dataset(file_path)
399
+
400
+ with console.status("[bold cyan]Generating report...", spinner="dots"):
401
+ report_data = doc.analyze()
402
+ recommendations = doc.recommend()
403
+ plugin_info = doc.list_plugins()
404
+
405
+ md_lines = [
406
+ f"# DATADOC Report: {os.path.basename(file_path)}",
407
+ "",
408
+ "---",
409
+ "",
410
+ "## Dataset Overview",
411
+ "",
412
+ f"| Metric | Value |",
413
+ f"|--------|-------|",
414
+ f"| File | `{file_path}` |",
415
+ f"| Rows | {report_data['rows']:,} |",
416
+ f"| Columns | {report_data['cols']} |",
417
+ "",
418
+ ]
419
+
420
+ # Plugin analysis details
421
+ md_lines.append("## Health Analysis")
422
+ md_lines.append("")
423
+ for p_name, p_stats in report_data["plugins"].items():
424
+ md_lines.append(f"### {p_name}")
425
+ md_lines.append("")
426
+ for key, val in p_stats.items():
427
+ md_lines.append(f"- **{key}**: `{val}`")
428
+ md_lines.append("")
429
+
430
+ # Recommendations
431
+ md_lines.append("## Recommendations")
432
+ md_lines.append("")
433
+ if recommendations:
434
+ for i, rec in enumerate(recommendations, 1):
435
+ md_lines.append(f"{i}. {rec}")
436
+ else:
437
+ md_lines.append("No issues found. Dataset is healthy!")
438
+ md_lines.append("")
439
+
440
+ # Plugin registry
441
+ md_lines.append("## Plugin Registry")
442
+ md_lines.append("")
443
+ md_lines.append("| Plugin | Version | Priority | Will Trigger |")
444
+ md_lines.append("|--------|---------|----------|-------------|")
445
+ for p in plugin_info:
446
+ trigger = "Yes" if p["will_trigger"] else "No"
447
+ md_lines.append(f"| {p['name']} | {p['version']} | {p['priority']} | {trigger} |")
448
+ md_lines.append("")
449
+ md_lines.append("---")
450
+ md_lines.append(f"*Generated by DATADOC v{VERSION}*")
451
+ md_lines.append("")
452
+
453
+ output_path = f"report_{os.path.basename(file_path).split('.')[0]}.md"
454
+ with open(output_path, "w") as f:
455
+ f.write("\n".join(md_lines))
456
+
457
+ console.print()
458
+ console.print(Panel(
459
+ f"[bold green][OK] Report generated![/bold green]\n\n"
460
+ f" Saved to: [bold cyan]{output_path}[/bold cyan]",
461
+ title="[bold]Report Export[/bold]",
462
+ border_style="cyan",
463
+ ))
464
+ console.print()
465
+
466
+
467
+ # ──────────────────────────────────────────────────────────────
468
+ # COMMAND: plugin
469
+ # ──────────────────────────────────────────────────────────────
470
+ @app.command(name="plugin")
471
+ def plugin_list():
472
+ """
473
+ Lists all registered plugins and their status.
474
+ """
475
+ print_banner()
476
+
477
+ # Create a temporary DATADOC instance with a minimal df just to list plugins
478
+ # We don't need a real file for this
479
+ import io
480
+ dummy_csv = b"a,b\n1,2\n"
481
+ import polars as pl
482
+ dummy_df = pl.read_csv(dummy_csv)
483
+
484
+ from datadoc.plugins.missing_values import MissingValuePlugin
485
+ from datadoc.plugins.outliers import OutlierPlugin
486
+ from datadoc.plugins.datetime_feat import DatetimePlugin
487
+ from datadoc.plugins.encoders import CategoricalEncoderPlugin
488
+ from datadoc.plugins.scaling import ScalingPlugin
489
+
490
+ plugins = sorted([
491
+ MissingValuePlugin(),
492
+ OutlierPlugin(),
493
+ DatetimePlugin(),
494
+ CategoricalEncoderPlugin(),
495
+ ScalingPlugin(),
496
+ ], key=lambda p: p.priority)
497
+
498
+ console.print()
499
+ table = Table(
500
+ title="[bold]Registered Plugins[/bold]",
501
+ box=box.ROUNDED,
502
+ title_style="bold cyan",
503
+ header_style="bold bright_white",
504
+ border_style="bright_blue",
505
+ show_lines=True,
506
+ )
507
+ table.add_column("Priority", justify="center", style="yellow", width=10)
508
+ table.add_column("Plugin", style="bold white", min_width=25)
509
+ table.add_column("Version", justify="center", style="cyan", width=10)
510
+ table.add_column("Description", style="dim white")
511
+
512
+ for p in plugins:
513
+ table.add_row(str(p.priority), p.name, p.version, p.description)
514
+
515
+ console.print(table)
516
+
517
+ # Plugin explanations
518
+ console.print()
519
+ console.print(Panel(
520
+ "\n".join([f" [bold cyan]{p.name}[/bold cyan]: {p.explain()}" for p in plugins]),
521
+ title="[bold]Plugin Explanations[/bold]",
522
+ border_style="dim",
523
+ ))
524
+ console.print()
525
+
526
+
527
+ # ──────────────────────────────────────────────────────────────
528
+ # COMMAND: version
529
+ # ──────────────────────────────────────────────────────────────
530
+ @app.command()
531
+ def version():
532
+ """
533
+ Displays the current DATADOC version.
534
+ """
535
+ print_banner()
536
+
537
+
538
+ if __name__ == "__main__":
539
+ app()
File without changes
datadoc/core/engine.py ADDED
@@ -0,0 +1,142 @@
1
+ import polars as pl
2
+ from typing import Dict, Any, List
3
+ from collections import Counter
4
+ from datadoc.plugins.base import BasePlugin
5
+ from datadoc.plugins.missing_values import MissingValuePlugin
6
+ from datadoc.plugins.outliers import OutlierPlugin
7
+ from datadoc.plugins.datetime_feat import DatetimePlugin
8
+ from datadoc.plugins.encoders import CategoricalEncoderPlugin
9
+ from datadoc.plugins.scaling import ScalingPlugin
10
+
11
+ class DATADOC:
12
+ def __init__(self, file_path: str):
13
+ self.file_path = file_path
14
+ self.df = pl.read_csv(file_path, infer_schema_length=10000)
15
+ self._original_df = self.df.clone()
16
+
17
+ # Plugins sorted by priority (lower = runs first)
18
+ self.plugins: List[BasePlugin] = sorted([
19
+ MissingValuePlugin(),
20
+ OutlierPlugin(),
21
+ DatetimePlugin(),
22
+ CategoricalEncoderPlugin(),
23
+ ScalingPlugin(),
24
+ ], key=lambda p: p.priority)
25
+
26
+ def analyze(self) -> Dict[str, Any]:
27
+ """Runs the analyze step across all plugins."""
28
+ dtype_counts = Counter([str(dt) for dt in self.df.dtypes])
29
+ report = {
30
+ "rows": self.df.height,
31
+ "cols": self.df.width,
32
+ "dtypes": dict(dtype_counts),
33
+ "plugins": {}
34
+ }
35
+
36
+ for plugin in self.plugins:
37
+ report["plugins"][plugin.name] = plugin.analyze(self.df)
38
+
39
+ return report
40
+
41
+ def recommend(self) -> list[str]:
42
+ """Runs analysis and aggregates recommendations from all plugins."""
43
+ recommendations = []
44
+ for plugin in self.plugins:
45
+ analysis = plugin.analyze(self.df)
46
+ recs = plugin.recommend(analysis)
47
+ recommendations.extend(recs)
48
+ return recommendations
49
+
50
+ def pipeline(self) -> str:
51
+ """Generates the full Python script representing the pipeline."""
52
+ lines = [
53
+ "import polars as pl",
54
+ "import numpy as np",
55
+ "",
56
+ "",
57
+ f"def load_and_clean_data(file_path: str = '{self.file_path}') -> pl.DataFrame:",
58
+ " df = pl.read_csv(file_path, infer_schema_length=10000)",
59
+ "",
60
+ ]
61
+
62
+ for plugin in self.plugins:
63
+ analysis = plugin.analyze(self.df)
64
+ should_apply = any(bool(v) for k, v in analysis.items() if k.startswith('has_'))
65
+ if should_apply:
66
+ code_snippet = plugin.generate_code(analysis)
67
+ if code_snippet:
68
+ # Indent each line of the code snippet
69
+ for line in code_snippet.split("\n"):
70
+ lines.append(f" {line}")
71
+ lines.append("")
72
+
73
+ lines.append(" return df")
74
+ lines.append("")
75
+ lines.append("")
76
+ lines.append("if __name__ == '__main__':")
77
+ lines.append(" clean_df = load_and_clean_data()")
78
+ lines.append(" print(f'Successfully processed {clean_df.height} rows x {clean_df.width} columns!')")
79
+ lines.append("")
80
+
81
+ return "\n".join(lines)
82
+
83
+ def engineer(self, progress_callback=None) -> pl.DataFrame:
84
+ """Automatically triggers plugins that are needed."""
85
+ df_transformed = self.df.clone()
86
+
87
+ self._applied_plugins = []
88
+ self._skipped_plugins = []
89
+
90
+ for plugin in self.plugins:
91
+ if progress_callback:
92
+ progress_callback(plugin.name, "running", [])
93
+
94
+ analysis = plugin.analyze(df_transformed)
95
+ should_apply = any(bool(v) for k, v in analysis.items() if k.startswith('has_'))
96
+ if should_apply:
97
+ details = plugin.recommend(analysis)
98
+ df_transformed = plugin.apply(df_transformed)
99
+ self._applied_plugins.append(plugin.name)
100
+ if progress_callback:
101
+ progress_callback(plugin.name, "applied", details)
102
+ else:
103
+ self._skipped_plugins.append(plugin.name)
104
+ if progress_callback:
105
+ progress_callback(plugin.name, "skipped", [])
106
+
107
+ return df_transformed
108
+
109
+ def compare(self, clean_df: pl.DataFrame) -> Dict[str, Any]:
110
+ """Compare original and engineered dataframes."""
111
+ original = self.df
112
+
113
+ orig_missing = sum(original[c].null_count() for c in original.columns)
114
+ clean_missing = sum(clean_df[c].null_count() for c in clean_df.columns)
115
+
116
+ result = {
117
+ "original_shape": original.shape,
118
+ "clean_shape": clean_df.shape,
119
+ "rows_changed": original.height != clean_df.height,
120
+ "cols_added": clean_df.width - original.width,
121
+ "original_missing": orig_missing,
122
+ "clean_missing": clean_missing,
123
+ "original_dtypes": dict(Counter([str(dt) for dt in original.dtypes])),
124
+ "clean_dtypes": dict(Counter([str(dt) for dt in clean_df.dtypes])),
125
+ }
126
+ return result
127
+
128
+ def list_plugins(self) -> List[Dict[str, Any]]:
129
+ """Returns info about all registered plugins."""
130
+ info = []
131
+ for plugin in self.plugins:
132
+ analysis = plugin.analyze(self.df)
133
+ has_work = any(bool(v) for k, v in analysis.items() if k.startswith('has_'))
134
+ info.append({
135
+ "name": plugin.name,
136
+ "version": plugin.version,
137
+ "description": plugin.description,
138
+ "priority": plugin.priority,
139
+ "will_trigger": has_work,
140
+ "dependencies": plugin.dependencies,
141
+ })
142
+ return info
File without changes