data-advisor 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_advisor/__init__.py +38 -0
- data_advisor/__main__.py +9 -0
- data_advisor/advisor.py +436 -0
- data_advisor/cli.py +160 -0
- data_advisor/codegen.py +515 -0
- data_advisor/rules.py +380 -0
- data_advisor/scanner.py +655 -0
- data_advisor/search.py +679 -0
- data_advisor/visualizer.py +236 -0
- data_advisor-0.1.0.dist-info/METADATA +303 -0
- data_advisor-0.1.0.dist-info/RECORD +14 -0
- data_advisor-0.1.0.dist-info/WHEEL +5 -0
- data_advisor-0.1.0.dist-info/entry_points.txt +2 -0
- data_advisor-0.1.0.dist-info/top_level.txt +1 -0
data_advisor/__init__.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""
|
|
2
|
+
data_advisor - Production-grade diagnostic advisor and code generator for data science examinations.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from data_advisor.advisor import DataAdvisor, data_advisor
|
|
6
|
+
from data_advisor.codegen import CodeGenerator
|
|
7
|
+
from data_advisor.rules import Recommendation, RuleEngine
|
|
8
|
+
from data_advisor.scanner import DataScanner, load_dataset
|
|
9
|
+
from data_advisor.search import (
|
|
10
|
+
CodeSearchEngine,
|
|
11
|
+
get_file,
|
|
12
|
+
list_files,
|
|
13
|
+
read_file,
|
|
14
|
+
run_file,
|
|
15
|
+
search,
|
|
16
|
+
view_file,
|
|
17
|
+
)
|
|
18
|
+
from data_advisor.visualizer import VisualizationAdvisor
|
|
19
|
+
|
|
20
|
+
__version__ = "0.1.0"
|
|
21
|
+
__all__ = [
|
|
22
|
+
"data_advisor",
|
|
23
|
+
"DataAdvisor",
|
|
24
|
+
"DataScanner",
|
|
25
|
+
"RuleEngine",
|
|
26
|
+
"CodeGenerator",
|
|
27
|
+
"VisualizationAdvisor",
|
|
28
|
+
"Recommendation",
|
|
29
|
+
"CodeSearchEngine",
|
|
30
|
+
"search",
|
|
31
|
+
"get_file",
|
|
32
|
+
"read_file",
|
|
33
|
+
"view_file",
|
|
34
|
+
"run_file",
|
|
35
|
+
"list_files",
|
|
36
|
+
"load_dataset",
|
|
37
|
+
"__version__",
|
|
38
|
+
]
|
data_advisor/__main__.py
ADDED
data_advisor/advisor.py
ADDED
|
@@ -0,0 +1,436 @@
|
|
|
1
|
+
"""
|
|
2
|
+
advisor.py - The primary DataAdvisor interface class.
|
|
3
|
+
|
|
4
|
+
Orchestrates scanning, rule evaluation, interactive terminal reporting,
|
|
5
|
+
and automated script generation.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import sys
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any, Dict, List, Optional, Union
|
|
13
|
+
|
|
14
|
+
import pandas as pd
|
|
15
|
+
|
|
16
|
+
from data_advisor.codegen import CodeGenerator
|
|
17
|
+
from data_advisor.rules import Recommendation, RuleEngine
|
|
18
|
+
from data_advisor.scanner import DataScanner, load_dataset
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class DataAdvisor:
|
|
22
|
+
"""
|
|
23
|
+
High-level facade for autonomous dataset inspection, diagnostics,
|
|
24
|
+
statistical hypothesis testing, and code generation.
|
|
25
|
+
|
|
26
|
+
Parameters
|
|
27
|
+
----------
|
|
28
|
+
data : Union[str, Path, pd.DataFrame]
|
|
29
|
+
A pandas DataFrame or path to a dataset file (CSV, TSV, Excel, Parquet).
|
|
30
|
+
target : Optional[str], default=None
|
|
31
|
+
Name of the target / dependent variable column. If None, DataAdvisor
|
|
32
|
+
attempts auto-detection based on common naming patterns.
|
|
33
|
+
id_cols : Optional[List[str]], default=None
|
|
34
|
+
Optional list of explicit identifier columns to exclude from modeling.
|
|
35
|
+
|
|
36
|
+
Examples
|
|
37
|
+
--------
|
|
38
|
+
>>> from data_advisor import DataAdvisor
|
|
39
|
+
>>> advisor = DataAdvisor("customer_churn.csv", target="churn")
|
|
40
|
+
>>> results = advisor.scan()
|
|
41
|
+
>>> advisor.report()
|
|
42
|
+
>>> advisor.generate_script("run_analysis.py")
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
def __init__(
|
|
46
|
+
self,
|
|
47
|
+
data: Union[str, Path, pd.DataFrame],
|
|
48
|
+
target: Optional[str] = None,
|
|
49
|
+
id_cols: Optional[List[str]] = None,
|
|
50
|
+
) -> None:
|
|
51
|
+
self.data_source_str = str(data) if isinstance(data, (str, Path)) else "dataset.csv"
|
|
52
|
+
self.df: pd.DataFrame = load_dataset(data)
|
|
53
|
+
self.target: Optional[str] = target
|
|
54
|
+
self.id_cols: List[str] = list(id_cols or [])
|
|
55
|
+
|
|
56
|
+
# Internal components
|
|
57
|
+
self._scanner = DataScanner(self.df, target=self.target, id_cols=self.id_cols)
|
|
58
|
+
self._scan_cache: Optional[Dict[str, Any]] = None
|
|
59
|
+
self._recommendations_cache: Optional[List[Recommendation]] = None
|
|
60
|
+
|
|
61
|
+
# -------------------------------------------------------------------------
|
|
62
|
+
# 1. Scan Method
|
|
63
|
+
# -------------------------------------------------------------------------
|
|
64
|
+
def scan(self) -> Dict[str, Any]:
|
|
65
|
+
"""
|
|
66
|
+
Execute profiling and statistical audits.
|
|
67
|
+
|
|
68
|
+
Returns
|
|
69
|
+
-------
|
|
70
|
+
Dict[str, Any]
|
|
71
|
+
A comprehensive dictionary containing metrics for overview,
|
|
72
|
+
missing values, data types, distributions, categoricals,
|
|
73
|
+
target analysis, hypothesis tests, and recommendations.
|
|
74
|
+
"""
|
|
75
|
+
if self._scan_cache is None:
|
|
76
|
+
results = self._scanner.scan()
|
|
77
|
+
rule_engine = RuleEngine(results)
|
|
78
|
+
recs = rule_engine.generate_all_recommendations()
|
|
79
|
+
|
|
80
|
+
# Attach recommendations into scan dictionary
|
|
81
|
+
results["recommendations"] = [r.to_dict() for r in recs]
|
|
82
|
+
self._scan_cache = results
|
|
83
|
+
self._recommendations_cache = recs
|
|
84
|
+
|
|
85
|
+
return self._scan_cache
|
|
86
|
+
|
|
87
|
+
# -------------------------------------------------------------------------
|
|
88
|
+
# 2. Recommendations & Step Access
|
|
89
|
+
# -------------------------------------------------------------------------
|
|
90
|
+
def get_recommendations(self) -> List[Dict[str, Any]]:
|
|
91
|
+
"""Return the prioritized list of diagnostic recommendations."""
|
|
92
|
+
scan_res = self.scan()
|
|
93
|
+
return scan_res.get("recommendations", [])
|
|
94
|
+
|
|
95
|
+
def get_code_block(self, step: str) -> str:
|
|
96
|
+
"""
|
|
97
|
+
Retrieve executable code for a specific analysis stage:
|
|
98
|
+
'cleaning', 'transformations', 'encoding', 'hypothesis_tests', or 'modeling'.
|
|
99
|
+
"""
|
|
100
|
+
scan_res = self.scan()
|
|
101
|
+
generator = CodeGenerator(scan_res, data_source_str=self.data_source_str)
|
|
102
|
+
|
|
103
|
+
step_map = {
|
|
104
|
+
"cleaning": generator.generate_cleaning_code,
|
|
105
|
+
"transformations": generator.generate_transformations_code,
|
|
106
|
+
"encoding": generator.generate_encoding_code,
|
|
107
|
+
"hypothesis_tests": generator.generate_hypothesis_testing_code,
|
|
108
|
+
"tests": generator.generate_hypothesis_testing_code,
|
|
109
|
+
"modeling": generator.generate_modeling_code,
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
func = step_map.get(step.lower())
|
|
113
|
+
if not func:
|
|
114
|
+
valid_keys = list(step_map.keys())
|
|
115
|
+
raise ValueError(f"Invalid step '{step}'. Choose from: {valid_keys}")
|
|
116
|
+
|
|
117
|
+
return func()
|
|
118
|
+
|
|
119
|
+
# -------------------------------------------------------------------------
|
|
120
|
+
# 3. Code Generation
|
|
121
|
+
# -------------------------------------------------------------------------
|
|
122
|
+
def generate_script(self, filename: Optional[Union[str, Path]] = None) -> str:
|
|
123
|
+
"""
|
|
124
|
+
Produce a complete, self-contained, executable Python script tailored
|
|
125
|
+
to the audited dataset.
|
|
126
|
+
|
|
127
|
+
Parameters
|
|
128
|
+
----------
|
|
129
|
+
filename : Optional[Union[str, Path]], default=None
|
|
130
|
+
If provided, saves the generated script to this file path.
|
|
131
|
+
|
|
132
|
+
Returns
|
|
133
|
+
-------
|
|
134
|
+
str
|
|
135
|
+
The complete Python script string.
|
|
136
|
+
"""
|
|
137
|
+
scan_res = self.scan()
|
|
138
|
+
generator = CodeGenerator(scan_res, data_source_str=self.data_source_str)
|
|
139
|
+
script_code = generator.generate_full_script()
|
|
140
|
+
|
|
141
|
+
if filename:
|
|
142
|
+
file_path = Path(filename)
|
|
143
|
+
file_path.write_text(script_code, encoding="utf-8")
|
|
144
|
+
print(f"\n[+] Production script successfully exported to: {file_path.resolve()}\n")
|
|
145
|
+
|
|
146
|
+
return script_code
|
|
147
|
+
|
|
148
|
+
# -------------------------------------------------------------------------
|
|
149
|
+
# 4. Local Code Search
|
|
150
|
+
# -------------------------------------------------------------------------
|
|
151
|
+
@staticmethod
|
|
152
|
+
def search(
|
|
153
|
+
key: str,
|
|
154
|
+
folder: Optional[Union[str, Path, List[Union[str, Path]]]] = None,
|
|
155
|
+
max_results: int = 3,
|
|
156
|
+
print_output: bool = True,
|
|
157
|
+
) -> str:
|
|
158
|
+
"""
|
|
159
|
+
Search local code files and code banks for matching snippets and implementations.
|
|
160
|
+
|
|
161
|
+
Parameters
|
|
162
|
+
----------
|
|
163
|
+
key : str
|
|
164
|
+
Search keyword, e.g. 'wokwi', 'linear regression', 'dht22', 'coap server'.
|
|
165
|
+
folder : Optional[Union[str, Path, List[Union[str, Path]]]], default=None
|
|
166
|
+
Folder path to search. If None, auto-discovers 'code_bank', 'snippets',
|
|
167
|
+
'examples', or current directory.
|
|
168
|
+
max_results : int, default=3
|
|
169
|
+
Maximum number of matching code files to return.
|
|
170
|
+
print_output : bool, default=True
|
|
171
|
+
Whether to print formatted code to the console.
|
|
172
|
+
|
|
173
|
+
Returns
|
|
174
|
+
-------
|
|
175
|
+
str
|
|
176
|
+
The formatted code text response.
|
|
177
|
+
"""
|
|
178
|
+
from data_advisor.search import search as search_fn
|
|
179
|
+
return search_fn(key=key, folder=folder, max_results=max_results, print_output=print_output)
|
|
180
|
+
|
|
181
|
+
@staticmethod
|
|
182
|
+
def get_file(
|
|
183
|
+
query: Union[str, int],
|
|
184
|
+
folder: Optional[Union[str, Path, List[Union[str, Path]]]] = None,
|
|
185
|
+
print_output: bool = True,
|
|
186
|
+
line_numbers: bool = False,
|
|
187
|
+
) -> str:
|
|
188
|
+
"""
|
|
189
|
+
Retrieve and display the COMPLETE raw file content by question number, filename, or keyword.
|
|
190
|
+
"""
|
|
191
|
+
from data_advisor.search import get_file as get_file_fn
|
|
192
|
+
return get_file_fn(query=query, folder=folder, print_output=print_output, line_numbers=line_numbers)
|
|
193
|
+
|
|
194
|
+
@staticmethod
|
|
195
|
+
def read_file(
|
|
196
|
+
query: Union[str, int],
|
|
197
|
+
folder: Optional[Union[str, Path, List[Union[str, Path]]]] = None,
|
|
198
|
+
) -> str:
|
|
199
|
+
"""Return the COMPLETE raw file content as a string without printing."""
|
|
200
|
+
from data_advisor.search import read_file as read_file_fn
|
|
201
|
+
return read_file_fn(query=query, folder=folder)
|
|
202
|
+
|
|
203
|
+
@staticmethod
|
|
204
|
+
def view_file(
|
|
205
|
+
query: Union[str, int],
|
|
206
|
+
folder: Optional[Union[str, Path, List[Union[str, Path]]]] = None,
|
|
207
|
+
line_numbers: bool = False,
|
|
208
|
+
) -> str:
|
|
209
|
+
"""Display the complete file in the console with metadata header."""
|
|
210
|
+
from data_advisor.search import view_file as view_file_fn
|
|
211
|
+
return view_file_fn(query=query, folder=folder, line_numbers=line_numbers)
|
|
212
|
+
|
|
213
|
+
@staticmethod
|
|
214
|
+
def run_file(
|
|
215
|
+
query: Union[str, int],
|
|
216
|
+
folder: Optional[Union[str, Path, List[Union[str, Path]]]] = None,
|
|
217
|
+
args: Optional[List[str]] = None,
|
|
218
|
+
timeout: Optional[int] = None,
|
|
219
|
+
) -> int:
|
|
220
|
+
"""Execute a Python file directly by question number or filename."""
|
|
221
|
+
from data_advisor.search import run_file as run_file_fn
|
|
222
|
+
return run_file_fn(query=query, folder=folder, args=args, timeout=timeout)
|
|
223
|
+
|
|
224
|
+
@staticmethod
|
|
225
|
+
def list_files(
|
|
226
|
+
folder: Optional[Union[str, Path, List[Union[str, Path]]]] = None,
|
|
227
|
+
print_output: bool = True,
|
|
228
|
+
) -> List[Path]:
|
|
229
|
+
"""List all indexed code snippets with a 1-line summary."""
|
|
230
|
+
from data_advisor.search import list_files as list_files_fn
|
|
231
|
+
return list_files_fn(folder=folder, print_output=print_output)
|
|
232
|
+
|
|
233
|
+
# -------------------------------------------------------------------------
|
|
234
|
+
# 5. CLI / Terminal Reporting Engine
|
|
235
|
+
# -------------------------------------------------------------------------
|
|
236
|
+
def report(self) -> None:
|
|
237
|
+
"""
|
|
238
|
+
Print an executive, structured diagnostic report directly to the terminal
|
|
239
|
+
with rich visual formatting and actionable guidance.
|
|
240
|
+
"""
|
|
241
|
+
data = self.scan()
|
|
242
|
+
overview = data["overview"]
|
|
243
|
+
missing = data["missing_values"]
|
|
244
|
+
dtypes = data["data_types"]
|
|
245
|
+
distributions = data["distributions"]
|
|
246
|
+
categoricals = data["categoricals"]
|
|
247
|
+
target = data["target"]
|
|
248
|
+
hypothesis_tests = data["hypothesis_tests"]
|
|
249
|
+
recommendations = data["recommendations"]
|
|
250
|
+
|
|
251
|
+
divider = "=" * 80
|
|
252
|
+
thin_div = "-" * 80
|
|
253
|
+
|
|
254
|
+
print("\n" + divider)
|
|
255
|
+
print(" DATA ADVISOR - EXAM & PRODUCTION DATASET DIAGNOSTIC REPORT")
|
|
256
|
+
print(divider)
|
|
257
|
+
|
|
258
|
+
# ---------------- Section 1: Overview ----------------
|
|
259
|
+
print("\n[+] SECTION 1: DATASET ARCHITECTURE & HEALTH")
|
|
260
|
+
print(thin_div)
|
|
261
|
+
print(f" * Dimensions : {overview['n_rows']} rows x {overview['n_cols']} columns")
|
|
262
|
+
print(f" * Memory Footprint : {overview['memory_mb']} MB")
|
|
263
|
+
dup_flag = "[WARNING]" if overview["duplicate_rows"] > 0 else "[OK]"
|
|
264
|
+
print(f" * Duplicated Rows : {overview['duplicate_rows']} ({overview['duplicate_pct']}%) --> {dup_flag}")
|
|
265
|
+
|
|
266
|
+
dtypes_str = ", ".join([f"{k}: {v}" for k, v in overview["dtypes_summary"].items()])
|
|
267
|
+
print(f" * Dtypes Breakdown : {dtypes_str}")
|
|
268
|
+
|
|
269
|
+
# ---------------- Section 2: Missing Values & Anomalies ----------------
|
|
270
|
+
print("\n[+] SECTION 2: DATA INTEGRITY & TYPE MISMATCH AUDIT")
|
|
271
|
+
print(thin_div)
|
|
272
|
+
|
|
273
|
+
null_cols = [(c, info) for c, info in missing.items() if info["null_count"] > 0]
|
|
274
|
+
if null_cols:
|
|
275
|
+
print(f" {'Column':<25} | {'Null Count':<10} | {'Null %':<8} | {'Strategy'}")
|
|
276
|
+
print(f" {'-'*25}-|-{'-'*10}-|-{'-'*8}-|-{'-'*28}")
|
|
277
|
+
for col, info in null_cols:
|
|
278
|
+
pct = info["null_pct"]
|
|
279
|
+
strat = "Drop Column" if pct >= 60 else ("Median/Mean" if info["is_numeric"] else "Mode/Missing")
|
|
280
|
+
print(f" {col:<25} | {info['null_count']:<10} | {pct:<7.1f}% | {strat}")
|
|
281
|
+
else:
|
|
282
|
+
print(" * Zero missing values detected across all columns. [OK]")
|
|
283
|
+
|
|
284
|
+
if dtypes.get("potential_dates"):
|
|
285
|
+
print(f" * Potential Date Strings Flagged : {dtypes['potential_dates']}")
|
|
286
|
+
if dtypes.get("potential_ids"):
|
|
287
|
+
print(f" * Unique Identifiers / IDs Flagged: {dtypes['potential_ids']}")
|
|
288
|
+
if dtypes.get("constant_columns"):
|
|
289
|
+
print(f" * Zero-Variance Constant Columns : {dtypes['constant_columns']} [CRITICAL - Drop]")
|
|
290
|
+
if dtypes.get("low_card_numerics"):
|
|
291
|
+
low_cards = [f"{item['column']} ({item['unique_count']} vals)" for item in dtypes["low_card_numerics"]]
|
|
292
|
+
print(f" * Discrete Categorical Numerics : {low_cards}")
|
|
293
|
+
|
|
294
|
+
# ---------------- Section 3: Distributions & Outliers ----------------
|
|
295
|
+
if distributions:
|
|
296
|
+
print("\n[+] SECTION 3: NUMERIC DISTRIBUTIONS & OUTLIER ANOMALIES")
|
|
297
|
+
print(thin_div)
|
|
298
|
+
print(f" {'Column':<20} | {'Skewness':<8} | {'Kurtosis':<8} | {'IQR Outliers':<15} | {'Z>3 Outliers':<12} | {'Transform'}")
|
|
299
|
+
print(f" {'-'*20}-|-{'-'*8}-|-{'-'*8}-|-{'-'*15}-|-{'-'*12}-|-{'-'*12}")
|
|
300
|
+
for col, dist in list(distributions.items())[:10]:
|
|
301
|
+
skew = dist["skewness"]
|
|
302
|
+
kurt = dist["kurtosis"]
|
|
303
|
+
iqr_out = f"{dist['iqr_outliers']} ({dist['iqr_outlier_pct']}%)"
|
|
304
|
+
z_out = f"{dist['z_outliers']} ({dist['z_outlier_pct']}%)"
|
|
305
|
+
|
|
306
|
+
trans = "None"
|
|
307
|
+
if skew > 1.2:
|
|
308
|
+
trans = "log1p" if dist.get("is_non_negative", True) else "Yeo-Johnson"
|
|
309
|
+
elif skew < -1.2:
|
|
310
|
+
trans = "Yeo-Johnson"
|
|
311
|
+
elif dist["iqr_outlier_pct"] > 5.0:
|
|
312
|
+
trans = "IQR Clip"
|
|
313
|
+
|
|
314
|
+
print(f" {col:<20} | {skew:<8.2f} | {kurt:<8.2f} | {iqr_out:<15} | {z_out:<12} | {trans}")
|
|
315
|
+
|
|
316
|
+
# ---------------- Section 4: Categorical Processing ----------------
|
|
317
|
+
if categoricals:
|
|
318
|
+
print("\n[+] SECTION 4: CATEGORICAL ENCODING STRATEGY")
|
|
319
|
+
print(thin_div)
|
|
320
|
+
print(f" {'Column':<22} | {'Levels':<7} | {'Top Category':<18} | {'Recommended Encoding'}")
|
|
321
|
+
print(f" {'-'*22}-|-{'-'*7}-|-{'-'*18}-|-{'-'*25}")
|
|
322
|
+
for col, cat in list(categoricals.items())[:10]:
|
|
323
|
+
card = cat["cardinality"]
|
|
324
|
+
top_v = f"{cat['top_value']} ({cat['top_value_pct']}%)"
|
|
325
|
+
rec_enc = "Binary (0/1)" if card == 2 else ("One-Hot (drop_first)" if card <= 10 else "Frequency / Target")
|
|
326
|
+
print(f" {col:<22} | {card:<7} | {top_v:<18} | {rec_enc}")
|
|
327
|
+
|
|
328
|
+
# ---------------- Section 5: Hypothesis Testing ----------------
|
|
329
|
+
if hypothesis_tests:
|
|
330
|
+
print("\n[+] SECTION 5: STATISTICAL HYPOTHESIS TESTING RESULTS")
|
|
331
|
+
print(thin_div)
|
|
332
|
+
for idx, test in enumerate(hypothesis_tests[:6], 1):
|
|
333
|
+
f1, f2 = test["feature_1"], test["feature_2"]
|
|
334
|
+
sig = "[SIGNIFICANT]" if test["is_significant"] else "[NOT SIGNIFICANT]"
|
|
335
|
+
p_str = f"{test['p_value']:.2e}" if test["p_value"] < 0.001 else f"{test['p_value']:.4f}"
|
|
336
|
+
print(f" ({idx}) {test['test_type']} : {f1} vs {f2} --> {sig}")
|
|
337
|
+
print(f" Statistic: {test['statistic']} | p-value: {p_str}")
|
|
338
|
+
print(f" Interpretation: {test['conclusion']}")
|
|
339
|
+
|
|
340
|
+
# ---------------- Section 6: Target & Modeling Strategy ----------------
|
|
341
|
+
if target:
|
|
342
|
+
print("\n[+] SECTION 6: TARGET VARIABLE & MODEL SELECTION")
|
|
343
|
+
print(thin_div)
|
|
344
|
+
t_col = target["target_column"]
|
|
345
|
+
p_type = target["problem_type"]
|
|
346
|
+
print(f" * Target Column : '{t_col}'")
|
|
347
|
+
print(f" * Problem Type : {p_type.upper()}")
|
|
348
|
+
|
|
349
|
+
if p_type == "classification":
|
|
350
|
+
imb_flag = "[CRITICAL IMBALANCE]" if target.get("is_imbalanced") else "[BALANCED]"
|
|
351
|
+
print(f" * Classes : {target['n_classes']} levels")
|
|
352
|
+
print(f" * Class Balance : Ratio 1:{target.get('imbalance_ratio', 1.0):.1f} (Minority: {target.get('minority_percentage', 50)}%) --> {imb_flag}")
|
|
353
|
+
print(" * Models Baseline: LogisticRegression(class_weight='balanced') -> RandomForestClassifier")
|
|
354
|
+
print(" * Primary Metrics: F1-Score (macro/weighted), PR-AUC, Confusion Matrix")
|
|
355
|
+
else:
|
|
356
|
+
print(f" * Target Range : [{target['min']} to {target['max']}] (Mean: {target['mean']}, Std: {target['std']})")
|
|
357
|
+
print(" * Models Baseline: LinearRegression -> Ridge(alpha=1.0) -> RandomForestRegressor")
|
|
358
|
+
print(" * Primary Metrics: RMSE, MAE, R² Score")
|
|
359
|
+
|
|
360
|
+
# ---------------- Section 7: Prioritized Action Plan ----------------
|
|
361
|
+
print("\n[+] SECTION 7: PRIORITIZED ACTION PLAN")
|
|
362
|
+
print(thin_div)
|
|
363
|
+
for idx, rec in enumerate(recommendations[:8], 1):
|
|
364
|
+
badge = f"[{rec.get('severity')}]"
|
|
365
|
+
col_target = f" on '{rec.get('column')}'" if rec.get('column') else ""
|
|
366
|
+
print(f" {idx}. {badge:<14} {rec.get('action')}{col_target}")
|
|
367
|
+
print(f" Why: {rec.get('rationale')}")
|
|
368
|
+
|
|
369
|
+
print(divider + "\n")
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def data_advisor(
|
|
373
|
+
data: Union[str, Path, pd.DataFrame],
|
|
374
|
+
target: Optional[str] = None,
|
|
375
|
+
output_script: Optional[Union[str, Path]] = None,
|
|
376
|
+
id_cols: Optional[List[str]] = None,
|
|
377
|
+
print_report: bool = True,
|
|
378
|
+
print_code: bool = True,
|
|
379
|
+
return_script: bool = False,
|
|
380
|
+
) -> Union[DataAdvisor, str]:
|
|
381
|
+
"""
|
|
382
|
+
Given a CSV file (or DataFrame), diagnoses what all analysis should be done
|
|
383
|
+
and immediately outputs the complete executable Python code to do that analysis.
|
|
384
|
+
|
|
385
|
+
Parameters
|
|
386
|
+
----------
|
|
387
|
+
data : Union[str, Path, pd.DataFrame]
|
|
388
|
+
A CSV file path (e.g. "dataset.csv") or an existing pandas DataFrame.
|
|
389
|
+
target : Optional[str], default=None
|
|
390
|
+
Target/dependent variable column name (if known). If None, auto-detected.
|
|
391
|
+
output_script : Optional[Union[str, Path]], default=None
|
|
392
|
+
Optional file path to save the generated Python script (e.g. "solution.py").
|
|
393
|
+
id_cols : Optional[List[str]], default=None
|
|
394
|
+
Optional list of identifier columns to exclude from modeling.
|
|
395
|
+
print_report : bool, default=True
|
|
396
|
+
Whether to print the structured diagnostic report (what all analysis to do).
|
|
397
|
+
print_code : bool, default=True
|
|
398
|
+
Whether to print the complete executable Python code to do that analysis.
|
|
399
|
+
return_script : bool, default=False
|
|
400
|
+
If True, returns the generated script string. Otherwise returns the DataAdvisor instance.
|
|
401
|
+
|
|
402
|
+
Returns
|
|
403
|
+
-------
|
|
404
|
+
Union[DataAdvisor, str]
|
|
405
|
+
The DataAdvisor instance (or the script string if return_script=True).
|
|
406
|
+
|
|
407
|
+
Examples
|
|
408
|
+
--------
|
|
409
|
+
>>> from data_advisor import data_advisor
|
|
410
|
+
>>> data_advisor("exam_dataset.csv")
|
|
411
|
+
>>> data_advisor("exam_dataset.csv", target="churn", output_script="solution.py")
|
|
412
|
+
"""
|
|
413
|
+
advisor = DataAdvisor(data=data, target=target, id_cols=id_cols)
|
|
414
|
+
|
|
415
|
+
# 1. Output what all analysis should be done (Diagnostic Report)
|
|
416
|
+
if print_report:
|
|
417
|
+
advisor.report()
|
|
418
|
+
|
|
419
|
+
# 2. Generate the complete executable code to do that analysis
|
|
420
|
+
script_code = advisor.generate_script(filename=output_script)
|
|
421
|
+
|
|
422
|
+
# 3. Print the code to the console
|
|
423
|
+
if print_code:
|
|
424
|
+
divider = "=" * 80
|
|
425
|
+
print("\n" + divider)
|
|
426
|
+
print(" DATA ADVISOR - EXECUTABLE PYTHON CODE FOR ALL RECOMMENDED ANALYSES")
|
|
427
|
+
print(" Copy-paste and run this complete pipeline in your Jupyter Notebook or script:")
|
|
428
|
+
print(divider + "\n")
|
|
429
|
+
print(script_code)
|
|
430
|
+
print("\n" + divider + "\n")
|
|
431
|
+
|
|
432
|
+
if return_script:
|
|
433
|
+
return script_code
|
|
434
|
+
|
|
435
|
+
return advisor
|
|
436
|
+
|
data_advisor/cli.py
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"""
|
|
2
|
+
cli.py - Command-line interface for DataAdvisor.
|
|
3
|
+
Allows instant dataset audits, script generation, and local code search directly from the terminal.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
import sys
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import List, Optional
|
|
12
|
+
|
|
13
|
+
from data_advisor.advisor import DataAdvisor
|
|
14
|
+
from data_advisor.search import search
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def parse_args(args: Optional[List[str]] = None) -> argparse.Namespace:
|
|
18
|
+
parser = argparse.ArgumentParser(
|
|
19
|
+
prog="data-advisor",
|
|
20
|
+
description="DataAdvisor: Automated dataset auditor, exam-ready code generator, and local code snippet search.",
|
|
21
|
+
)
|
|
22
|
+
parser.add_argument(
|
|
23
|
+
"dataset",
|
|
24
|
+
type=str,
|
|
25
|
+
nargs="?",
|
|
26
|
+
default=None,
|
|
27
|
+
help="Path to dataset file (CSV, TSV, Excel, Parquet). Optional if using --search.",
|
|
28
|
+
)
|
|
29
|
+
parser.add_argument(
|
|
30
|
+
"-s", "--search",
|
|
31
|
+
type=str,
|
|
32
|
+
default=None,
|
|
33
|
+
help="Search query to find code from local code files (e.g. 'wokwi', 'linear regression').",
|
|
34
|
+
)
|
|
35
|
+
parser.add_argument(
|
|
36
|
+
"-g", "--get",
|
|
37
|
+
type=str,
|
|
38
|
+
default=None,
|
|
39
|
+
help="Retrieve and display the COMPLETE raw file content by question or name (e.g. 'q1', 24, 'question_01.py').",
|
|
40
|
+
)
|
|
41
|
+
parser.add_argument(
|
|
42
|
+
"-r", "--run",
|
|
43
|
+
type=str,
|
|
44
|
+
default=None,
|
|
45
|
+
help="Execute a script directly from the code bank by question or name (e.g. 'q24', '14').",
|
|
46
|
+
)
|
|
47
|
+
parser.add_argument(
|
|
48
|
+
"-l", "--list",
|
|
49
|
+
action="store_true",
|
|
50
|
+
help="List all indexed code files in the code bank.",
|
|
51
|
+
)
|
|
52
|
+
parser.add_argument(
|
|
53
|
+
"-n", "--line-numbers",
|
|
54
|
+
action="store_true",
|
|
55
|
+
help="Show line numbers when viewing files with --get.",
|
|
56
|
+
)
|
|
57
|
+
parser.add_argument(
|
|
58
|
+
"--folder",
|
|
59
|
+
type=str,
|
|
60
|
+
default=None,
|
|
61
|
+
help="Custom folder to search code files within (default: ./code_bank or ./snippets).",
|
|
62
|
+
)
|
|
63
|
+
parser.add_argument(
|
|
64
|
+
"-t", "--target",
|
|
65
|
+
type=str,
|
|
66
|
+
default=None,
|
|
67
|
+
help="Target / response variable column name.",
|
|
68
|
+
)
|
|
69
|
+
parser.add_argument(
|
|
70
|
+
"--id-cols",
|
|
71
|
+
type=str,
|
|
72
|
+
nargs="*",
|
|
73
|
+
default=None,
|
|
74
|
+
help="Column names representing IDs/keys to exclude from modeling.",
|
|
75
|
+
)
|
|
76
|
+
parser.add_argument(
|
|
77
|
+
"-c", "--code",
|
|
78
|
+
action="store_true",
|
|
79
|
+
help="Print the complete executable Python code pipeline to the console.",
|
|
80
|
+
)
|
|
81
|
+
parser.add_argument(
|
|
82
|
+
"-o", "--output",
|
|
83
|
+
type=str,
|
|
84
|
+
default=None,
|
|
85
|
+
help="Output file path to save the generated Python script.",
|
|
86
|
+
)
|
|
87
|
+
parser.add_argument(
|
|
88
|
+
"--no-report",
|
|
89
|
+
action="store_true",
|
|
90
|
+
help="Do not print terminal diagnostic report.",
|
|
91
|
+
)
|
|
92
|
+
return parser.parse_args(args)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def main(args: Optional[List[str]] = None) -> int:
|
|
96
|
+
parsed = parse_args(args)
|
|
97
|
+
|
|
98
|
+
# 1. Handle code bank retrieval / execution modes
|
|
99
|
+
if parsed.get is not None:
|
|
100
|
+
from data_advisor.search import get_file
|
|
101
|
+
get_file(query=parsed.get, folder=parsed.folder, print_output=True, line_numbers=parsed.line_numbers)
|
|
102
|
+
return 0
|
|
103
|
+
|
|
104
|
+
if parsed.run is not None:
|
|
105
|
+
from data_advisor.search import run_file
|
|
106
|
+
return run_file(query=parsed.run, folder=parsed.folder)
|
|
107
|
+
|
|
108
|
+
if parsed.list:
|
|
109
|
+
from data_advisor.search import list_files
|
|
110
|
+
list_files(folder=parsed.folder, print_output=True)
|
|
111
|
+
return 0
|
|
112
|
+
|
|
113
|
+
if parsed.search is not None:
|
|
114
|
+
search(key=parsed.search, folder=parsed.folder, print_output=True)
|
|
115
|
+
return 0
|
|
116
|
+
|
|
117
|
+
# 2. Handle dataset auditing mode
|
|
118
|
+
if not parsed.dataset:
|
|
119
|
+
print("[!] Error: Please provide a dataset file path, or use '--get <q>', '--search <key>', or '--run <q>'.")
|
|
120
|
+
print(" Example: data-advisor --get q1 # Display complete Question 1 code")
|
|
121
|
+
print(" Example: data-advisor --run q24 # Run Question 24 directly")
|
|
122
|
+
print(" Example: data-advisor --search 'linear regression'")
|
|
123
|
+
print(" Example: data-advisor dataset.csv --target churn")
|
|
124
|
+
print(" Example: data-advisor dataset.csv --target churn")
|
|
125
|
+
print(" Example: data-advisor dataset.csv --code")
|
|
126
|
+
print(" Example: data-advisor --search 'linear regression'")
|
|
127
|
+
return 1
|
|
128
|
+
|
|
129
|
+
dataset_path = Path(parsed.dataset)
|
|
130
|
+
if not dataset_path.exists():
|
|
131
|
+
print(f"[!] Error: File not found: {dataset_path}", file=sys.stderr)
|
|
132
|
+
return 1
|
|
133
|
+
|
|
134
|
+
try:
|
|
135
|
+
from data_advisor.advisor import data_advisor
|
|
136
|
+
|
|
137
|
+
# When --code is specified, print_code is True; otherwise, if output is specified, save it.
|
|
138
|
+
data_advisor(
|
|
139
|
+
data=dataset_path,
|
|
140
|
+
target=parsed.target,
|
|
141
|
+
output_script=parsed.output,
|
|
142
|
+
id_cols=parsed.id_cols,
|
|
143
|
+
print_report=not parsed.no_report,
|
|
144
|
+
print_code=parsed.code,
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
if not parsed.output and not parsed.code:
|
|
148
|
+
print("[*] To view the complete executable Python code, run with `--code` / `-c`.")
|
|
149
|
+
print("[*] To export to a Python file, run with `--output run_analysis.py` / `-o run_analysis.py`.")
|
|
150
|
+
|
|
151
|
+
return 0
|
|
152
|
+
except Exception as exc:
|
|
153
|
+
print(f"[!] Execution failed: {exc}", file=sys.stderr)
|
|
154
|
+
import traceback
|
|
155
|
+
traceback.print_exc()
|
|
156
|
+
return 1
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
if __name__ == "__main__":
|
|
160
|
+
sys.exit(main())
|