data-advisor 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,236 @@
1
+ """
2
+ visualizer.py - Generates beautiful, publication-ready Matplotlib & Seaborn visualization code.
3
+
4
+ Specifically crafted to produce exam-winning EDA charts that immediately prove hypotheses
5
+ and illustrate data distributions.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from typing import Any, Dict, List, Optional
11
+
12
+
13
+ class VisualizationAdvisor:
14
+ """
15
+ Constructs executable Matplotlib and Seaborn code blocks tailored to
16
+ the dataset's detected variable types and target relationships.
17
+ """
18
+
19
+ def __init__(self, scan_results: Dict[str, Any]) -> None:
20
+ self.scan = scan_results
21
+ self.distributions = scan_results.get("distributions", {})
22
+ self.categoricals = scan_results.get("categoricals", {})
23
+ self.target = scan_results.get("target")
24
+ self.dtypes = scan_results.get("data_types", {})
25
+
26
+ # -------------------------------------------------------------------------
27
+ # 1. Univariate Code Generation
28
+ # -------------------------------------------------------------------------
29
+ def generate_univariate_code(self) -> str:
30
+ """Generate code for numeric distribution histograms and categorical countplots."""
31
+ excluded_cats = set(
32
+ list(self.dtypes.get("potential_dates", []))
33
+ + list(self.dtypes.get("potential_ids", []))
34
+ + list(self.dtypes.get("constant_columns", []))
35
+ )
36
+ num_cols = list(self.distributions.keys())[:6] # Focus on top 6 to prevent plot clutter
37
+ cat_cols = [c for c in self.categoricals.keys() if c not in excluded_cats][:6]
38
+
39
+ code_lines = [
40
+ "# =============================================================================",
41
+ "# SECTION 1: UNIVARIATE DISTRIBUTION PLOTS (Matplotlib & Seaborn)",
42
+ "# =============================================================================",
43
+ "import matplotlib.pyplot as plt",
44
+ "import seaborn as sns",
45
+ "",
46
+ "# Set aesthetic style",
47
+ "sns.set_theme(style='whitegrid', palette='muted')",
48
+ "plt.rcParams.update({'font.sans-serif': 'Arial', 'font.size': 11})",
49
+ "",
50
+ ]
51
+
52
+ if num_cols:
53
+ code_lines.extend([
54
+ f"# Numeric Distributions: Histograms + Boxplots for {len(num_cols)} key features",
55
+ f"numeric_features = {repr(num_cols)}",
56
+ "for col in numeric_features:",
57
+ " if col in df.columns:",
58
+ " fig, (ax_box, ax_hist) = plt.subplots(",
59
+ " nrows=2, sharex=True, gridspec_kw={'height_ratios': [0.2, 0.8]}, figsize=(8, 5)",
60
+ " )",
61
+ " # Outlier boxplot on top",
62
+ " sns.boxplot(x=df[col], ax=ax_box, color='#4C72B0', fliersize=4)",
63
+ " ax_box.set(xlabel='')",
64
+ " ax_box.set_title(f'Distribution & Outliers: {col}', fontsize=12, fontweight='bold')",
65
+ " # Density histogram with KDE on bottom",
66
+ " sns.histplot(df[col], kde=True, ax=ax_hist, color='#4C72B0', bins=30)",
67
+ " ax_hist.axvline(df[col].mean(), color='crimson', linestyle='--', linewidth=1.5, label='Mean')",
68
+ " ax_hist.axvline(df[col].median(), color='darkgreen', linestyle='-', linewidth=1.5, label='Median')",
69
+ " ax_hist.legend()",
70
+ " plt.tight_layout()",
71
+ " plt.show()",
72
+ " plt.close('all')",
73
+ "",
74
+ ])
75
+
76
+ if cat_cols:
77
+ code_lines.extend([
78
+ f"# Categorical Distributions: Frequency Countplots for {len(cat_cols)} categories",
79
+ f"categorical_features = {repr(cat_cols)}",
80
+ "for col in categorical_features:",
81
+ " if col in df.columns:",
82
+ " plt.figure(figsize=(8, 4))",
83
+ " order = df[col].value_counts().index[:10] # Show top 10 categories",
84
+ " ax = sns.countplot(data=df, x=col, order=order, color='#4C72B0')",
85
+ " plt.title(f'Frequency Distribution: {col}', fontsize=12, fontweight='bold')",
86
+ " plt.xticks(rotation=45, ha='right')",
87
+ " # Annotate bar percentages",
88
+ " total = len(df[col].dropna())",
89
+ " for p in ax.patches:",
90
+ " height = p.get_height()",
91
+ " if height > 0:",
92
+ " pct = f'{100 * height / total:.1f}%'",
93
+ " ax.annotate(pct, (p.get_x() + p.get_width() / 2., height),",
94
+ " ha='center', va='bottom', fontsize=9, xytext=(0, 3), textcoords='offset points')",
95
+ " plt.tight_layout()",
96
+ " plt.show()",
97
+ " plt.close('all')",
98
+ "",
99
+ ])
100
+
101
+ return "\n".join(code_lines)
102
+
103
+ # -------------------------------------------------------------------------
104
+ # 2. Correlation Matrix Heatmap Code
105
+ # -------------------------------------------------------------------------
106
+ def generate_correlation_code(self) -> str:
107
+ """Generate Seaborn correlation heatmap with upper-triangle masking."""
108
+ num_cols = list(self.distributions.keys())
109
+ if len(num_cols) < 2:
110
+ return "# Less than 2 numeric columns; correlation heatmap skipped.\n"
111
+
112
+ return "\n".join([
113
+ "# =============================================================================",
114
+ "# SECTION 2: CORRELATION MATRIX HEATMAP",
115
+ "# =============================================================================",
116
+ "import numpy as np",
117
+ "import matplotlib.pyplot as plt",
118
+ "import seaborn as sns",
119
+ "",
120
+ "plt.figure(figsize=(10, 8))",
121
+ "corr = df.select_dtypes(include=[np.number]).corr(method='pearson')",
122
+ "# Generate mask for the upper triangle",
123
+ "mask = np.triu(np.ones_like(corr, dtype=bool))",
124
+ "# Plot annotated diverging heatmap",
125
+ "sns.heatmap(",
126
+ " corr, mask=mask, annot=True, fmt='.2f', cmap='vlag',",
127
+ " vmin=-1.0, vmax=1.0, center=0, square=True, linewidths=0.5, cbar_kws={'shrink': 0.8}",
128
+ ")",
129
+ "plt.title('Feature Correlation Matrix (Pearson)', fontsize=14, fontweight='bold', pad=12)",
130
+ "plt.tight_layout()",
131
+ "plt.show()",
132
+ "",
133
+ ])
134
+
135
+ # -------------------------------------------------------------------------
136
+ # 3. Bivariate & Target-Specific Code
137
+ # -------------------------------------------------------------------------
138
+ def generate_target_plots_code(self) -> str:
139
+ """Generate targeted bivariate plots verifying relationships with the response variable."""
140
+ if not self.target:
141
+ return "# No target variable detected/specified for bivariate target visualizations.\n"
142
+
143
+ target_col = self.target.get("target_column")
144
+ prob_type = self.target.get("problem_type")
145
+ excluded_cats = set(
146
+ list(self.dtypes.get("potential_dates", []))
147
+ + list(self.dtypes.get("potential_ids", []))
148
+ + list(self.dtypes.get("constant_columns", []))
149
+ )
150
+ num_cols = [c for c in self.distributions.keys() if c != target_col][:4]
151
+ cat_cols = [c for c in self.categoricals.keys() if c != target_col and c not in excluded_cats][:3]
152
+
153
+ lines = [
154
+ "# =============================================================================",
155
+ f"# SECTION 3: BIVARIATE RELATIONSHIPS WITH TARGET: '{target_col}' ({prob_type.upper()})",
156
+ "# =============================================================================",
157
+ "import matplotlib.pyplot as plt",
158
+ "import seaborn as sns",
159
+ "",
160
+ ]
161
+
162
+ if prob_type == "regression":
163
+ # Target is continuous: Scatter plots with regression trends
164
+ lines.extend([
165
+ f"# Scatter Plots with Linear Regression Trendlines vs Target '{target_col}'",
166
+ f"top_numeric_features = {repr(num_cols)}",
167
+ "for col in top_numeric_features:",
168
+ " if col in df.columns and TARGET in df.columns:",
169
+ " plt.figure(figsize=(7, 4.5))",
170
+ f" sns.regplot(data=df, x=col, y='{target_col}', scatter_kws={{'alpha': 0.4}}, line_kws={{'color': 'red'}})",
171
+ f" plt.title(f'Relationship: {{col}} vs {target_col}', fontsize=12, fontweight='bold')",
172
+ " plt.tight_layout()",
173
+ " plt.show()",
174
+ " plt.close('all')",
175
+ "",
176
+ ])
177
+ if cat_cols:
178
+ lines.extend([
179
+ f"# Categorical Groups vs Continuous Target '{target_col}' (Boxplots)",
180
+ f"key_cat_features = {repr(cat_cols)}",
181
+ "for col in key_cat_features:",
182
+ " if col in df.columns and TARGET in df.columns:",
183
+ " plt.figure(figsize=(8, 4.5))",
184
+ f" sns.boxplot(data=df, x=col, y='{target_col}', palette='Set2')",
185
+ f" plt.title(f'{target_col} Distribution across {{col}} Levels', fontsize=12, fontweight='bold')",
186
+ " plt.xticks(rotation=30, ha='right')",
187
+ " plt.tight_layout()",
188
+ " plt.show()",
189
+ " plt.close('all')",
190
+ "",
191
+ ])
192
+ else:
193
+ # Target is categorical: Boxplots/Violins grouped by target hue, and grouped countplots
194
+ lines.extend([
195
+ f"# Feature Distributions Conditioned on Target Classes '{target_col}'",
196
+ f"top_numeric_features = {repr(num_cols)}",
197
+ "for col in top_numeric_features:",
198
+ f" if col in df.columns and '{target_col}' in df.columns:",
199
+ " plt.figure(figsize=(7, 4.5))",
200
+ f" sns.boxplot(data=df, x='{target_col}', y=col, color='#4C72B0')",
201
+ f" plt.title(f'Distribution of {{col}} by {target_col}', fontsize=12, fontweight='bold')",
202
+ " plt.tight_layout()",
203
+ " plt.show()",
204
+ " plt.close('all')",
205
+ "",
206
+ ])
207
+ if cat_cols:
208
+ lines.extend([
209
+ f"# Categorical Contingency: Stacked/Grouped Countplots vs Target '{target_col}'",
210
+ f"key_cat_features = {repr(cat_cols)}",
211
+ "for col in key_cat_features:",
212
+ f" if col in df.columns and '{target_col}' in df.columns:",
213
+ " plt.figure(figsize=(8, 4.5))",
214
+ f" sns.countplot(data=df, x=col, hue='{target_col}', palette='Set1')",
215
+ f" plt.title(f'Contingency Distribution: {{col}} by {target_col}', fontsize=12, fontweight='bold')",
216
+ " plt.xticks(rotation=30, ha='right')",
217
+ " plt.legend(title='Target Class')",
218
+ " plt.tight_layout()",
219
+ " plt.show()",
220
+ " plt.close('all')",
221
+ "",
222
+ ])
223
+
224
+ return "\n".join(lines)
225
+
226
+ # -------------------------------------------------------------------------
227
+ # 4. Master Plot Generator
228
+ # -------------------------------------------------------------------------
229
+ def generate_all_plots_code(self) -> str:
230
+ """Combine all visualization modules into a cohesive EDA script block."""
231
+ blocks = [
232
+ self.generate_univariate_code(),
233
+ self.generate_correlation_code(),
234
+ self.generate_target_plots_code(),
235
+ ]
236
+ return "\n\n".join(blocks)
@@ -0,0 +1,303 @@
1
+ Metadata-Version: 2.4
2
+ Name: data_advisor
3
+ Version: 0.1.0
4
+ Summary: A production-grade diagnostic and code generation advisor for timed data analysis exams and rapid exploratory data science.
5
+ Author: Principal Data Science Architect
6
+ License-Expression: MIT
7
+ Keywords: data-analysis,eda,data-science,hypothesis-testing,code-generation,machine-learning,automated-eda,pandas,scipy,scikit-learn
8
+ Classifier: Development Status :: 4 - Beta
9
+ Classifier: Intended Audience :: Developers
10
+ Classifier: Intended Audience :: Education
11
+ Classifier: Intended Audience :: Science/Research
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3.9
14
+ Classifier: Programming Language :: Python :: 3.10
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Programming Language :: Python :: 3.13
18
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
19
+ Classifier: Topic :: Software Development :: Code Generators
20
+ Requires-Python: >=3.9
21
+ Description-Content-Type: text/markdown
22
+ Requires-Dist: pandas>=2.0.0
23
+ Requires-Dist: numpy>=1.24.0
24
+ Requires-Dist: scipy>=1.10.0
25
+ Requires-Dist: scikit-learn>=1.2.0
26
+ Requires-Dist: matplotlib>=3.7.0
27
+ Requires-Dist: seaborn>=0.12.0
28
+ Provides-Extra: dev
29
+ Requires-Dist: pytest>=7.0.0; extra == "dev"
30
+ Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
31
+
32
+ # DataAdvisor (data_advisor)
33
+
34
+ > **Autonomous Data Science Diagnostic Engine & Exam Code Generator**
35
+ > Engineered for students, practitioners, and data scientists to audit unknown datasets, formulate statistical hypotheses, and generate copy-pasteable, production-ready Python pipelines during timed data analysis examinations.
36
+
37
+ ---
38
+
39
+ ## πŸš€ Key Highlights
40
+
41
+ 1. **Deterministic Auditing Engine**: Deeply profiles any `pd.DataFrame` (or CSV, TSV, Excel, Parquet file) in seconds.
42
+ 2. **Automated Hypothesis Testing**:
43
+ - Numeric vs. Numeric: **Pearson** & **Spearman** correlations with $p$-values.
44
+ - Categorical vs. Categorical: **Chi-Square Test of Independence** with CramΓ©r's V effect size.
45
+ - Numeric vs. Binary Target: **Welch's Two-Sample t-test** & **Mann-Whitney U test**.
46
+ - Numeric vs. Multi-class Target: **One-Way ANOVA** ($F$-test) & **Kruskal-Wallis $H$-test**.
47
+ 3. **Outlier & Skewness Diagnostics**: Combines **IQR fences (1.5Γ— and 3.0Γ—)** and **Z-score criteria ($|z| > 3$)**; automatically recommends `np.log1p`, Yeo-Johnson power transformations, or winsorization.
48
+ 4. **Target & Model Strategy Selector**: Automatically identifies whether the task is **Regression** or **Classification** (Binary or Multiclass), detects critical class imbalance, and selects baseline models (`LogisticRegression`, `Ridge`, `RandomForest`).
49
+ 5. **Copy-Pasteable Code Generator**: Emits clean, PEP 8 compliant, runnable Python scripts containing complete Pandas, Scipy, and Scikit-Learn code tailored to the audited dataset.
50
+
51
+ ---
52
+
53
+ ## πŸ“¦ Installation
54
+
55
+ To install `data_advisor` in editable mode directly from your local repository:
56
+
57
+ ```bash
58
+ # Navigate to the project root directory
59
+ cd "path/to/Custom Library"
60
+
61
+ # Install in editable mode with dependencies
62
+ pip install -e .
63
+ ```
64
+
65
+ ### Verified Dependencies
66
+ - Python `>= 3.9`
67
+ - `pandas >= 2.0.0`
68
+ - `numpy >= 1.24.0`
69
+ - `scipy >= 1.10.0`
70
+ - `scikit-learn >= 1.2.0`
71
+ - `matplotlib >= 3.7.0`
72
+ - `seaborn >= 0.12.0`
73
+
74
+ ---
75
+
76
+ ## ⚑ Quickstart: The 1-Line Exam Workflow
77
+
78
+ When handed an unfamiliar exam dataset (`exam_dataset.csv`), run the top-level `data_advisor` function in your Jupyter Notebook or Python terminal:
79
+
80
+ ```python
81
+ from data_advisor import data_advisor
82
+
83
+ # 1-Line Diagnostic & Code Generation:
84
+ # Outputs:
85
+ # 1. Full diagnostic report stating what all analysis to perform
86
+ # 2. Executable, copy-pasteable Python code to execute all steps
87
+ data_advisor("exam_dataset.csv")
88
+ ```
89
+
90
+ Or with an optional target column or output script:
91
+ ```python
92
+ data_advisor("exam_dataset.csv", target="churn", output_script="solution.py")
93
+ ```
94
+
95
+ You can also use the object-oriented API:
96
+ ```python
97
+ from data_advisor import DataAdvisor
98
+
99
+ advisor = DataAdvisor("exam_dataset.csv", target="churn", id_cols=["customer_id"])
100
+ advisor.report() # Diagnostic report
101
+ advisor.generate_script("solution.py") # Full Python script
102
+ clean_code = advisor.get_code_block("cleaning") # Specific stage code
103
+ ```
104
+
105
+ Or directly from your command-line terminal:
106
+ ```bash
107
+ # Diagnostic report + code prompt
108
+ data-advisor exam_dataset.csv
109
+
110
+ # Print complete executable Python code directly to the terminal
111
+ data-advisor exam_dataset.csv --code
112
+
113
+ # Save complete pipeline to a Python file
114
+ data-advisor exam_dataset.csv --output solution.py
115
+ ```
116
+
117
+ ---
118
+
119
+ ## πŸ” Instant Code Snippet Search: `search(key)`
120
+
121
+ `data_advisor` includes a search engine that searches **any keyword** across your own local code folder (`code_bank/`).
122
+
123
+ ### How It Works:
124
+ 1. **Your Folder**: Simply paste your custom Python scripts (`.py`), Jupyter Notebooks (`.ipynb`), Arduino/ESP32 sketches (`.ino`, `.cpp`), or text files into `code_bank/`.
125
+ 2. **Dynamic Scanning**: The search engine scans `code_bank/` in real time on every search callβ€”no indexing or restarts required. You can add, edit, or delete files anytime.
126
+ 3. **Any Keyword**: Search for single terms, phrases, or keyword lists.
127
+
128
+ ```python
129
+ from data_advisor import search
130
+
131
+ # 1. List all available snippets in your code_bank
132
+ search()
133
+
134
+ # 2. Search for any keyword (e.g., machine learning, IoT, algorithms)
135
+ search("linear regression")
136
+ search("dht22")
137
+ search("kmeans")
138
+ search("wokwi")
139
+
140
+ # 3. Search with multiple keywords or a list
141
+ search(["esp32", "temperature"])
142
+ search("decision tree, entropy")
143
+
144
+ # 4. Search within a custom directory
145
+ search("apriori", folder="my_other_codes")
146
+ ```
147
+
148
+ Or search directly via the CLI:
149
+ ```bash
150
+ # List all files currently in code_bank
151
+ data-advisor --search "list"
152
+
153
+ # Search for any keyword
154
+ data-advisor --search "linear regression"
155
+ data-advisor --search "wokwi"
156
+ ```
157
+
158
+ ---
159
+
160
+ ## πŸ“„ Complete File Viewing, Retrieval & Execution: `get_file()`, `read_file()`, `run_file()`
161
+
162
+ To retrieve or execute the **entire complete code file** from `code_bank/` (not just search snippets):
163
+
164
+ ### 1. View / Print Complete File
165
+ ```python
166
+ from data_advisor import get_file, view_file
167
+
168
+ # By question number:
169
+ get_file("q1") # Complete code for Question 1
170
+ get_file(24) # Complete code for Question 24
171
+ get_file("q32", line_numbers=True) # With line numbers
172
+
173
+ # By filename or keyword:
174
+ get_file("sketch.ino") # Arduino / ESP32 sketch
175
+ get_file("subscriber.py")
176
+ ```
177
+
178
+ ### 2. Retrieve Complete Raw Code String (No Printing)
179
+ ```python
180
+ from data_advisor import read_file
181
+
182
+ # Return complete raw code as a Python string
183
+ code_string = read_file("q24")
184
+ print(f"Loaded {len(code_string)} characters")
185
+ ```
186
+
187
+ ### 3. Run Any File Directly
188
+ ```python
189
+ from data_advisor import run_file
190
+
191
+ # Execute a Python file directly via subprocess
192
+ run_file("q24")
193
+ run_file("iss")
194
+ ```
195
+
196
+ ### 4. Direct CLI Commands
197
+ ```bash
198
+ # Print complete file to console
199
+ data-advisor --get q1
200
+ data-advisor -g q24 --line-numbers
201
+ data-advisor --get sketch.ino
202
+
203
+ # Execute script directly
204
+ data-advisor --run q24
205
+ data-advisor -r 14
206
+
207
+ # List all available code files
208
+ data-advisor --list
209
+ ```
210
+
211
+
212
+
213
+ ---
214
+
215
+ ## 🧠 Decision Rules Matrix
216
+
217
+ ### 1. Data Cleaning & Integrity Rules
218
+ | Condition | Severity | Recommended Action | Generated Code |
219
+ | :--- | :--- | :--- | :--- |
220
+ | **Duplicates Detected** (`> 0`) | `WARNING` / `CRITICAL` | Drop duplicate rows | `df = df.drop_duplicates().reset_index(drop=True)` |
221
+ | **High Missingness** (`>= 60%`) | `CRITICAL` | Drop column entirely | `df = df.drop(columns=[col])` |
222
+ | **Low Missingness** (`< 5%`) | `INFO` | Impute median (num) / mode (cat) | `df[col] = df[col].fillna(df[col].median())` |
223
+ | **Moderate Missingness** (`5% - 60%`) | `WARNING` | Skewed: Median; Symmetric: Mean | `SimpleImputer(strategy='median')` |
224
+ | **Constant Column** (variance = 0) | `CRITICAL` | Drop column | `df = df.drop(columns=[col])` |
225
+ | **Identifier Column** (IDs, hashes) | `WARNING` | Exclude from feature matrix $X$ | `X = df.drop(columns=[id_col])` |
226
+ | **Date String Column** | `RECOMMENDED` | Parse date & extract temporal features | `.dt.year`, `.dt.month`, `.dt.dayofweek` |
227
+
228
+ ### 2. Distribution & Outlier Rules
229
+ | Metric | Condition | Transformation Strategy | Rationale |
230
+ | :--- | :--- | :--- | :--- |
231
+ | **Skewness** | $> 1.2$ (Non-negative) | `np.log1p(df[col])` | Stabilizes variance, compresses long positive tail |
232
+ | **Skewness** | $> 1.2$ (Contains negatives) | `PowerTransformer(method='yeo-johnson')` | Normalizes non-Gaussian data with non-positive values |
233
+ | **Skewness** | $< -1.2$ (Left-skewed) | `PowerTransformer(method='yeo-johnson')` | Corrects severe left tail |
234
+ | **IQR Outliers** | $> 5\%$ of samples | IQR clipping / `RobustScaler()` | Caps extreme leverage points |
235
+
236
+ ### 3. Categorical Processing Rules
237
+ | Cardinality | Encoding Strategy | Implementation |
238
+ | :--- | :--- | :--- |
239
+ | **Binary** ($2$ levels) | Direct Binary Mapping ($0/1$) | `df[col].map({'No': 0, 'Yes': 1})` |
240
+ | **Low-to-Medium** ($3$ to $10$ levels) | One-Hot Encoding (`drop_first=True`) | `pd.get_dummies(..., drop_first=True, dtype=int)` |
241
+ | **High** ($> 10$ levels) | Frequency Encoding / Target Encoding | `df[col].map(df[col].value_counts(normalize=True))` |
242
+
243
+ ### 4. Hypothesis Testing Decision Engine
244
+ | Variable Types | Primary Test | Non-Parametric Alternative | Effect Size Metric |
245
+ | :--- | :--- | :--- | :--- |
246
+ | **2 Numeric Features** | Pearson Correlation ($r, p$) | Spearman Rank Correlation ($\rho, p$) | Correlation Coefficient ($r$) |
247
+ | **2 Categorical Features** | Chi-Square Test of Independence ($\chi^2$) | Fisher's Exact Test (for $2\times 2$) | CramΓ©r's $V$ |
248
+ | **1 Numeric + 1 Binary Cat** | Welch's Independent $t$-test | Mann-Whitney $U$ test | Cohen's $d$ / Rank-biserial |
249
+ | **1 Numeric + 1 Multi-class Cat** | One-Way ANOVA ($F, p$) | Kruskal-Wallis $H$-test | Eta-squared ($\eta^2$) |
250
+
251
+ ### 5. Target Variable & Model Baseline Matrix
252
+ | Target Type | Detected Problem | Imbalance Threshold | Recommended Baselines | Evaluation Metrics |
253
+ | :--- | :--- | :--- | :--- | :--- |
254
+ | **Continuous Numeric** | Regression | N/A | Linear Regression, Ridge ($L_2$), Random Forest Regressor | RMSE, MAE, $R^2$ Score |
255
+ | **Binary / Discrete** | Classification | Minority $< 20\%$ (Ratio $\ge 3:1$) | Logistic Regression (`class_weight='balanced'`), Random Forest Classifier | F1-Score, PR-AUC, ROC-AUC, Confusion Matrix |
256
+
257
+ ---
258
+
259
+ ## πŸ“‚ Project Architecture
260
+
261
+ ```
262
+ Custom Library/
263
+ β”œβ”€β”€ pyproject.toml # Modern setuptools packaging & dependency metadata
264
+ β”œβ”€β”€ README.md # Comprehensive library documentation & exam cheat sheet
265
+ β”œβ”€β”€ src/
266
+ β”‚ └── data_advisor/
267
+ β”‚ β”œβ”€β”€ __init__.py # Package exports (DataAdvisor, DataScanner, RuleEngine)
268
+ β”‚ β”œβ”€β”€ advisor.py # Primary facade class & CLI reporter
269
+ β”‚ β”œβ”€β”€ scanner.py # Core profiling engine & statistical test runner
270
+ β”‚ β”œβ”€β”€ rules.py # Heuristic recommendation engine & severity prioritizer
271
+ β”‚ β”œβ”€β”€ codegen.py # Sequential Python script & code block generator
272
+ β”‚ β”œβ”€β”€ visualizer.py # Publication-ready Matplotlib & Seaborn code generator
273
+ β”‚ └── cli.py # Command-line entry point (`data-advisor`)
274
+ β”œβ”€β”€ tests/
275
+ β”‚ └── test_advisor.py # Automated pytest suite covering regression & classification
276
+ └── examples/
277
+ β”œβ”€β”€ demo_exam_workflow.py # End-to-end exam simulation walkthrough
278
+ β”œβ”€β”€ student_exam_data.csv # Sample dataset with outliers, nulls, and duplicates
279
+ └── generated_analysis_pipeline.py # Fully generated, executable production script
280
+ ```
281
+
282
+ ---
283
+
284
+ ## πŸ§ͺ Running Tests
285
+
286
+ Run the test suite using `pytest`:
287
+
288
+ ```bash
289
+ python -m pytest -v
290
+ ```
291
+
292
+ All tests inspect and validate:
293
+ - Missing value audit & column dropping (>60%).
294
+ - Identifier and constant column detection.
295
+ - Skewness and IQR/Z-score outlier calculations.
296
+ - Parametric and non-parametric hypothesis testing.
297
+ - Target classification and regression auto-detection.
298
+ - Full end-to-end Python script code generation and execution.
299
+
300
+ ---
301
+
302
+ ## πŸ“œ License
303
+ MIT License. Built for rigorous data science education, competitive coding, and exam excellence.
@@ -0,0 +1,14 @@
1
+ data_advisor/__init__.py,sha256=4TULFgIzA0A-QUthWOvpgsOtMII6VRybzEpe8M6umtw,879
2
+ data_advisor/__main__.py,sha256=J4_2-KC-FORv8kBEJE0TPqbDlwY80l4Uto8BewcKdkQ,191
3
+ data_advisor/advisor.py,sha256=nv-UR0tDFEJ60FksqISm-3NkjZFScIRXQKwx2CEc3Bo,19063
4
+ data_advisor/cli.py,sha256=GKwtctPR-lM2zFna4gCQy3IDA9SlH503gIx5gIXyH_k,5370
5
+ data_advisor/codegen.py,sha256=zkIgVNSXusrmZ5Y8cjSS7_c2UNOWl8MDY23tc08f1NM,24989
6
+ data_advisor/rules.py,sha256=QuBDOvdQGhQ7gohFFsOIbyaIk95r5DaeIHIkr0J7HGk,19174
7
+ data_advisor/scanner.py,sha256=NQzwq4pT8dlrOHPUS_O4SaFe_66eELJP7huvtot_Ay0,30645
8
+ data_advisor/search.py,sha256=bSkCE6RUxvOxaLv6Vg2lS9xSfXjBV8xeFz4CS4VMHiU,24980
9
+ data_advisor/visualizer.py,sha256=VhlQZEePVZm1Lpn0Q7wS1LcZZtS6saW28_xDSjVQuB0,12188
10
+ data_advisor-0.1.0.dist-info/METADATA,sha256=8uNX0f6S1ypFgNaKhBATKtdgaqsch1q_1-D7GIoTB8g,12732
11
+ data_advisor-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
12
+ data_advisor-0.1.0.dist-info/entry_points.txt,sha256=VduGLJ3bCfhCe5hL6dKjRbr0V4VbMv059LZrdIFDaAI,55
13
+ data_advisor-0.1.0.dist-info/top_level.txt,sha256=CPvucnS0dOg3PgL7a7fl_JmuxdZrWRXQJqEbjJ_Z0hg,13
14
+ data_advisor-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ data-advisor = data_advisor.cli:main
@@ -0,0 +1 @@
1
+ data_advisor