refine-ai 0.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,283 @@
1
+ Metadata-Version: 2.4
2
+ Name: refine-ai
3
+ Version: 0.3.1
4
+ Summary: Autonomous Data Pipeline & Synthesis Agent with Human-in-the-Loop Governance
5
+ Author: Aziz Toprak Yilmaz
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/Toprak1yu/refine-ai
8
+ Project-URL: Repository, https://github.com/Toprak1yu/refine-ai
9
+ Project-URL: Issues, https://github.com/Toprak1yu/refine-ai/issues
10
+ Keywords: data-engineering,data-quality,langgraph,synthetic-data,cli,human-in-the-loop,polars
11
+ Requires-Python: >=3.11
12
+ Description-Content-Type: text/markdown
13
+ Requires-Dist: langgraph>=0.2.0
14
+ Requires-Dist: langgraph-checkpoint-sqlite>=2.0.0
15
+ Requires-Dist: langchain-core>=0.3.0
16
+ Requires-Dist: polars>=1.0.0
17
+ Requires-Dist: rich>=13.0.0
18
+ Requires-Dist: typer>=0.12.0
19
+ Requires-Dist: scipy>=1.13.0
20
+ Requires-Dist: numpy>=1.26.0
21
+ Requires-Dist: faker>=25.0.0
22
+ Requires-Dist: pydantic>=2.7.0
23
+ Requires-Dist: langchain-ollama>=0.1.0
24
+ Provides-Extra: dev
25
+ Requires-Dist: pytest>=8.0.0; extra == "dev"
26
+ Requires-Dist: ruff>=0.4.0; extra == "dev"
27
+
28
+ # refine-ai `v0.3.0`
29
+
30
+ [![Version](https://img.shields.io/badge/version-0.3.0-blue.svg)](https://github.com/Toprak1yu/refine-ai)
31
+ [![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12-blue.svg)](https://www.python.org/)
32
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
33
+ [![Code style: ruff](https://img.shields.io/badge/code%20style-ruff-000000.svg)](https://github.com/astral-sh/ruff)
34
+
35
+ > **Autonomous Data Pipeline & Synthetic Data Agent with Human-in-the-Loop (HITL) Governance.**
36
+
37
+ **refine-ai** is a stateful, autonomous data engineering agent built on top of [LangGraph](https://github.com/langchain-ai/langgraph), [Polars](https://pola.rs/), and [Rich](https://github.com/Textualize/rich). It deterministically profiles arbitrary datasets, enforces strict governance invariants defined in an operational constitution (`RULES.md`), safely pauses execution via native state-machine interrupts when critical anomalies occur, and resumes with human-approved remediation strategies.
38
+
39
+ ---
40
+
41
+ ## Key Features
42
+
43
+ - **Constitutional Governance (`RULES.md`):** Enforces strict statistical invariants (null ratio thresholds, Z-score bounds, domain range constraints, class imbalance limits).
44
+ - **Human-in-the-Loop (HITL) State Machine:** Employs LangGraph's native `interrupt()` mechanism and persistent SQLite checkpointers (`.checkpoints.db`) to pause graph execution and yield control to the human operator.
45
+ - **AI-Driven Streaming Output:** Experience live typewriter-style streaming (`--stream / --no-stream`) for operational logs, status steps, and expert guidance.
46
+ - **Interactive CLI Wizard (`refine`):** Run `refine` directly without arguments to launch an interactive wizard that prompts for dataset location and launches the pipeline seamlessly.
47
+ - **Planned Execution Manifest:** High-visibility audit plan rendered before modifying data. Review all planned pre-cleaning and remediation actions with one-click approval or individual column-level overrides.
48
+ - **Dry-Run Mode (`--dry-run`):** Inspect inferred schemas, anomaly profiles, AI recommendations, and planned execution manifests without modifying source data or writing any files to disk.
49
+ - **AI Dataset Schema Inference:** Automatically infers column roles (`id`, `target`, `feature`, `ignore`), semantic types (`numerical`, `categorical`, `binary`, `text`), domain bounds, and canonical aliases via local Ollama LLM with intelligent heuristic fallbacks.
50
+ - **Pure Column `DROP` Strategy:** Selecting `DROP` removes only the specified anomalous feature column while strictly preserving 100% of rows across all other features.
51
+ - **Distribution-Preserving Synthetic Synthesis:** Gaussian sampling for numerical outliers, frequency-weighted sampling for categoricals, and minority oversampling up to 35% balance for imbalanced target classes.
52
+ - **Executive Audit Reports:** Automatically produces clean CSVs alongside an executive Markdown audit report (`*_audit_report.md`) detailing row deltas, human decisions, execution timings, and seed telemetry.
53
+
54
+ ---
55
+
56
+ ## Architecture & Workflow
57
+
58
+ ```
59
+ [Raw Dataset (.csv)]
60
+
61
+
62
+ [Node: Schema Inference] ───────► Detects ID, Target, and Feature roles (LLM / Heuristic)
63
+
64
+
65
+ [Node: Profiler] ───────────────► Computes missingness, Z-scores, domain bounds & imbalance
66
+
67
+
68
+ [Node: Deterministic Cleaner] ──► Strips whitespace, standardizes aliases & validates types
69
+
70
+
71
+ [Node: Evaluator (Advisor)] ────► Synthesizes remediation advice against RULES.md
72
+
73
+ ├── Invariants violated?
74
+ │ │
75
+ │ YES ──► [LangGraph Interrupt] (State checkpointed to SQLite)
76
+ │ │
77
+ │ ▼
78
+ │ [Execution Manifest Panel]
79
+ │ ├── Approve AI Manifest? [Y/n]
80
+ │ └── (If 'n') Interactive Column Prompts (DROP / IMPUTE / SYNTHESIS / MANUAL)
81
+ │ │
82
+ │ ▼
83
+ │ [Command(resume=...)] ◄─ Graph Resumed
84
+ │ ┌─────────────┘
85
+ ▼ ▼
86
+ [Node: Transformer] ────────────► Imputes, removes column, or synthesizes minority records
87
+
88
+
89
+ [Node: Exporter] ───────────────► Produces Clean CSV + Governance Audit Markdown Report
90
+ ```
91
+
92
+ ---
93
+
94
+ ## Project Structure
95
+
96
+ ```
97
+ refine-ai/
98
+ ├── RULES.md # Operational governance constitution
99
+ ├── pyproject.toml # Project packaging, dependencies & entrypoint
100
+ ├── ruff.toml # Linter & code formatter configuration
101
+ ├── data/
102
+ │ ├── generate_dirty_data.py # Synthetic dirty customer data generator
103
+ │ ├── raw/ # Untouched source datasets
104
+ │ └── processed/ # Cleaned datasets and markdown audit reports
105
+ ├── refine/
106
+ │ ├── __init__.py # Package version (0.3.0)
107
+ │ ├── cli.py # Typer & Rich interactive CLI interface
108
+ │ ├── graph.py # LangGraph StateGraph & interrupt workflow
109
+ │ ├── logger.py # Structured console logging utilities
110
+ │ ├── schema_inference.py # AI & heuristic schema inference engine
111
+ │ ├── state.py # TypedDict pipeline state schema
112
+ │ ├── profiler/
113
+ │ │ ├── stats.py # Polars & SciPy statistical profiling engine
114
+ │ │ └── reporters.py # Rich terminal tables & execution manifests
115
+ │ └── tools/
116
+ │ ├── advisor.py # Local LLM / Ollama reasoning & rule-engine fallback
117
+ │ ├── cleaner.py # Deterministic invariant pre-cleaning
118
+ │ ├── reporter.py # Markdown governance audit report generator
119
+ │ ├── synthesizer.py # Distribution-preserving synthetic data generator
120
+ │ └── transformer.py # Remediation strategy execution engine
121
+ └── tests/
122
+ └── test_pipeline.py # End-to-end integration & unit test suite
123
+ ```
124
+
125
+ ---
126
+
127
+ ## Quickstart
128
+
129
+ ### 1. Installation
130
+
131
+ Clone the repository and install in editable mode:
132
+
133
+ ```bash
134
+ git clone https://github.com/Toprak1yu/refine-ai.git
135
+ cd refine-ai
136
+
137
+ python3 -m venv .venv
138
+ source .venv/bin/activate # On Windows: .venv\Scripts\activate
139
+ pip install -e ".[dev]"
140
+ ```
141
+
142
+ ### 2. (Optional) Local AI Reasoning with Ollama
143
+
144
+ `refine-ai` supports local LLM reasoning via [Ollama](https://ollama.com/):
145
+
146
+ ```bash
147
+ # Pull and start your preferred model (default: qwen2.5-coder:14b or llama3.2)
148
+ ollama run qwen2.5-coder:14b
149
+ ```
150
+
151
+ > [!NOTE]
152
+ > If Ollama is not installed or offline, `refine-ai` automatically and seamlessly falls back to its deterministic rule-based heuristic engine. No crashes, no manual configuration required.
153
+
154
+ ### 3. Generate Sample Dirty Dataset
155
+
156
+ Create a realistic test dataset containing missing values, negative ages, severe salary outliers, and class imbalance:
157
+
158
+ ```bash
159
+ python data/generate_dirty_data.py
160
+ ```
161
+
162
+ ### 4. Run the Pipeline
163
+
164
+ You can run `refine` directly for an interactive prompt:
165
+
166
+ ```bash
167
+ refine
168
+ ```
169
+
170
+ Or execute directly with full options:
171
+
172
+ ```bash
173
+ refine run --file data/raw/dirty_customers.csv --output data/processed/clean_customers.csv
174
+ ```
175
+
176
+ ---
177
+
178
+ ## CLI Command Reference
179
+
180
+ ### `refine` (Default Interactive Wizard)
181
+ Running `refine` without arguments prompts for the file path and launches the pipeline:
182
+ ```bash
183
+ refine
184
+ ```
185
+
186
+ ### `refine run` (Execute Pipeline)
187
+ Executes the autonomous data engineering workflow:
188
+ ```bash
189
+ refine run [OPTIONS]
190
+
191
+ Options:
192
+ -f, --file PATH Path to the raw CSV dataset [required]
193
+ -o, --output PATH Path for the clean CSV destination [default: data/processed/clean_<name>.csv]
194
+ -t, --thread-id TEXT Session identifier for SQLite checkpointer [default: session_001]
195
+ -s, --seed INTEGER Random seed for reproducibility [default: 42]
196
+ --stream / --no-stream Toggle typewriter streaming effect [default: --stream]
197
+ --dry-run Inspect manifest & recommendations without writing files
198
+ --help Show this message and exit
199
+ ```
200
+
201
+ ### `refine profile` (Standalone Profiler)
202
+ Inspects a dataset's missingness, outliers, and class distributions without executing cleaning:
203
+ ```bash
204
+ refine profile data/raw/dirty_customers.csv
205
+ ```
206
+
207
+ ### `refine status` (Check State)
208
+ Checks whether an active SQLite checkpoint exists for the given thread:
209
+ ```bash
210
+ refine status --thread-id session_001
211
+ ```
212
+
213
+ ### `refine reset` (Clear Checkpoints)
214
+ Removes SQLite checkpoint databases (`.checkpoints.db`) to start fresh:
215
+ ```bash
216
+ refine reset
217
+ ```
218
+
219
+ ### `refine version` (Display Version)
220
+ Prints current package version:
221
+ ```bash
222
+ refine version
223
+ ```
224
+
225
+ ---
226
+
227
+ ## Example Human-in-the-Loop Interaction
228
+
229
+ When critical anomalies violate `RULES.md`, execution pauses and displays:
230
+
231
+ 1. **Dataset Profile & Anomaly Audit** table.
232
+ 2. **Senior Data Architect Reasoning** synthesized by local LLM or rule engine:
233
+ ```text
234
+ [Local Rule-Engine Fallback]:
235
+
236
+ • 'age' (Invalid Bounds & Statistical Outliers):
237
+ ➜ Recommended Decision: STATISTICAL_IMPUTE
238
+ ➜ Decision Impact: Both out-of-bounds values and statistical outliers are resolved in a single step using median imputation; preserves data integrity and keeps row count constant.
239
+
240
+ • 'churn' (Class Imbalance - 8.0%):
241
+ ➜ Recommended Decision: SYNTHETIC_SYNTHESIS
242
+ ➜ Decision Impact: Synthesize records for minority class (1) to reach 35% balance; mitigates model bias and increases total row count.
243
+
244
+ • 'salary' (Statistical Outliers):
245
+ ➜ Recommended Decision: SYNTHETIC_SYNTHESIS
246
+ ➜ Decision Impact: Outliers are replaced with realistic Gaussian distribution values; preserves sample variance and bell-curve geometry.
247
+ ```
248
+ 3. **Planned Execution Manifest** table:
249
+ ```
250
+ 📋 PLANNED EXECUTION MANIFEST
251
+ • CLEANER country -> Canonicalized aliases to standard values
252
+ • TRANSFORMER age -> STATISTICAL_IMPUTE
253
+ • TRANSFORMER churn -> SYNTHETIC_SYNTHESIS (Oversampling minority class)
254
+ • TRANSFORMER salary -> SYNTHETIC_SYNTHESIS (Gaussian outlier replacement)
255
+ ```
256
+ 4. **Interactive Prompt**:
257
+ - Press **`Y`** to approve all recommendations instantly.
258
+ - Press **`n`** to enter interactive column-by-column mode (`DROP`, `STATISTICAL_IMPUTE`, `SYNTHETIC_SYNTHESIS`, or `MANUAL_INPUT`).
259
+
260
+ Upon completion, output files and executive audit documentation (`clean_customers_audit_report.md`) are saved.
261
+
262
+ ---
263
+
264
+ ## Development & Verification
265
+
266
+ Run the test suite:
267
+
268
+ ```bash
269
+ pytest tests/ -v
270
+ ```
271
+
272
+ Check formatting and lint rules:
273
+
274
+ ```bash
275
+ ruff check .
276
+ ruff format --check .
277
+ ```
278
+
279
+ ---
280
+
281
+ ## License
282
+
283
+ MIT License. See `LICENSE` for details.
@@ -0,0 +1,256 @@
1
+ # refine-ai `v0.3.0`
2
+
3
+ [![Version](https://img.shields.io/badge/version-0.3.0-blue.svg)](https://github.com/Toprak1yu/refine-ai)
4
+ [![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12-blue.svg)](https://www.python.org/)
5
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
6
+ [![Code style: ruff](https://img.shields.io/badge/code%20style-ruff-000000.svg)](https://github.com/astral-sh/ruff)
7
+
8
+ > **Autonomous Data Pipeline & Synthetic Data Agent with Human-in-the-Loop (HITL) Governance.**
9
+
10
+ **refine-ai** is a stateful, autonomous data engineering agent built on top of [LangGraph](https://github.com/langchain-ai/langgraph), [Polars](https://pola.rs/), and [Rich](https://github.com/Textualize/rich). It deterministically profiles arbitrary datasets, enforces strict governance invariants defined in an operational constitution (`RULES.md`), safely pauses execution via native state-machine interrupts when critical anomalies occur, and resumes with human-approved remediation strategies.
11
+
12
+ ---
13
+
14
+ ## Key Features
15
+
16
+ - **Constitutional Governance (`RULES.md`):** Enforces strict statistical invariants (null ratio thresholds, Z-score bounds, domain range constraints, class imbalance limits).
17
+ - **Human-in-the-Loop (HITL) State Machine:** Employs LangGraph's native `interrupt()` mechanism and persistent SQLite checkpointers (`.checkpoints.db`) to pause graph execution and yield control to the human operator.
18
+ - **AI-Driven Streaming Output:** Experience live typewriter-style streaming (`--stream / --no-stream`) for operational logs, status steps, and expert guidance.
19
+ - **Interactive CLI Wizard (`refine`):** Run `refine` directly without arguments to launch an interactive wizard that prompts for dataset location and launches the pipeline seamlessly.
20
+ - **Planned Execution Manifest:** High-visibility audit plan rendered before modifying data. Review all planned pre-cleaning and remediation actions with one-click approval or individual column-level overrides.
21
+ - **Dry-Run Mode (`--dry-run`):** Inspect inferred schemas, anomaly profiles, AI recommendations, and planned execution manifests without modifying source data or writing any files to disk.
22
+ - **AI Dataset Schema Inference:** Automatically infers column roles (`id`, `target`, `feature`, `ignore`), semantic types (`numerical`, `categorical`, `binary`, `text`), domain bounds, and canonical aliases via local Ollama LLM with intelligent heuristic fallbacks.
23
+ - **Pure Column `DROP` Strategy:** Selecting `DROP` removes only the specified anomalous feature column while strictly preserving 100% of rows across all other features.
24
+ - **Distribution-Preserving Synthetic Synthesis:** Gaussian sampling for numerical outliers, frequency-weighted sampling for categoricals, and minority oversampling up to 35% balance for imbalanced target classes.
25
+ - **Executive Audit Reports:** Automatically produces clean CSVs alongside an executive Markdown audit report (`*_audit_report.md`) detailing row deltas, human decisions, execution timings, and seed telemetry.
26
+
27
+ ---
28
+
29
+ ## Architecture & Workflow
30
+
31
+ ```
32
+ [Raw Dataset (.csv)]
33
+
34
+
35
+ [Node: Schema Inference] ───────► Detects ID, Target, and Feature roles (LLM / Heuristic)
36
+
37
+
38
+ [Node: Profiler] ───────────────► Computes missingness, Z-scores, domain bounds & imbalance
39
+
40
+
41
+ [Node: Deterministic Cleaner] ──► Strips whitespace, standardizes aliases & validates types
42
+
43
+
44
+ [Node: Evaluator (Advisor)] ────► Synthesizes remediation advice against RULES.md
45
+
46
+ ├── Invariants violated?
47
+ │ │
48
+ │ YES ──► [LangGraph Interrupt] (State checkpointed to SQLite)
49
+ │ │
50
+ │ ▼
51
+ │ [Execution Manifest Panel]
52
+ │ ├── Approve AI Manifest? [Y/n]
53
+ │ └── (If 'n') Interactive Column Prompts (DROP / IMPUTE / SYNTHESIS / MANUAL)
54
+ │ │
55
+ │ ▼
56
+ │ [Command(resume=...)] ◄─ Graph Resumed
57
+ │ ┌─────────────┘
58
+ ▼ ▼
59
+ [Node: Transformer] ────────────► Imputes, removes column, or synthesizes minority records
60
+
61
+
62
+ [Node: Exporter] ───────────────► Produces Clean CSV + Governance Audit Markdown Report
63
+ ```
64
+
65
+ ---
66
+
67
+ ## Project Structure
68
+
69
+ ```
70
+ refine-ai/
71
+ ├── RULES.md # Operational governance constitution
72
+ ├── pyproject.toml # Project packaging, dependencies & entrypoint
73
+ ├── ruff.toml # Linter & code formatter configuration
74
+ ├── data/
75
+ │ ├── generate_dirty_data.py # Synthetic dirty customer data generator
76
+ │ ├── raw/ # Untouched source datasets
77
+ │ └── processed/ # Cleaned datasets and markdown audit reports
78
+ ├── refine/
79
+ │ ├── __init__.py # Package version (0.3.0)
80
+ │ ├── cli.py # Typer & Rich interactive CLI interface
81
+ │ ├── graph.py # LangGraph StateGraph & interrupt workflow
82
+ │ ├── logger.py # Structured console logging utilities
83
+ │ ├── schema_inference.py # AI & heuristic schema inference engine
84
+ │ ├── state.py # TypedDict pipeline state schema
85
+ │ ├── profiler/
86
+ │ │ ├── stats.py # Polars & SciPy statistical profiling engine
87
+ │ │ └── reporters.py # Rich terminal tables & execution manifests
88
+ │ └── tools/
89
+ │ ├── advisor.py # Local LLM / Ollama reasoning & rule-engine fallback
90
+ │ ├── cleaner.py # Deterministic invariant pre-cleaning
91
+ │ ├── reporter.py # Markdown governance audit report generator
92
+ │ ├── synthesizer.py # Distribution-preserving synthetic data generator
93
+ │ └── transformer.py # Remediation strategy execution engine
94
+ └── tests/
95
+ └── test_pipeline.py # End-to-end integration & unit test suite
96
+ ```
97
+
98
+ ---
99
+
100
+ ## Quickstart
101
+
102
+ ### 1. Installation
103
+
104
+ Clone the repository and install in editable mode:
105
+
106
+ ```bash
107
+ git clone https://github.com/Toprak1yu/refine-ai.git
108
+ cd refine-ai
109
+
110
+ python3 -m venv .venv
111
+ source .venv/bin/activate # On Windows: .venv\Scripts\activate
112
+ pip install -e ".[dev]"
113
+ ```
114
+
115
+ ### 2. (Optional) Local AI Reasoning with Ollama
116
+
117
+ `refine-ai` supports local LLM reasoning via [Ollama](https://ollama.com/):
118
+
119
+ ```bash
120
+ # Pull and start your preferred model (default: qwen2.5-coder:14b or llama3.2)
121
+ ollama run qwen2.5-coder:14b
122
+ ```
123
+
124
+ > [!NOTE]
125
+ > If Ollama is not installed or offline, `refine-ai` automatically and seamlessly falls back to its deterministic rule-based heuristic engine. No crashes, no manual configuration required.
126
+
127
+ ### 3. Generate Sample Dirty Dataset
128
+
129
+ Create a realistic test dataset containing missing values, negative ages, severe salary outliers, and class imbalance:
130
+
131
+ ```bash
132
+ python data/generate_dirty_data.py
133
+ ```
134
+
135
+ ### 4. Run the Pipeline
136
+
137
+ You can run `refine` directly for an interactive prompt:
138
+
139
+ ```bash
140
+ refine
141
+ ```
142
+
143
+ Or execute directly with full options:
144
+
145
+ ```bash
146
+ refine run --file data/raw/dirty_customers.csv --output data/processed/clean_customers.csv
147
+ ```
148
+
149
+ ---
150
+
151
+ ## CLI Command Reference
152
+
153
+ ### `refine` (Default Interactive Wizard)
154
+ Running `refine` without arguments prompts for the file path and launches the pipeline:
155
+ ```bash
156
+ refine
157
+ ```
158
+
159
+ ### `refine run` (Execute Pipeline)
160
+ Executes the autonomous data engineering workflow:
161
+ ```bash
162
+ refine run [OPTIONS]
163
+
164
+ Options:
165
+ -f, --file PATH Path to the raw CSV dataset [required]
166
+ -o, --output PATH Path for the clean CSV destination [default: data/processed/clean_<name>.csv]
167
+ -t, --thread-id TEXT Session identifier for SQLite checkpointer [default: session_001]
168
+ -s, --seed INTEGER Random seed for reproducibility [default: 42]
169
+ --stream / --no-stream Toggle typewriter streaming effect [default: --stream]
170
+ --dry-run Inspect manifest & recommendations without writing files
171
+ --help Show this message and exit
172
+ ```
173
+
174
+ ### `refine profile` (Standalone Profiler)
175
+ Inspects a dataset's missingness, outliers, and class distributions without executing cleaning:
176
+ ```bash
177
+ refine profile data/raw/dirty_customers.csv
178
+ ```
179
+
180
+ ### `refine status` (Check State)
181
+ Checks whether an active SQLite checkpoint exists for the given thread:
182
+ ```bash
183
+ refine status --thread-id session_001
184
+ ```
185
+
186
+ ### `refine reset` (Clear Checkpoints)
187
+ Removes SQLite checkpoint databases (`.checkpoints.db`) to start fresh:
188
+ ```bash
189
+ refine reset
190
+ ```
191
+
192
+ ### `refine version` (Display Version)
193
+ Prints current package version:
194
+ ```bash
195
+ refine version
196
+ ```
197
+
198
+ ---
199
+
200
+ ## Example Human-in-the-Loop Interaction
201
+
202
+ When critical anomalies violate `RULES.md`, execution pauses and displays:
203
+
204
+ 1. **Dataset Profile & Anomaly Audit** table.
205
+ 2. **Senior Data Architect Reasoning** synthesized by local LLM or rule engine:
206
+ ```text
207
+ [Local Rule-Engine Fallback]:
208
+
209
+ • 'age' (Invalid Bounds & Statistical Outliers):
210
+ ➜ Recommended Decision: STATISTICAL_IMPUTE
211
+ ➜ Decision Impact: Both out-of-bounds values and statistical outliers are resolved in a single step using median imputation; preserves data integrity and keeps row count constant.
212
+
213
+ • 'churn' (Class Imbalance - 8.0%):
214
+ ➜ Recommended Decision: SYNTHETIC_SYNTHESIS
215
+ ➜ Decision Impact: Synthesize records for minority class (1) to reach 35% balance; mitigates model bias and increases total row count.
216
+
217
+ • 'salary' (Statistical Outliers):
218
+ ➜ Recommended Decision: SYNTHETIC_SYNTHESIS
219
+ ➜ Decision Impact: Outliers are replaced with realistic Gaussian distribution values; preserves sample variance and bell-curve geometry.
220
+ ```
221
+ 3. **Planned Execution Manifest** table:
222
+ ```
223
+ 📋 PLANNED EXECUTION MANIFEST
224
+ • CLEANER country -> Canonicalized aliases to standard values
225
+ • TRANSFORMER age -> STATISTICAL_IMPUTE
226
+ • TRANSFORMER churn -> SYNTHETIC_SYNTHESIS (Oversampling minority class)
227
+ • TRANSFORMER salary -> SYNTHETIC_SYNTHESIS (Gaussian outlier replacement)
228
+ ```
229
+ 4. **Interactive Prompt**:
230
+ - Press **`Y`** to approve all recommendations instantly.
231
+ - Press **`n`** to enter interactive column-by-column mode (`DROP`, `STATISTICAL_IMPUTE`, `SYNTHETIC_SYNTHESIS`, or `MANUAL_INPUT`).
232
+
233
+ Upon completion, output files and executive audit documentation (`clean_customers_audit_report.md`) are saved.
234
+
235
+ ---
236
+
237
+ ## Development & Verification
238
+
239
+ Run the test suite:
240
+
241
+ ```bash
242
+ pytest tests/ -v
243
+ ```
244
+
245
+ Check formatting and lint rules:
246
+
247
+ ```bash
248
+ ruff check .
249
+ ruff format --check .
250
+ ```
251
+
252
+ ---
253
+
254
+ ## License
255
+
256
+ MIT License. See `LICENSE` for details.
@@ -0,0 +1,55 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61.0"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "refine-ai"
7
+ version = "0.3.1"
8
+ description = "Autonomous Data Pipeline & Synthesis Agent with Human-in-the-Loop Governance"
9
+ readme = "README.md"
10
+ requires-python = ">=3.11"
11
+ authors = [
12
+ { name = "Aziz Toprak Yilmaz" }
13
+ ]
14
+ license = "MIT"
15
+ keywords = [
16
+ "data-engineering",
17
+ "data-quality",
18
+ "langgraph",
19
+ "synthetic-data",
20
+ "cli",
21
+ "human-in-the-loop",
22
+ "polars",
23
+ ]
24
+
25
+ dependencies = [
26
+ "langgraph>=0.2.0",
27
+ "langgraph-checkpoint-sqlite>=2.0.0",
28
+ "langchain-core>=0.3.0",
29
+ "polars>=1.0.0",
30
+ "rich>=13.0.0",
31
+ "typer>=0.12.0",
32
+ "scipy>=1.13.0",
33
+ "numpy>=1.26.0",
34
+ "faker>=25.0.0",
35
+ "pydantic>=2.7.0",
36
+ "langchain-ollama>=0.1.0",
37
+ ]
38
+
39
+ [project.urls]
40
+ Homepage = "https://github.com/Toprak1yu/refine-ai"
41
+ Repository = "https://github.com/Toprak1yu/refine-ai"
42
+ Issues = "https://github.com/Toprak1yu/refine-ai/issues"
43
+
44
+ [project.optional-dependencies]
45
+ dev = [
46
+ "pytest>=8.0.0",
47
+ "ruff>=0.4.0",
48
+ ]
49
+
50
+ [project.scripts]
51
+ refine = "refine.cli:app"
52
+
53
+ [tool.setuptools.packages.find]
54
+ where = ["."]
55
+ include = ["refine*"]
@@ -0,0 +1,3 @@
1
+ """Autonomous Data Pipeline & Synthesis Agent with Human-in-the-Loop Governance."""
2
+
3
+ __version__ = "0.3.1"