refine-ai 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- refine_ai-0.3.1/PKG-INFO +283 -0
- refine_ai-0.3.1/README.md +256 -0
- refine_ai-0.3.1/pyproject.toml +55 -0
- refine_ai-0.3.1/refine/__init__.py +3 -0
- refine_ai-0.3.1/refine/cli.py +589 -0
- refine_ai-0.3.1/refine/graph.py +201 -0
- refine_ai-0.3.1/refine/logger.py +41 -0
- refine_ai-0.3.1/refine/profiler/__init__.py +1 -0
- refine_ai-0.3.1/refine/profiler/reporters.py +301 -0
- refine_ai-0.3.1/refine/profiler/stats.py +182 -0
- refine_ai-0.3.1/refine/schema_inference.py +383 -0
- refine_ai-0.3.1/refine/state.py +21 -0
- refine_ai-0.3.1/refine/tools/__init__.py +1 -0
- refine_ai-0.3.1/refine/tools/advisor.py +185 -0
- refine_ai-0.3.1/refine/tools/cleaner.py +44 -0
- refine_ai-0.3.1/refine/tools/reporter.py +82 -0
- refine_ai-0.3.1/refine/tools/synthesizer.py +241 -0
- refine_ai-0.3.1/refine/tools/transformer.py +196 -0
- refine_ai-0.3.1/refine_ai.egg-info/PKG-INFO +283 -0
- refine_ai-0.3.1/refine_ai.egg-info/SOURCES.txt +24 -0
- refine_ai-0.3.1/refine_ai.egg-info/dependency_links.txt +1 -0
- refine_ai-0.3.1/refine_ai.egg-info/entry_points.txt +2 -0
- refine_ai-0.3.1/refine_ai.egg-info/requires.txt +15 -0
- refine_ai-0.3.1/refine_ai.egg-info/top_level.txt +1 -0
- refine_ai-0.3.1/setup.cfg +4 -0
- refine_ai-0.3.1/tests/test_pipeline.py +515 -0
refine_ai-0.3.1/PKG-INFO
ADDED
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: refine-ai
|
|
3
|
+
Version: 0.3.1
|
|
4
|
+
Summary: Autonomous Data Pipeline & Synthesis Agent with Human-in-the-Loop Governance
|
|
5
|
+
Author: Aziz Toprak Yilmaz
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Toprak1yu/refine-ai
|
|
8
|
+
Project-URL: Repository, https://github.com/Toprak1yu/refine-ai
|
|
9
|
+
Project-URL: Issues, https://github.com/Toprak1yu/refine-ai/issues
|
|
10
|
+
Keywords: data-engineering,data-quality,langgraph,synthetic-data,cli,human-in-the-loop,polars
|
|
11
|
+
Requires-Python: >=3.11
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
Requires-Dist: langgraph>=0.2.0
|
|
14
|
+
Requires-Dist: langgraph-checkpoint-sqlite>=2.0.0
|
|
15
|
+
Requires-Dist: langchain-core>=0.3.0
|
|
16
|
+
Requires-Dist: polars>=1.0.0
|
|
17
|
+
Requires-Dist: rich>=13.0.0
|
|
18
|
+
Requires-Dist: typer>=0.12.0
|
|
19
|
+
Requires-Dist: scipy>=1.13.0
|
|
20
|
+
Requires-Dist: numpy>=1.26.0
|
|
21
|
+
Requires-Dist: faker>=25.0.0
|
|
22
|
+
Requires-Dist: pydantic>=2.7.0
|
|
23
|
+
Requires-Dist: langchain-ollama>=0.1.0
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
|
26
|
+
Requires-Dist: ruff>=0.4.0; extra == "dev"
|
|
27
|
+
|
|
28
|
+
# refine-ai `v0.3.0`
|
|
29
|
+
|
|
30
|
+
[](https://github.com/Toprak1yu/refine-ai)
|
|
31
|
+
[](https://www.python.org/)
|
|
32
|
+
[](https://opensource.org/licenses/MIT)
|
|
33
|
+
[](https://github.com/astral-sh/ruff)
|
|
34
|
+
|
|
35
|
+
> **Autonomous Data Pipeline & Synthetic Data Agent with Human-in-the-Loop (HITL) Governance.**
|
|
36
|
+
|
|
37
|
+
**refine-ai** is a stateful, autonomous data engineering agent built on top of [LangGraph](https://github.com/langchain-ai/langgraph), [Polars](https://pola.rs/), and [Rich](https://github.com/Textualize/rich). It deterministically profiles arbitrary datasets, enforces strict governance invariants defined in an operational constitution (`RULES.md`), safely pauses execution via native state-machine interrupts when critical anomalies occur, and resumes with human-approved remediation strategies.
|
|
38
|
+
|
|
39
|
+
---
|
|
40
|
+
|
|
41
|
+
## Key Features
|
|
42
|
+
|
|
43
|
+
- **Constitutional Governance (`RULES.md`):** Enforces strict statistical invariants (null ratio thresholds, Z-score bounds, domain range constraints, class imbalance limits).
|
|
44
|
+
- **Human-in-the-Loop (HITL) State Machine:** Employs LangGraph's native `interrupt()` mechanism and persistent SQLite checkpointers (`.checkpoints.db`) to pause graph execution and yield control to the human operator.
|
|
45
|
+
- **AI-Driven Streaming Output:** Experience live typewriter-style streaming (`--stream / --no-stream`) for operational logs, status steps, and expert guidance.
|
|
46
|
+
- **Interactive CLI Wizard (`refine`):** Run `refine` directly without arguments to launch an interactive wizard that prompts for dataset location and launches the pipeline seamlessly.
|
|
47
|
+
- **Planned Execution Manifest:** High-visibility audit plan rendered before modifying data. Review all planned pre-cleaning and remediation actions with one-click approval or individual column-level overrides.
|
|
48
|
+
- **Dry-Run Mode (`--dry-run`):** Inspect inferred schemas, anomaly profiles, AI recommendations, and planned execution manifests without modifying source data or writing any files to disk.
|
|
49
|
+
- **AI Dataset Schema Inference:** Automatically infers column roles (`id`, `target`, `feature`, `ignore`), semantic types (`numerical`, `categorical`, `binary`, `text`), domain bounds, and canonical aliases via local Ollama LLM with intelligent heuristic fallbacks.
|
|
50
|
+
- **Pure Column `DROP` Strategy:** Selecting `DROP` removes only the specified anomalous feature column while strictly preserving 100% of rows across all other features.
|
|
51
|
+
- **Distribution-Preserving Synthetic Synthesis:** Gaussian sampling for numerical outliers, frequency-weighted sampling for categoricals, and minority oversampling up to 35% balance for imbalanced target classes.
|
|
52
|
+
- **Executive Audit Reports:** Automatically produces clean CSVs alongside an executive Markdown audit report (`*_audit_report.md`) detailing row deltas, human decisions, execution timings, and seed telemetry.
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## Architecture & Workflow
|
|
57
|
+
|
|
58
|
+
```
|
|
59
|
+
[Raw Dataset (.csv)]
|
|
60
|
+
│
|
|
61
|
+
▼
|
|
62
|
+
[Node: Schema Inference] ───────► Detects ID, Target, and Feature roles (LLM / Heuristic)
|
|
63
|
+
│
|
|
64
|
+
▼
|
|
65
|
+
[Node: Profiler] ───────────────► Computes missingness, Z-scores, domain bounds & imbalance
|
|
66
|
+
│
|
|
67
|
+
▼
|
|
68
|
+
[Node: Deterministic Cleaner] ──► Strips whitespace, standardizes aliases & validates types
|
|
69
|
+
│
|
|
70
|
+
▼
|
|
71
|
+
[Node: Evaluator (Advisor)] ────► Synthesizes remediation advice against RULES.md
|
|
72
|
+
│
|
|
73
|
+
├── Invariants violated?
|
|
74
|
+
│ │
|
|
75
|
+
│ YES ──► [LangGraph Interrupt] (State checkpointed to SQLite)
|
|
76
|
+
│ │
|
|
77
|
+
│ ▼
|
|
78
|
+
│ [Execution Manifest Panel]
|
|
79
|
+
│ ├── Approve AI Manifest? [Y/n]
|
|
80
|
+
│ └── (If 'n') Interactive Column Prompts (DROP / IMPUTE / SYNTHESIS / MANUAL)
|
|
81
|
+
│ │
|
|
82
|
+
│ ▼
|
|
83
|
+
│ [Command(resume=...)] ◄─ Graph Resumed
|
|
84
|
+
│ ┌─────────────┘
|
|
85
|
+
▼ ▼
|
|
86
|
+
[Node: Transformer] ────────────► Imputes, removes column, or synthesizes minority records
|
|
87
|
+
│
|
|
88
|
+
▼
|
|
89
|
+
[Node: Exporter] ───────────────► Produces Clean CSV + Governance Audit Markdown Report
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
---
|
|
93
|
+
|
|
94
|
+
## Project Structure
|
|
95
|
+
|
|
96
|
+
```
|
|
97
|
+
refine-ai/
|
|
98
|
+
├── RULES.md # Operational governance constitution
|
|
99
|
+
├── pyproject.toml # Project packaging, dependencies & entrypoint
|
|
100
|
+
├── ruff.toml # Linter & code formatter configuration
|
|
101
|
+
├── data/
|
|
102
|
+
│ ├── generate_dirty_data.py # Synthetic dirty customer data generator
|
|
103
|
+
│ ├── raw/ # Untouched source datasets
|
|
104
|
+
│ └── processed/ # Cleaned datasets and markdown audit reports
|
|
105
|
+
├── refine/
|
|
106
|
+
│ ├── __init__.py # Package version (0.3.0)
|
|
107
|
+
│ ├── cli.py # Typer & Rich interactive CLI interface
|
|
108
|
+
│ ├── graph.py # LangGraph StateGraph & interrupt workflow
|
|
109
|
+
│ ├── logger.py # Structured console logging utilities
|
|
110
|
+
│ ├── schema_inference.py # AI & heuristic schema inference engine
|
|
111
|
+
│ ├── state.py # TypedDict pipeline state schema
|
|
112
|
+
│ ├── profiler/
|
|
113
|
+
│ │ ├── stats.py # Polars & SciPy statistical profiling engine
|
|
114
|
+
│ │ └── reporters.py # Rich terminal tables & execution manifests
|
|
115
|
+
│ └── tools/
|
|
116
|
+
│ ├── advisor.py # Local LLM / Ollama reasoning & rule-engine fallback
|
|
117
|
+
│ ├── cleaner.py # Deterministic invariant pre-cleaning
|
|
118
|
+
│ ├── reporter.py # Markdown governance audit report generator
|
|
119
|
+
│ ├── synthesizer.py # Distribution-preserving synthetic data generator
|
|
120
|
+
│ └── transformer.py # Remediation strategy execution engine
|
|
121
|
+
└── tests/
|
|
122
|
+
└── test_pipeline.py # End-to-end integration & unit test suite
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
---
|
|
126
|
+
|
|
127
|
+
## Quickstart
|
|
128
|
+
|
|
129
|
+
### 1. Installation
|
|
130
|
+
|
|
131
|
+
Clone the repository and install in editable mode:
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
git clone https://github.com/Toprak1yu/refine-ai.git
|
|
135
|
+
cd refine-ai
|
|
136
|
+
|
|
137
|
+
python3 -m venv .venv
|
|
138
|
+
source .venv/bin/activate # On Windows: .venv\Scripts\activate
|
|
139
|
+
pip install -e ".[dev]"
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
### 2. (Optional) Local AI Reasoning with Ollama
|
|
143
|
+
|
|
144
|
+
`refine-ai` supports local LLM reasoning via [Ollama](https://ollama.com/):
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
# Pull and start your preferred model (default: qwen2.5-coder:14b or llama3.2)
|
|
148
|
+
ollama run qwen2.5-coder:14b
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
> [!NOTE]
|
|
152
|
+
> If Ollama is not installed or offline, `refine-ai` automatically and seamlessly falls back to its deterministic rule-based heuristic engine. No crashes, no manual configuration required.
|
|
153
|
+
|
|
154
|
+
### 3. Generate Sample Dirty Dataset
|
|
155
|
+
|
|
156
|
+
Create a realistic test dataset containing missing values, negative ages, severe salary outliers, and class imbalance:
|
|
157
|
+
|
|
158
|
+
```bash
|
|
159
|
+
python data/generate_dirty_data.py
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
### 4. Run the Pipeline
|
|
163
|
+
|
|
164
|
+
You can run `refine` directly for an interactive prompt:
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
refine
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
Or execute directly with full options:
|
|
171
|
+
|
|
172
|
+
```bash
|
|
173
|
+
refine run --file data/raw/dirty_customers.csv --output data/processed/clean_customers.csv
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
---
|
|
177
|
+
|
|
178
|
+
## CLI Command Reference
|
|
179
|
+
|
|
180
|
+
### `refine` (Default Interactive Wizard)
|
|
181
|
+
Running `refine` without arguments prompts for the file path and launches the pipeline:
|
|
182
|
+
```bash
|
|
183
|
+
refine
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
### `refine run` (Execute Pipeline)
|
|
187
|
+
Executes the autonomous data engineering workflow:
|
|
188
|
+
```bash
|
|
189
|
+
refine run [OPTIONS]
|
|
190
|
+
|
|
191
|
+
Options:
|
|
192
|
+
-f, --file PATH Path to the raw CSV dataset [required]
|
|
193
|
+
-o, --output PATH Path for the clean CSV destination [default: data/processed/clean_<name>.csv]
|
|
194
|
+
-t, --thread-id TEXT Session identifier for SQLite checkpointer [default: session_001]
|
|
195
|
+
-s, --seed INTEGER Random seed for reproducibility [default: 42]
|
|
196
|
+
--stream / --no-stream Toggle typewriter streaming effect [default: --stream]
|
|
197
|
+
--dry-run Inspect manifest & recommendations without writing files
|
|
198
|
+
--help Show this message and exit
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
### `refine profile` (Standalone Profiler)
|
|
202
|
+
Inspects a dataset's missingness, outliers, and class distributions without executing cleaning:
|
|
203
|
+
```bash
|
|
204
|
+
refine profile data/raw/dirty_customers.csv
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
### `refine status` (Check State)
|
|
208
|
+
Checks whether an active SQLite checkpoint exists for the given thread:
|
|
209
|
+
```bash
|
|
210
|
+
refine status --thread-id session_001
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
### `refine reset` (Clear Checkpoints)
|
|
214
|
+
Removes SQLite checkpoint databases (`.checkpoints.db`) to start fresh:
|
|
215
|
+
```bash
|
|
216
|
+
refine reset
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
### `refine version` (Display Version)
|
|
220
|
+
Prints current package version:
|
|
221
|
+
```bash
|
|
222
|
+
refine version
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
---
|
|
226
|
+
|
|
227
|
+
## Example Human-in-the-Loop Interaction
|
|
228
|
+
|
|
229
|
+
When critical anomalies violate `RULES.md`, execution pauses and displays:
|
|
230
|
+
|
|
231
|
+
1. **Dataset Profile & Anomaly Audit** table.
|
|
232
|
+
2. **Senior Data Architect Reasoning** synthesized by local LLM or rule engine:
|
|
233
|
+
```text
|
|
234
|
+
[Local Rule-Engine Fallback]:
|
|
235
|
+
|
|
236
|
+
• 'age' (Invalid Bounds & Statistical Outliers):
|
|
237
|
+
➜ Recommended Decision: STATISTICAL_IMPUTE
|
|
238
|
+
➜ Decision Impact: Both out-of-bounds values and statistical outliers are resolved in a single step using median imputation; preserves data integrity and keeps row count constant.
|
|
239
|
+
|
|
240
|
+
• 'churn' (Class Imbalance - 8.0%):
|
|
241
|
+
➜ Recommended Decision: SYNTHETIC_SYNTHESIS
|
|
242
|
+
➜ Decision Impact: Synthesize records for minority class (1) to reach 35% balance; mitigates model bias and increases total row count.
|
|
243
|
+
|
|
244
|
+
• 'salary' (Statistical Outliers):
|
|
245
|
+
➜ Recommended Decision: SYNTHETIC_SYNTHESIS
|
|
246
|
+
➜ Decision Impact: Outliers are replaced with realistic Gaussian distribution values; preserves sample variance and bell-curve geometry.
|
|
247
|
+
```
|
|
248
|
+
3. **Planned Execution Manifest** table:
|
|
249
|
+
```
|
|
250
|
+
📋 PLANNED EXECUTION MANIFEST
|
|
251
|
+
• CLEANER country -> Canonicalized aliases to standard values
|
|
252
|
+
• TRANSFORMER age -> STATISTICAL_IMPUTE
|
|
253
|
+
• TRANSFORMER churn -> SYNTHETIC_SYNTHESIS (Oversampling minority class)
|
|
254
|
+
• TRANSFORMER salary -> SYNTHETIC_SYNTHESIS (Gaussian outlier replacement)
|
|
255
|
+
```
|
|
256
|
+
4. **Interactive Prompt**:
|
|
257
|
+
- Press **`Y`** to approve all recommendations instantly.
|
|
258
|
+
- Press **`n`** to enter interactive column-by-column mode (`DROP`, `STATISTICAL_IMPUTE`, `SYNTHETIC_SYNTHESIS`, or `MANUAL_INPUT`).
|
|
259
|
+
|
|
260
|
+
Upon completion, output files and executive audit documentation (`clean_customers_audit_report.md`) are saved.
|
|
261
|
+
|
|
262
|
+
---
|
|
263
|
+
|
|
264
|
+
## Development & Verification
|
|
265
|
+
|
|
266
|
+
Run the test suite:
|
|
267
|
+
|
|
268
|
+
```bash
|
|
269
|
+
pytest tests/ -v
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
Check formatting and lint rules:
|
|
273
|
+
|
|
274
|
+
```bash
|
|
275
|
+
ruff check .
|
|
276
|
+
ruff format --check .
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
---
|
|
280
|
+
|
|
281
|
+
## License
|
|
282
|
+
|
|
283
|
+
MIT License. See `LICENSE` for details.
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
# refine-ai `v0.3.0`
|
|
2
|
+
|
|
3
|
+
[](https://github.com/Toprak1yu/refine-ai)
|
|
4
|
+
[](https://www.python.org/)
|
|
5
|
+
[](https://opensource.org/licenses/MIT)
|
|
6
|
+
[](https://github.com/astral-sh/ruff)
|
|
7
|
+
|
|
8
|
+
> **Autonomous Data Pipeline & Synthetic Data Agent with Human-in-the-Loop (HITL) Governance.**
|
|
9
|
+
|
|
10
|
+
**refine-ai** is a stateful, autonomous data engineering agent built on top of [LangGraph](https://github.com/langchain-ai/langgraph), [Polars](https://pola.rs/), and [Rich](https://github.com/Textualize/rich). It deterministically profiles arbitrary datasets, enforces strict governance invariants defined in an operational constitution (`RULES.md`), safely pauses execution via native state-machine interrupts when critical anomalies occur, and resumes with human-approved remediation strategies.
|
|
11
|
+
|
|
12
|
+
---
|
|
13
|
+
|
|
14
|
+
## Key Features
|
|
15
|
+
|
|
16
|
+
- **Constitutional Governance (`RULES.md`):** Enforces strict statistical invariants (null ratio thresholds, Z-score bounds, domain range constraints, class imbalance limits).
|
|
17
|
+
- **Human-in-the-Loop (HITL) State Machine:** Employs LangGraph's native `interrupt()` mechanism and persistent SQLite checkpointers (`.checkpoints.db`) to pause graph execution and yield control to the human operator.
|
|
18
|
+
- **AI-Driven Streaming Output:** Experience live typewriter-style streaming (`--stream / --no-stream`) for operational logs, status steps, and expert guidance.
|
|
19
|
+
- **Interactive CLI Wizard (`refine`):** Run `refine` directly without arguments to launch an interactive wizard that prompts for dataset location and launches the pipeline seamlessly.
|
|
20
|
+
- **Planned Execution Manifest:** High-visibility audit plan rendered before modifying data. Review all planned pre-cleaning and remediation actions with one-click approval or individual column-level overrides.
|
|
21
|
+
- **Dry-Run Mode (`--dry-run`):** Inspect inferred schemas, anomaly profiles, AI recommendations, and planned execution manifests without modifying source data or writing any files to disk.
|
|
22
|
+
- **AI Dataset Schema Inference:** Automatically infers column roles (`id`, `target`, `feature`, `ignore`), semantic types (`numerical`, `categorical`, `binary`, `text`), domain bounds, and canonical aliases via local Ollama LLM with intelligent heuristic fallbacks.
|
|
23
|
+
- **Pure Column `DROP` Strategy:** Selecting `DROP` removes only the specified anomalous feature column while strictly preserving 100% of rows across all other features.
|
|
24
|
+
- **Distribution-Preserving Synthetic Synthesis:** Gaussian sampling for numerical outliers, frequency-weighted sampling for categoricals, and minority oversampling up to 35% balance for imbalanced target classes.
|
|
25
|
+
- **Executive Audit Reports:** Automatically produces clean CSVs alongside an executive Markdown audit report (`*_audit_report.md`) detailing row deltas, human decisions, execution timings, and seed telemetry.
|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
|
|
29
|
+
## Architecture & Workflow
|
|
30
|
+
|
|
31
|
+
```
|
|
32
|
+
[Raw Dataset (.csv)]
|
|
33
|
+
│
|
|
34
|
+
▼
|
|
35
|
+
[Node: Schema Inference] ───────► Detects ID, Target, and Feature roles (LLM / Heuristic)
|
|
36
|
+
│
|
|
37
|
+
▼
|
|
38
|
+
[Node: Profiler] ───────────────► Computes missingness, Z-scores, domain bounds & imbalance
|
|
39
|
+
│
|
|
40
|
+
▼
|
|
41
|
+
[Node: Deterministic Cleaner] ──► Strips whitespace, standardizes aliases & validates types
|
|
42
|
+
│
|
|
43
|
+
▼
|
|
44
|
+
[Node: Evaluator (Advisor)] ────► Synthesizes remediation advice against RULES.md
|
|
45
|
+
│
|
|
46
|
+
├── Invariants violated?
|
|
47
|
+
│ │
|
|
48
|
+
│ YES ──► [LangGraph Interrupt] (State checkpointed to SQLite)
|
|
49
|
+
│ │
|
|
50
|
+
│ ▼
|
|
51
|
+
│ [Execution Manifest Panel]
|
|
52
|
+
│ ├── Approve AI Manifest? [Y/n]
|
|
53
|
+
│ └── (If 'n') Interactive Column Prompts (DROP / IMPUTE / SYNTHESIS / MANUAL)
|
|
54
|
+
│ │
|
|
55
|
+
│ ▼
|
|
56
|
+
│ [Command(resume=...)] ◄─ Graph Resumed
|
|
57
|
+
│ ┌─────────────┘
|
|
58
|
+
▼ ▼
|
|
59
|
+
[Node: Transformer] ────────────► Imputes, removes column, or synthesizes minority records
|
|
60
|
+
│
|
|
61
|
+
▼
|
|
62
|
+
[Node: Exporter] ───────────────► Produces Clean CSV + Governance Audit Markdown Report
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
---
|
|
66
|
+
|
|
67
|
+
## Project Structure
|
|
68
|
+
|
|
69
|
+
```
|
|
70
|
+
refine-ai/
|
|
71
|
+
├── RULES.md # Operational governance constitution
|
|
72
|
+
├── pyproject.toml # Project packaging, dependencies & entrypoint
|
|
73
|
+
├── ruff.toml # Linter & code formatter configuration
|
|
74
|
+
├── data/
|
|
75
|
+
│ ├── generate_dirty_data.py # Synthetic dirty customer data generator
|
|
76
|
+
│ ├── raw/ # Untouched source datasets
|
|
77
|
+
│ └── processed/ # Cleaned datasets and markdown audit reports
|
|
78
|
+
├── refine/
|
|
79
|
+
│ ├── __init__.py # Package version (0.3.0)
|
|
80
|
+
│ ├── cli.py # Typer & Rich interactive CLI interface
|
|
81
|
+
│ ├── graph.py # LangGraph StateGraph & interrupt workflow
|
|
82
|
+
│ ├── logger.py # Structured console logging utilities
|
|
83
|
+
│ ├── schema_inference.py # AI & heuristic schema inference engine
|
|
84
|
+
│ ├── state.py # TypedDict pipeline state schema
|
|
85
|
+
│ ├── profiler/
|
|
86
|
+
│ │ ├── stats.py # Polars & SciPy statistical profiling engine
|
|
87
|
+
│ │ └── reporters.py # Rich terminal tables & execution manifests
|
|
88
|
+
│ └── tools/
|
|
89
|
+
│ ├── advisor.py # Local LLM / Ollama reasoning & rule-engine fallback
|
|
90
|
+
│ ├── cleaner.py # Deterministic invariant pre-cleaning
|
|
91
|
+
│ ├── reporter.py # Markdown governance audit report generator
|
|
92
|
+
│ ├── synthesizer.py # Distribution-preserving synthetic data generator
|
|
93
|
+
│ └── transformer.py # Remediation strategy execution engine
|
|
94
|
+
└── tests/
|
|
95
|
+
└── test_pipeline.py # End-to-end integration & unit test suite
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
---
|
|
99
|
+
|
|
100
|
+
## Quickstart
|
|
101
|
+
|
|
102
|
+
### 1. Installation
|
|
103
|
+
|
|
104
|
+
Clone the repository and install in editable mode:
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
git clone https://github.com/Toprak1yu/refine-ai.git
|
|
108
|
+
cd refine-ai
|
|
109
|
+
|
|
110
|
+
python3 -m venv .venv
|
|
111
|
+
source .venv/bin/activate # On Windows: .venv\Scripts\activate
|
|
112
|
+
pip install -e ".[dev]"
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
### 2. (Optional) Local AI Reasoning with Ollama
|
|
116
|
+
|
|
117
|
+
`refine-ai` supports local LLM reasoning via [Ollama](https://ollama.com/):
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
# Pull and start your preferred model (default: qwen2.5-coder:14b or llama3.2)
|
|
121
|
+
ollama run qwen2.5-coder:14b
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
> [!NOTE]
|
|
125
|
+
> If Ollama is not installed or offline, `refine-ai` automatically and seamlessly falls back to its deterministic rule-based heuristic engine. No crashes, no manual configuration required.
|
|
126
|
+
|
|
127
|
+
### 3. Generate Sample Dirty Dataset
|
|
128
|
+
|
|
129
|
+
Create a realistic test dataset containing missing values, negative ages, severe salary outliers, and class imbalance:
|
|
130
|
+
|
|
131
|
+
```bash
|
|
132
|
+
python data/generate_dirty_data.py
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
### 4. Run the Pipeline
|
|
136
|
+
|
|
137
|
+
You can run `refine` directly for an interactive prompt:
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
refine
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
Or execute directly with full options:
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
refine run --file data/raw/dirty_customers.csv --output data/processed/clean_customers.csv
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
---
|
|
150
|
+
|
|
151
|
+
## CLI Command Reference
|
|
152
|
+
|
|
153
|
+
### `refine` (Default Interactive Wizard)
|
|
154
|
+
Running `refine` without arguments prompts for the file path and launches the pipeline:
|
|
155
|
+
```bash
|
|
156
|
+
refine
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
### `refine run` (Execute Pipeline)
|
|
160
|
+
Executes the autonomous data engineering workflow:
|
|
161
|
+
```bash
|
|
162
|
+
refine run [OPTIONS]
|
|
163
|
+
|
|
164
|
+
Options:
|
|
165
|
+
-f, --file PATH Path to the raw CSV dataset [required]
|
|
166
|
+
-o, --output PATH Path for the clean CSV destination [default: data/processed/clean_<name>.csv]
|
|
167
|
+
-t, --thread-id TEXT Session identifier for SQLite checkpointer [default: session_001]
|
|
168
|
+
-s, --seed INTEGER Random seed for reproducibility [default: 42]
|
|
169
|
+
--stream / --no-stream Toggle typewriter streaming effect [default: --stream]
|
|
170
|
+
--dry-run Inspect manifest & recommendations without writing files
|
|
171
|
+
--help Show this message and exit
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
### `refine profile` (Standalone Profiler)
|
|
175
|
+
Inspects a dataset's missingness, outliers, and class distributions without executing cleaning:
|
|
176
|
+
```bash
|
|
177
|
+
refine profile data/raw/dirty_customers.csv
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
### `refine status` (Check State)
|
|
181
|
+
Checks whether an active SQLite checkpoint exists for the given thread:
|
|
182
|
+
```bash
|
|
183
|
+
refine status --thread-id session_001
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
### `refine reset` (Clear Checkpoints)
|
|
187
|
+
Removes SQLite checkpoint databases (`.checkpoints.db`) to start fresh:
|
|
188
|
+
```bash
|
|
189
|
+
refine reset
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
### `refine version` (Display Version)
|
|
193
|
+
Prints current package version:
|
|
194
|
+
```bash
|
|
195
|
+
refine version
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
---
|
|
199
|
+
|
|
200
|
+
## Example Human-in-the-Loop Interaction
|
|
201
|
+
|
|
202
|
+
When critical anomalies violate `RULES.md`, execution pauses and displays:
|
|
203
|
+
|
|
204
|
+
1. **Dataset Profile & Anomaly Audit** table.
|
|
205
|
+
2. **Senior Data Architect Reasoning** synthesized by local LLM or rule engine:
|
|
206
|
+
```text
|
|
207
|
+
[Local Rule-Engine Fallback]:
|
|
208
|
+
|
|
209
|
+
• 'age' (Invalid Bounds & Statistical Outliers):
|
|
210
|
+
➜ Recommended Decision: STATISTICAL_IMPUTE
|
|
211
|
+
➜ Decision Impact: Both out-of-bounds values and statistical outliers are resolved in a single step using median imputation; preserves data integrity and keeps row count constant.
|
|
212
|
+
|
|
213
|
+
• 'churn' (Class Imbalance - 8.0%):
|
|
214
|
+
➜ Recommended Decision: SYNTHETIC_SYNTHESIS
|
|
215
|
+
➜ Decision Impact: Synthesize records for minority class (1) to reach 35% balance; mitigates model bias and increases total row count.
|
|
216
|
+
|
|
217
|
+
• 'salary' (Statistical Outliers):
|
|
218
|
+
➜ Recommended Decision: SYNTHETIC_SYNTHESIS
|
|
219
|
+
➜ Decision Impact: Outliers are replaced with realistic Gaussian distribution values; preserves sample variance and bell-curve geometry.
|
|
220
|
+
```
|
|
221
|
+
3. **Planned Execution Manifest** table:
|
|
222
|
+
```
|
|
223
|
+
📋 PLANNED EXECUTION MANIFEST
|
|
224
|
+
• CLEANER country -> Canonicalized aliases to standard values
|
|
225
|
+
• TRANSFORMER age -> STATISTICAL_IMPUTE
|
|
226
|
+
• TRANSFORMER churn -> SYNTHETIC_SYNTHESIS (Oversampling minority class)
|
|
227
|
+
• TRANSFORMER salary -> SYNTHETIC_SYNTHESIS (Gaussian outlier replacement)
|
|
228
|
+
```
|
|
229
|
+
4. **Interactive Prompt**:
|
|
230
|
+
- Press **`Y`** to approve all recommendations instantly.
|
|
231
|
+
- Press **`n`** to enter interactive column-by-column mode (`DROP`, `STATISTICAL_IMPUTE`, `SYNTHETIC_SYNTHESIS`, or `MANUAL_INPUT`).
|
|
232
|
+
|
|
233
|
+
Upon completion, output files and executive audit documentation (`clean_customers_audit_report.md`) are saved.
|
|
234
|
+
|
|
235
|
+
---
|
|
236
|
+
|
|
237
|
+
## Development & Verification
|
|
238
|
+
|
|
239
|
+
Run the test suite:
|
|
240
|
+
|
|
241
|
+
```bash
|
|
242
|
+
pytest tests/ -v
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
Check formatting and lint rules:
|
|
246
|
+
|
|
247
|
+
```bash
|
|
248
|
+
ruff check .
|
|
249
|
+
ruff format --check .
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
---
|
|
253
|
+
|
|
254
|
+
## License
|
|
255
|
+
|
|
256
|
+
MIT License. See `LICENSE` for details.
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "refine-ai"
|
|
7
|
+
version = "0.3.1"
|
|
8
|
+
description = "Autonomous Data Pipeline & Synthesis Agent with Human-in-the-Loop Governance"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
authors = [
|
|
12
|
+
{ name = "Aziz Toprak Yilmaz" }
|
|
13
|
+
]
|
|
14
|
+
license = "MIT"
|
|
15
|
+
keywords = [
|
|
16
|
+
"data-engineering",
|
|
17
|
+
"data-quality",
|
|
18
|
+
"langgraph",
|
|
19
|
+
"synthetic-data",
|
|
20
|
+
"cli",
|
|
21
|
+
"human-in-the-loop",
|
|
22
|
+
"polars",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
dependencies = [
|
|
26
|
+
"langgraph>=0.2.0",
|
|
27
|
+
"langgraph-checkpoint-sqlite>=2.0.0",
|
|
28
|
+
"langchain-core>=0.3.0",
|
|
29
|
+
"polars>=1.0.0",
|
|
30
|
+
"rich>=13.0.0",
|
|
31
|
+
"typer>=0.12.0",
|
|
32
|
+
"scipy>=1.13.0",
|
|
33
|
+
"numpy>=1.26.0",
|
|
34
|
+
"faker>=25.0.0",
|
|
35
|
+
"pydantic>=2.7.0",
|
|
36
|
+
"langchain-ollama>=0.1.0",
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
[project.urls]
|
|
40
|
+
Homepage = "https://github.com/Toprak1yu/refine-ai"
|
|
41
|
+
Repository = "https://github.com/Toprak1yu/refine-ai"
|
|
42
|
+
Issues = "https://github.com/Toprak1yu/refine-ai/issues"
|
|
43
|
+
|
|
44
|
+
[project.optional-dependencies]
|
|
45
|
+
dev = [
|
|
46
|
+
"pytest>=8.0.0",
|
|
47
|
+
"ruff>=0.4.0",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
[project.scripts]
|
|
51
|
+
refine = "refine.cli:app"
|
|
52
|
+
|
|
53
|
+
[tool.setuptools.packages.find]
|
|
54
|
+
where = ["."]
|
|
55
|
+
include = ["refine*"]
|