datawash 0.2.0__py3-none-any.whl

datawash-0.2.0.dist-info/METADATA

@@ -0,0 +1,353 @@
+Metadata-Version: 2.4
+Name: datawash
+Version: 0.2.0
+Summary: Intelligent data cleaning and quality analysis
+Author: Sai Pranav Krovvidi
+License: MIT
+Project-URL: Homepage, https://github.com/Pranav1011/DataWash
+Project-URL: Repository, https://github.com/Pranav1011/DataWash
+Project-URL: Issues, https://github.com/Pranav1011/DataWash/issues
+Keywords: data-cleaning,data-quality,pandas,etl,data-analysis
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas>=1.5.0
+Requires-Dist: numpy>=1.21.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: typer>=0.9.0
+Provides-Extra: ml
+Requires-Dist: sentence-transformers>=2.2.0; extra == "ml"
+Requires-Dist: datasketch>=1.5.0; extra == "ml"
+Requires-Dist: scikit-learn>=1.0.0; extra == "ml"
+Requires-Dist: python-Levenshtein>=0.21.0; extra == "ml"
+Provides-Extra: formats
+Requires-Dist: pyarrow>=10.0.0; extra == "formats"
+Requires-Dist: openpyxl>=3.0.0; extra == "formats"
+Provides-Extra: all
+Requires-Dist: sentence-transformers>=2.2.0; extra == "all"
+Requires-Dist: datasketch>=1.5.0; extra == "all"
+Requires-Dist: scikit-learn>=1.0.0; extra == "all"
+Requires-Dist: python-Levenshtein>=0.21.0; extra == "all"
+Requires-Dist: pyarrow>=10.0.0; extra == "all"
+Requires-Dist: openpyxl>=3.0.0; extra == "all"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Dynamic: license-file
+
+# DataWash
+
+<p align="center">
+  <strong>Intelligent data cleaning and quality analysis for Python</strong>
+</p>
+
+<p align="center">
+  <a href="#installation">Installation</a> •
+  <a href="#quick-start">Quick Start</a> •
+  <a href="#features">Features</a> •
+  <a href="#documentation">Documentation</a> •
+  <a href="#examples">Examples</a>
+</p>
+
+<p align="center">
+  <img src="https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12-blue" alt="Python">
+  <img src="https://img.shields.io/badge/coverage-92%25-brightgreen" alt="Coverage">
+  <img src="https://img.shields.io/badge/tests-114%20passing-brightgreen" alt="Tests">
+  <img src="https://img.shields.io/badge/license-MIT-green" alt="License">
+</p>
+
+---
+
+DataWash analyzes your tabular data, detects quality issues, suggests prioritized fixes, and generates reproducible Python code — all in a few lines of code.
+
+```python
+from datawash import analyze
+
+report = analyze("messy_data.csv")
+print(f"Quality Score: {report.quality_score}/100")
+clean_df = report.apply_all()
+print(report.generate_code())
+```
+
+## Why DataWash?
+
+| Problem | DataWash Solution |
+|---------|-------------------|
+| Missing values silently break ML models | Automatic detection + smart filling strategies |
+| Inconsistent date formats cause parsing errors | Detects and standardizes to ISO format |
+| Duplicate rows inflate statistics | Identifies and removes exact duplicates |
+| Boolean values stored as "yes"/"no" strings | Converts to proper boolean type |
+| Manual data cleaning is tedious and error-prone | Generates reproducible Python code |
+
+## Installation
+
+```bash
+pip install datawash
+```
+
+**Optional extras:**
+
+```bash
+pip install datawash[formats]  # Parquet + Excel support
+pip install datawash[ml]       # ML-powered detection (coming soon)
+pip install datawash[all]      # All optional dependencies
+pip install datawash[dev]      # Development tools
+```
+
+## Quick Start
+
+### Python API
+
+```python
+from datawash import analyze
+
+# 1. Analyze your data (sampling enabled by default for large datasets)
+report = analyze("data.csv")  # or pass a DataFrame
+
+# 2. Check quality score
+print(f"Quality Score: {report.quality_score}/100")
+print(f"Issues Found: {len(report.issues)}")
+
+# 3. Review suggestions
+for s in report.suggestions:
+    print(f"[{s.id}] {s.action}")
+
+# 4. Apply all fixes
+clean_df = report.apply_all()
+
+# 5. Or apply selectively
+clean_df = report.apply([1, 3, 5])  # by suggestion ID
+
+# 6. Generate reproducible code
+print(report.generate_code())
+
+# Disable sampling for exact results on large datasets
+report = analyze("data.csv", sample=False)
+
+# Disable parallel processing
+report = analyze("data.csv", parallel=False)
+```
+
+### Command Line
+
+```bash
+# Analyze and see quality report
+datawash analyze data.csv
+
+# Get prioritized suggestions
+datawash suggest data.csv --use-case ml
+
+# Clean and export
+datawash clean data.csv -o clean.csv --apply-all
+
+# Generate Python code
+datawash codegen data.csv --apply-all
+```
+
+## Features
+
+### Data Quality Detection
+
+| Detector | What It Finds |
+|----------|---------------|
+| **Missing** | Null values, empty strings, whitespace-only values |
+| **Duplicates** | Exact duplicate rows |
+| **Formats** | Mixed case, inconsistent dates, whitespace padding |
+| **Outliers** | Statistical anomalies (IQR or Z-score) |
+| **Types** | Numbers/booleans stored as strings |
+| **Similarity** | Potentially duplicate columns |
+
+### Smart Transformations
+
+| Transformer | Operations |
+|-------------|------------|
+| **Missing** | Drop rows, fill with median/mode/value, clean empty strings |
+| **Duplicates** | Remove exact duplicates |
+| **Types** | Convert to numeric, boolean, datetime |
+| **Formats** | Standardize case, dates, strip whitespace |
+| **Columns** | Drop, rename, merge columns |
+| **Categories** | Normalize categorical values |
+
+### Intelligent Suggestion System
+
+- **Conflict Resolution**: Automatically prevents conflicting transformations
+- **Execution Ordering**: Applies fixes in optimal order (6 phases)
+- **Use-Case Aware**: Priorities adjust for ML, analytics, or export workflows
+- **Contextual Rationale**: Every suggestion explains why it's recommended
+
+### Code Generation
+
+```python
+# Generate a reusable cleaning function
+code = report.generate_code(style="function")
+
+# Or a standalone script
+code = report.generate_code(style="script")
+```
+
+## Performance
+
+DataWash v0.2.0 is optimized for large datasets:
+
+| Dataset | Time | Throughput |
+|---------|------|------------|
+| 1M rows x 10 cols | 0.72s | 1.4M rows/sec |
+| 100K rows x 50 cols | 2.13s | 47K rows/sec |
+| 10K rows x 100 cols | 4.35s | 2.3K rows/sec |
+| 1M rows x 50 cols | 3.24s | 309K rows/sec |
+| 50K rows x 250 cols | 9.99s | 5K rows/sec |
+
+**Optimizations include:**
+- Smart sampling for datasets >=50K rows (10-20x speedup)
+- Parallel column profiling and detection
+- 31% memory reduction via dtype optimization
+- O(n) similarity detection with MinHash + LSH
+
+## Examples
+
+We provide ready-to-run examples in the `examples/` directory:
+
+| Example | Description |
+|---------|-------------|
+| [`quickstart.py`](examples/quickstart.py) | Basic workflow: analyze → suggest → apply → codegen |
+| [`csv_cleaning.py`](examples/csv_cleaning.py) | Load CSV, clean, save with CLI equivalents |
+| [`ml_preprocessing.py`](examples/ml_preprocessing.py) | ML-optimized cleaning workflow |
+| [`jupyter_demo.ipynb`](examples/jupyter_demo.ipynb) | Interactive notebook with visualizations |
+
+**Sample datasets** in `examples/sample_data/`:
+- `customers_messy.csv` - Names, emails, phones with various issues
+- `orders_messy.csv` - Dates, amounts, categories with inconsistencies
+- `employees_messy.csv` - Mixed types, duplicates, outliers
+
+```bash
+# Run an example
+python examples/quickstart.py
+```
+
+## Documentation
+
+| Document | Description |
+|----------|-------------|
+| [Getting Started](docs/getting-started.md) | Installation and first steps |
+| [User Guide](docs/user-guide.md) | Complete feature walkthrough |
+| [API Reference](docs/api-reference.md) | Detailed API documentation |
+| [CLI Reference](docs/cli-reference.md) | Command-line interface guide |
+| [Configuration](docs/configuration.md) | Customization options |
+| [Contributing](docs/contributing.md) | How to contribute |
+
+## Use Cases
+
+Choose a use case to get optimized suggestions:
+
+```python
+report = analyze(df, use_case="ml")  # or "general", "analytics", "export"
+```
+
+| Use Case | Prioritizes |
+|----------|-------------|
+| `general` | Balanced approach for exploration |
+| `ml` | Duplicates, missing values, type conversions |
+| `analytics` | Consistency, date formats, outliers |
+| `export` | Format standardization, clean values |
+
+## Configuration
+
+```python
+report = analyze(
+    "data.csv",
+    use_case="ml",
+    config={
+        "detectors": {
+            "outlier_method": "zscore",  # or "iqr"
+            "outlier_threshold": 2.5,
+            "min_similarity": 0.8,
+        },
+        "suggestions": {
+            "max_suggestions": 20,
+        },
+    },
+)
+```
+
+## Project Status
+
+| Metric | Value |
+|--------|-------|
+| Source Code | ~2,900 lines |
+| Test Code | ~1,270 lines |
+| Tests | 114 passing |
+| Coverage | ~92% |
+| Python | 3.10, 3.11, 3.12 |
+| Platforms | Linux, macOS, Windows |
+
+### What's Working
+
+- ✅ Multi-format loading (CSV, JSON, Parquet, Excel)
+- ✅ Comprehensive profiling and statistics
+- ✅ 6 detectors for common data quality issues
+- ✅ 6 transformers with multiple operations each
+- ✅ Smart suggestion system with conflict resolution
+- ✅ Reproducible Python code generation
+- ✅ Rich CLI with colored output
+- ✅ Jupyter notebook support
+
+### What's Next
+
+- ML-powered semantic similarity detection
+- Fuzzy duplicate detection for near-duplicate rows
+- Advanced imputation (KNN, MICE)
+- Cloud storage connectors (S3, BigQuery)
+- PII detection for sensitive data
+- Schema validation for expected column types and constraints
+
+## Requirements
+
+- **Python** >= 3.10
+- **Core**: pandas, numpy, pydantic, rich, typer, scikit-learn
+- **Optional**: pyarrow (Parquet), openpyxl (Excel)
+
+## Development
+
+```bash
+# Clone and install
+git clone https://github.com/Pranav1011/DataWash.git
+cd DataWash
+pip install -e ".[dev,all]"
+
+# Run tests
+pytest
+
+# Format code
+black src tests
+ruff check src tests
+```
+
+## Contributing
+
+Contributions welcome! See [CONTRIBUTING.md](docs/contributing.md) for guidelines.
+
+**Areas where help is needed:**
+- ML module implementation (sentence-transformers)
+- Additional detectors (PII, schema validation)
+- Performance optimization
+- Documentation and examples
+- Cloud connectors
+
+## License
+
+MIT License - see [LICENSE](LICENSE) for details.
+
+## Acknowledgments
+
+Built with [pandas](https://pandas.pydata.org/), [pydantic](https://pydantic-docs.helpmanual.io/), [rich](https://rich.readthedocs.io/), [typer](https://typer.tiangolo.com/), and [scikit-learn](https://scikit-learn.org/).

datawash-0.2.0.dist-info/RECORD

@@ -0,0 +1,53 @@
+datawash/__init__.py,sha256=1Zj7T1Xs-t4uTI8c-i4xe9rF0WGwKxepuQ_9wdVbCpw,204
+datawash/adapters/__init__.py,sha256=LJnEVVlyq_LoHpW1hdxFIaE96oBwc8zme4YCqGusR0A,343
+datawash/adapters/base.py,sha256=9c0_qP7zHCf8t7J2cN2Evz9vPa4BbMIKrUQdxZAOGS8,1782
+datawash/adapters/csv_adapter.py,sha256=719WINqhKyHJyMHy9CBQdYNHPcoi7LuAFIeL8NTXYoc,564
+datawash/adapters/excel_adapter.py,sha256=bXAL9FLwo0twdCoLZZbRYc9MIH3yWyXv_qkWStX0g0k,1016
+datawash/adapters/json_adapter.py,sha256=93O3kmxlFCGzEyLkvkIEW1ozhd03Q-CFYESP3Ef98ps,492
+datawash/adapters/parquet_adapter.py,sha256=htK02H7lvim9REa4QDbr8H51DezCSreMggD8qnTRcmI,976
+datawash/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datawash/cli/formatters.py,sha256=vCRtBWYK9m3o7zR5z4K1KzuXYxSdHRn2sWTP7OAJfq0,3300
+datawash/cli/main.py,sha256=9e-SdbgZT17FUUljzwH60f_sVoiqxEkvXvl8E-Pxw64,5259
+datawash/codegen/__init__.py,sha256=_k0rnjiZqOkt825K3vJnjrNJuNwpXyNDpCSsvAe4McI,54
+datawash/codegen/generator.py,sha256=uU84nrwwyQoAn8UrG9p8c2q8AJlqGssBm6y30sDQOT8,2482
+datawash/core/__init__.py,sha256=TeULOhD1EXvygmAbuCIlRpyxYYMBM_-Z-RP0VMjBtqw,39
+datawash/core/cache.py,sha256=kFRl_giVvlyuHEbx9xfohyoO6ZFjQUUonyc3zClsJt8,2537
+datawash/core/config.py,sha256=2_KmLrZ97424VjBH4c8yRN1P5kf8JWG-iz9_ysXbGKk,1570
+datawash/core/dtypes.py,sha256=02vXzmYInDX7dS5AOEKoDBGt5FpUBRwPKH_ioA0dhp0,684
+datawash/core/exceptions.py,sha256=6L6G2S50D1xZwZuWTocEJcgH5Cukh8Oqb_I1HNI8GsY,424
+datawash/core/models.py,sha256=ZZCATjfphVnoW3a6GruOif3dfNAgKeva1_Q3W6tMCX4,1865
+datawash/core/report.py,sha256=IH4zfZ5zUlcRnkK5hgaxTZpp2F0sZWMeIExqo5yIjyY,15933
+datawash/core/sampling.py,sha256=8Bxlr9qhVlM17VOSUSsb9i4nWMKb72V7oOG8JxUtQe8,3063
+datawash/detectors/__init__.py,sha256=6c_MLlA0i-MK_8lhxMDpnH9CAMBh4K0DumcbbHHvVm8,372
+datawash/detectors/base.py,sha256=8Oii7RSCRb77de4CNi_XUH0NtA0fH3Hd-N8n7dhs4oA,620
+datawash/detectors/duplicate_detector.py,sha256=CxrGFmGXcx5_JICo2JRpTRI7IY1kdUd5IU0G7A8gMQY,1667
+datawash/detectors/format_detector.py,sha256=hUj8LGrSGDvII9BAZC3kRBYNGoFS6bYaRcO4thg3BiM,4710
+datawash/detectors/missing_detector.py,sha256=1ze-rzx07-G7mUGWPehc37qmfpv18ImRpd8ZJLeBryY,2818
+datawash/detectors/outlier_detector.py,sha256=eOqcHfY93BqNyQjOj1gotZ_z5p69t0e1YKUGiqcUD4I,3056
+datawash/detectors/registry.py,sha256=sbda5YRcu_a2Hoe3gtr104AVv5-6kRTVvPQnq3P9T0g,2050
+datawash/detectors/similarity_detector.py,sha256=ScgOfQyjGJcUyaNi__iHBQN0UEmLyaq93-4CZGGRsL0,10588
+datawash/detectors/type_detector.py,sha256=E_FOer8qgfLSaTKr8XsE7P5IEhmle_osH0bIJfYuDR8,3695
+datawash/profiler/__init__.py,sha256=NbAMeUn2EqIujVAgMg4Wwp5bnwU8iWCUR6ClJ2wzwQE,55
+datawash/profiler/engine.py,sha256=i6Sq5okeEiNmlKwf9SZSrpt5bg3zmOfhIsZVp7ZSd2s,2737
+datawash/profiler/parallel.py,sha256=TATKeCqBavfn4y5Uo8T-NeucqI1cOIhLEon5akzahTo,3772
+datawash/profiler/patterns.py,sha256=BSbLz4JMWxyS_h057s1eWQ9AglYB45HZy2M0X82n1Ec,2236
+datawash/profiler/statistics.py,sha256=MPGgl2MbcaVDwUIgdFzticiwQ_7TINYkVlNjRhayuuE,1399
+datawash/suggestors/__init__.py,sha256=47VPzsdGbzE6M3QQSjO9GmmAA0NgfMR5WeWpN3AhV40,65
+datawash/suggestors/base.py,sha256=VklzlzQCWSt-gDTZyuivyxAB7fk8nZIxkPdKnh218NQ,399
+datawash/suggestors/engine.py,sha256=iKvutrA4OcIDwauksqGyU8495NZP-PxQdk9kLVB134c,12121
+datawash/suggestors/prioritizer.py,sha256=jiOanLpieGLaPZHzkAJKnnpP6iLteDqLP22A2IIktkA,797
+datawash/transformers/__init__.py,sha256=6sn_xdbQqk5zFn1PpcUKndWq__eW7yW7VKYu7JrF0h0,287
+datawash/transformers/base.py,sha256=U2lLvrgaWu5DU5n1PkQjFIO_z_XqPff0ve0OoTmngQU,683
+datawash/transformers/categories.py,sha256=VjyXK1cKeiYfQfVb5NWC_L6uP0pwi4OeK2nO5waq24w,2035
+datawash/transformers/columns.py,sha256=qGB-yH9foVChZHSTQvbSgOATaDJ1MhJWelW2pV8nAls,2575
+datawash/transformers/duplicates.py,sha256=1tL51l4jZsEhvuLDPuu5-wLtQBCvUrdcULmCcS08NdE,1322
+datawash/transformers/formats.py,sha256=hJM2_8f9d_YGcreb_3lOdj9m29k8Wg5wGggKjcJJls8,3741
+datawash/transformers/missing.py,sha256=G4a1eZN8QrD-vMGD8_K-MIQ76Okw0znfrbmUXEvOLyI,8572
+datawash/transformers/registry.py,sha256=nKq4XU_TqWJg53wFuCVUc07XBVAQ_ehgrAb-K1BFjW4,768
+datawash/transformers/types.py,sha256=x4Im1b6_XbEXg685T4usZ_KQyDlM3YSAprs0VKEtM0A,3413
+datawash-0.2.0.dist-info/licenses/LICENSE,sha256=kNSkDIpMFky2dMMDOGB62O7DoC-o8Xno_7Nwp8ylcK4,1078
+datawash-0.2.0.dist-info/METADATA,sha256=U22gYry3P1D5X5ENAWatKzfVCBWYFfEJsQPB9cYxLn8,10866
+datawash-0.2.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+datawash-0.2.0.dist-info/entry_points.txt,sha256=zOhpea6Ad1tn5ofpYb6BEROG5bj6-LQh2eO6aVxP744,51
+datawash-0.2.0.dist-info/top_level.txt,sha256=xDL_KT2BiuAlMCI5iuXC6j-llUrx3XLYaiVgovjS-M4,9
+datawash-0.2.0.dist-info/RECORD,,

datawash-0.2.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.10.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

datawash-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+datawash = datawash.cli.main:app

datawash-0.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 DataWash Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

datawash-0.2.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+datawash