csvnorm 0.3.12__tar.gz

csvnorm-0.3.12/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 aborruso@gmail.com
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

csvnorm-0.3.12/PKG-INFO

@@ -0,0 +1,279 @@
+Metadata-Version: 2.4
+Name: csvnorm
+Version: 0.3.12
+Summary: A command-line utility to validate and normalize CSV files
+Author-email: aborruso <aborruso@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2026 aborruso@gmail.com
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/aborruso/prepare_data
+Project-URL: Issues, https://github.com/aborruso/prepare_data/issues
+Keywords: csv,data,normalization,validation,etl
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Utilities
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: charset-normalizer>=3.0.0
+Requires-Dist: duckdb>=0.9.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: rich-argparse>=1.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Dynamic: license-file
+
+[![PyPI version](https://badge.fury.io/py/csvnorm.svg)](https://pypi.org/project/csvnorm/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/)
+[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/aborruso/csvnorm)
+
+# csvnorm
+
+A command-line utility to validate and normalize CSV files for initial exploration.
+
+## Installation
+
+Recommended (uv):
+
+```bash
+uv tool install csvnorm
+```
+
+Or with pip:
+
+```bash
+pip install csvnorm
+```
+
+## Purpose
+
+This tool prepares CSV files for **basic exploratory data analysis (EDA)**, not for complex transformations. It focuses on achieving a clean, standardized baseline format that allows you to quickly assess data quality and structure before designing more sophisticated ETL pipelines.
+
+**What it does:**
+- Validates CSV structure and reports errors
+- Normalizes encoding to UTF-8 when needed
+- Normalizes delimiters and field names
+- Creates a consistent starting point for data exploration
+
+**What it doesn't do:**
+- Complex data transformations or business logic
+- Type inference or data validation beyond structure
+- Heavy processing or aggregations
+
+## Features
+
+- **CSV Validation**: Checks for common CSV errors and inconsistencies using DuckDB
+- **Delimiter Normalization**: Converts all field separators to standard commas (`,`)
+- **Field Name Normalization**: Converts column headers to snake_case format
+- **Encoding Normalization**: Auto-detects encoding and converts to UTF-8 when needed (ASCII is already UTF-8 compatible)
+- **Processing Summary**: Displays comprehensive statistics (rows, columns, file sizes) and error details
+- **Error Reporting**: Exports detailed error file for invalid rows with summary panel
+- **Remote URL Support**: Process CSV files directly from HTTP/HTTPS URLs without downloading
+
+## Usage
+
+```bash
+csvnorm input.csv [options]
+```
+
+### Options
+
+| Option | Description |
+|--------|-------------|
+| `-f, --force` | Force overwrite of existing output files |
+| `-k, --keep-names` | Keep original column names (disable snake_case) |
+| `-d, --delimiter CHAR` | Set custom output delimiter (default: `,`) |
+| `-o, --output-file PATH` | Set output file path (absolute or relative) |
+| `-V, --verbose` | Enable verbose output for debugging |
+| `-v, --version` | Show version number |
+| `-h, --help` | Show help message |
+
+### Examples
+
+```bash
+# Basic usage (output: data.csv in current directory)
+csvnorm data.csv
+
+# Specify output file path
+csvnorm data.csv -o output/processed.csv
+
+# Use absolute path
+csvnorm data.csv -o /tmp/data_normalized.csv
+
+# Process remote CSV from URL
+csvnorm "https://raw.githubusercontent.com/aborruso/csvnorm/refs/heads/main/test/Trasporto%20Pubblico%20Locale%20Settore%20Pubblico%20Allargato%20-%20Indicatore%202000-2020%20Trasferimenti%20Correnti%20su%20Entrate%20Correnti.csv" -o output.csv
+
+# With semicolon delimiter
+csvnorm data.csv -d ';' -o data_semicolon.csv
+
+# Keep original headers
+csvnorm data.csv --keep-names -o output.csv
+
+# Force overwrite with verbose output
+csvnorm data.csv -f -V -o processed.csv
+
+# Custom output name and extension
+csvnorm data.csv -o results.txt
+```
+
+### Output
+
+Creates a normalized CSV file at the specified path with:
+- UTF-8 encoding
+- Consistent field delimiters
+- Normalized column names (unless `--keep-names` is specified)
+- Error report if any invalid rows are found (saved as `{output_name}_reject_errors.csv` in the same directory)
+- Temporary encoding conversion files stored in system temp directory with auto-cleanup
+
+Output file path behavior:
+- If `-o` is specified: uses the exact path provided (supports absolute and relative paths)
+- If `-o` is omitted: uses input filename in current working directory
+- Any file extension is allowed (not limited to `.csv`)
+
+For remote URLs:
+- You must specify `-o` to set the output filename
+- Encoding is handled automatically by DuckDB
+- HTTP timeout is set to 30 seconds
+- Only public URLs are supported (no authentication)
+
+The tool provides modern terminal output with:
+- Progress indicators for multi-step processing
+- Color-coded error messages with panels
+- Success summary table with statistics (rows, columns, file sizes)
+- Encoding conversion status (converted/no conversion/remote; ASCII is already UTF-8 compatible)
+- Error summary panel with reject count and error types when validation fails
+- ASCII art banner with `--version` and `-V` verbose mode
+
+**Success Example:**
+```
+ ✓ Success
+ Input:        test/utf8_basic.csv
+ Output:       output/utf8_basic.csv
+ Encoding:     ascii (ASCII is UTF-8 compatible; no conversion needed)
+ Rows:         2
+ Columns:      3
+ Input size:   42 B
+ Output size:  43 B
+ Headers:      normalized to snake_case
+```
+
+**Error Example:**
+```
+ ✓ Success
+ Input:        test/malformed_rows.csv
+ Output:       output/malformed_rows.csv
+ Encoding:     ascii (ASCII is UTF-8 compatible; no conversion needed)
+ Rows:         1
+ Columns:      4
+ Input size:   24 B
+ Output size:  40 B
+ Headers:      normalized to snake_case
+
+╭──────────────────────────── ! Validation Failed ─────────────────────────────╮
+│ Validation Errors:                                                           │
+│                                                                              │
+│ Rejected rows: 2                                                             │
+│                                                                              │
+│ Error types:                                                                 │
+│   • Expected Number of Columns: 3 Found: 2                                   │
+│   • Expected Number of Columns: 3 Found: 4                                   │
+│                                                                              │
+│ Details: output/malformed_rows_reject_errors.csv                             │
+╰──────────────────────────────────────────────────────────────────────────────╯
+```
+
+### Exit Codes
+
+| Code | Meaning |
+|------|---------|
+| 0 | Success |
+| 1 | Error (validation failed, file not found, etc.) |
+
+## Requirements
+
+- Python 3.9+
+- Dependencies (automatically installed):
+  - `charset-normalizer>=3.0.0` - Encoding detection
+  - `duckdb>=0.9.0` - CSV validation and normalization
+  - `rich>=13.0.0` - Modern terminal output formatting
+  - `rich-argparse>=1.0.0` - Enhanced CLI help formatting
+  - `pyfiglet>=0.8.post1,<1.0.0` - ASCII art banner
+
+Optional extras:
+- `[dev]` - Development dependencies (`pytest>=7.0.0`, `pytest-cov>=4.0.0`, `ruff>=0.1.0`)
+
+## Development
+
+### Setup
+
+```bash
+git clone https://github.com/aborruso/csvnorm
+cd csvnorm
+
+# Create and activate venv with uv (recommended)
+uv venv
+source .venv/bin/activate
+uv pip install -e ".[dev]"
+
+# Or with pip
+pip install -e ".[dev]"
+```
+
+### Testing
+
+```bash
+pytest tests/ -v
+```
+
+### Project Structure
+
+```
+csvnorm/
+├── src/csvnorm/
+│   ├── __init__.py      # Package version
+│   ├── __main__.py      # python -m support
+│   ├── cli.py           # CLI argument parsing
+│   ├── core.py          # Main processing pipeline
+│   ├── encoding.py      # Encoding detection/conversion
+│   ├── validation.py    # DuckDB validation
+│   └── utils.py         # Helper functions
+├── tests/               # Test suite
+├── test/                # CSV fixtures
+└── pyproject.toml       # Package configuration
+```
+
+## License
+
+MIT License (c) 2026 aborruso@gmail.com - See LICENSE file for details

csvnorm-0.3.12/README.md

@@ -0,0 +1,223 @@
+[![PyPI version](https://badge.fury.io/py/csvnorm.svg)](https://pypi.org/project/csvnorm/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/)
+[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/aborruso/csvnorm)
+
+# csvnorm
+
+A command-line utility to validate and normalize CSV files for initial exploration.
+
+## Installation
+
+Recommended (uv):
+
+```bash
+uv tool install csvnorm
+```
+
+Or with pip:
+
+```bash
+pip install csvnorm
+```
+
+## Purpose
+
+This tool prepares CSV files for **basic exploratory data analysis (EDA)**, not for complex transformations. It focuses on achieving a clean, standardized baseline format that allows you to quickly assess data quality and structure before designing more sophisticated ETL pipelines.
+
+**What it does:**
+- Validates CSV structure and reports errors
+- Normalizes encoding to UTF-8 when needed
+- Normalizes delimiters and field names
+- Creates a consistent starting point for data exploration
+
+**What it doesn't do:**
+- Complex data transformations or business logic
+- Type inference or data validation beyond structure
+- Heavy processing or aggregations
+
+## Features
+
+- **CSV Validation**: Checks for common CSV errors and inconsistencies using DuckDB
+- **Delimiter Normalization**: Converts all field separators to standard commas (`,`)
+- **Field Name Normalization**: Converts column headers to snake_case format
+- **Encoding Normalization**: Auto-detects encoding and converts to UTF-8 when needed (ASCII is already UTF-8 compatible)
+- **Processing Summary**: Displays comprehensive statistics (rows, columns, file sizes) and error details
+- **Error Reporting**: Exports detailed error file for invalid rows with summary panel
+- **Remote URL Support**: Process CSV files directly from HTTP/HTTPS URLs without downloading
+
+## Usage
+
+```bash
+csvnorm input.csv [options]
+```
+
+### Options
+
+| Option | Description |
+|--------|-------------|
+| `-f, --force` | Force overwrite of existing output files |
+| `-k, --keep-names` | Keep original column names (disable snake_case) |
+| `-d, --delimiter CHAR` | Set custom output delimiter (default: `,`) |
+| `-o, --output-file PATH` | Set output file path (absolute or relative) |
+| `-V, --verbose` | Enable verbose output for debugging |
+| `-v, --version` | Show version number |
+| `-h, --help` | Show help message |
+
+### Examples
+
+```bash
+# Basic usage (output: data.csv in current directory)
+csvnorm data.csv
+
+# Specify output file path
+csvnorm data.csv -o output/processed.csv
+
+# Use absolute path
+csvnorm data.csv -o /tmp/data_normalized.csv
+
+# Process remote CSV from URL
+csvnorm "https://raw.githubusercontent.com/aborruso/csvnorm/refs/heads/main/test/Trasporto%20Pubblico%20Locale%20Settore%20Pubblico%20Allargato%20-%20Indicatore%202000-2020%20Trasferimenti%20Correnti%20su%20Entrate%20Correnti.csv" -o output.csv
+
+# With semicolon delimiter
+csvnorm data.csv -d ';' -o data_semicolon.csv
+
+# Keep original headers
+csvnorm data.csv --keep-names -o output.csv
+
+# Force overwrite with verbose output
+csvnorm data.csv -f -V -o processed.csv
+
+# Custom output name and extension
+csvnorm data.csv -o results.txt
+```
+
+### Output
+
+Creates a normalized CSV file at the specified path with:
+- UTF-8 encoding
+- Consistent field delimiters
+- Normalized column names (unless `--keep-names` is specified)
+- Error report if any invalid rows are found (saved as `{output_name}_reject_errors.csv` in the same directory)
+- Temporary encoding conversion files stored in system temp directory with auto-cleanup
+
+Output file path behavior:
+- If `-o` is specified: uses the exact path provided (supports absolute and relative paths)
+- If `-o` is omitted: uses input filename in current working directory
+- Any file extension is allowed (not limited to `.csv`)
+
+For remote URLs:
+- You must specify `-o` to set the output filename
+- Encoding is handled automatically by DuckDB
+- HTTP timeout is set to 30 seconds
+- Only public URLs are supported (no authentication)
+
+The tool provides modern terminal output with:
+- Progress indicators for multi-step processing
+- Color-coded error messages with panels
+- Success summary table with statistics (rows, columns, file sizes)
+- Encoding conversion status (converted/no conversion/remote; ASCII is already UTF-8 compatible)
+- Error summary panel with reject count and error types when validation fails
+- ASCII art banner with `--version` and `-V` verbose mode
+
+**Success Example:**
+```
+ ✓ Success
+ Input:        test/utf8_basic.csv
+ Output:       output/utf8_basic.csv
+ Encoding:     ascii (ASCII is UTF-8 compatible; no conversion needed)
+ Rows:         2
+ Columns:      3
+ Input size:   42 B
+ Output size:  43 B
+ Headers:      normalized to snake_case
+```
+
+**Error Example:**
+```
+ ✓ Success
+ Input:        test/malformed_rows.csv
+ Output:       output/malformed_rows.csv
+ Encoding:     ascii (ASCII is UTF-8 compatible; no conversion needed)
+ Rows:         1
+ Columns:      4
+ Input size:   24 B
+ Output size:  40 B
+ Headers:      normalized to snake_case
+
+╭──────────────────────────── ! Validation Failed ─────────────────────────────╮
+│ Validation Errors:                                                           │
+│                                                                              │
+│ Rejected rows: 2                                                             │
+│                                                                              │
+│ Error types:                                                                 │
+│   • Expected Number of Columns: 3 Found: 2                                   │
+│   • Expected Number of Columns: 3 Found: 4                                   │
+│                                                                              │
+│ Details: output/malformed_rows_reject_errors.csv                             │
+╰──────────────────────────────────────────────────────────────────────────────╯
+```
+
+### Exit Codes
+
+| Code | Meaning |
+|------|---------|
+| 0 | Success |
+| 1 | Error (validation failed, file not found, etc.) |
+
+## Requirements
+
+- Python 3.9+
+- Dependencies (automatically installed):
+  - `charset-normalizer>=3.0.0` - Encoding detection
+  - `duckdb>=0.9.0` - CSV validation and normalization
+  - `rich>=13.0.0` - Modern terminal output formatting
+  - `rich-argparse>=1.0.0` - Enhanced CLI help formatting
+  - `pyfiglet>=0.8.post1,<1.0.0` - ASCII art banner
+
+Optional extras:
+- `[dev]` - Development dependencies (`pytest>=7.0.0`, `pytest-cov>=4.0.0`, `ruff>=0.1.0`)
+
+## Development
+
+### Setup
+
+```bash
+git clone https://github.com/aborruso/csvnorm
+cd csvnorm
+
+# Create and activate venv with uv (recommended)
+uv venv
+source .venv/bin/activate
+uv pip install -e ".[dev]"
+
+# Or with pip
+pip install -e ".[dev]"
+```
+
+### Testing
+
+```bash
+pytest tests/ -v
+```
+
+### Project Structure
+
+```
+csvnorm/
+├── src/csvnorm/
+│   ├── __init__.py      # Package version
+│   ├── __main__.py      # python -m support
+│   ├── cli.py           # CLI argument parsing
+│   ├── core.py          # Main processing pipeline
+│   ├── encoding.py      # Encoding detection/conversion
+│   ├── validation.py    # DuckDB validation
+│   └── utils.py         # Helper functions
+├── tests/               # Test suite
+├── test/                # CSV fixtures
+└── pyproject.toml       # Package configuration
+```
+
+## License
+
+MIT License (c) 2026 aborruso@gmail.com - See LICENSE file for details

csvnorm-0.3.12/pyproject.toml

@@ -0,0 +1,63 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "csvnorm"
+version = "0.3.12"
+description = "A command-line utility to validate and normalize CSV files"
+readme = "README.md"
+requires-python = ">=3.9"
+license = {file = "LICENSE"}
+authors = [
+    {name = "aborruso", email = "aborruso@gmail.com"}
+]
+keywords = ["csv", "data", "normalization", "validation", "etl"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Information Analysis",
+    "Topic :: Software Development :: Libraries",
+    "Topic :: Utilities",
+]
+
+dependencies = [
+    "charset-normalizer>=3.0.0",
+    "duckdb>=0.9.0",
+    "rich>=13.0.0",
+    "rich-argparse>=1.0.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+    "pytest-cov>=4.0.0",
+    "ruff>=0.1.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/aborruso/prepare_data"
+Issues = "https://github.com/aborruso/prepare_data/issues"
+
+[project.scripts]
+csvnorm = "csvnorm.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.ruff]
+line-length = 88
+target-version = "py38"
+
+[tool.pytest.ini_options]
+markers = [
+    "network: tests that require network access",
+]

csvnorm-0.3.12/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

csvnorm-0.3.12/src/csvnorm/__init__.py

@@ -0,0 +1,8 @@
+"""csvnorm - Validate and normalize CSV files."""
+
+__version__ = "0.3.12"
+__all__ = ["normalize_csv", "detect_encoding", "process_csv"]
+
+from csvnorm.core import process_csv
+from csvnorm.encoding import detect_encoding
+from csvnorm.validation import normalize_csv

csvnorm-0.3.12/src/csvnorm/__main__.py

@@ -0,0 +1,6 @@
+"""Entry point for python -m csvnorm."""
+
+from csvnorm.cli import main
+
+if __name__ == "__main__":
+    main()