csvmedic 0.1.0__tar.gz

csvmedic-0.1.0/.gitignore

@@ -0,0 +1,52 @@
+# Byte-compiled
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+env/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Testing
+.coverage
+.pytest_cache/
+htmlcov/
+.tox/
+nox/
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# uv
+.uv/
+
+# OS
+.DS_Store
+Thumbs.db

csvmedic-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 csvmedic contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

csvmedic-0.1.0/PKG-INFO

@@ -0,0 +1,146 @@
+Metadata-Version: 2.4
+Name: csvmedic
+Version: 0.1.0
+Summary: Automatic locale-aware CSV and Excel reader with encoding, delimiter, date format, and number locale detection.
+Project-URL: Homepage, https://github.com/csvmedic/csvmedic
+Project-URL: Documentation, https://csvmedic.readthedocs.io
+Project-URL: Repository, https://github.com/csvmedic/csvmedic
+Author: csvmedic contributors
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Requires-Python: >=3.9
+Requires-Dist: charset-normalizer>=3.0.0
+Requires-Dist: pandas>=1.5.0
+Provides-Extra: all
+Requires-Dist: clevercsv>=0.8.0; extra == 'all'
+Requires-Dist: openpyxl>=3.1.0; extra == 'all'
+Provides-Extra: dev
+Requires-Dist: mkdocs-material>=9.0; extra == 'dev'
+Requires-Dist: mkdocstrings[python]>=0.24; extra == 'dev'
+Requires-Dist: mypy>=1.8; extra == 'dev'
+Requires-Dist: pandas-stubs>=2.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.0; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: ruff>=0.4.0; extra == 'dev'
+Provides-Extra: excel
+Requires-Dist: openpyxl>=3.1.0; extra == 'excel'
+Provides-Extra: fast
+Requires-Dist: clevercsv>=0.8.0; extra == 'fast'
+Description-Content-Type: text/markdown
+
+# csvmedic
+
+Automatic locale-aware CSV and Excel reader. One line to clean messy data:
+
+```python
+import csvmedic
+
+df = csvmedic.read("messy_file.csv")
+print(df.diagnosis)  # See what was detected and converted
+```
+
+## What it does
+
+| Detects | Examples |
+|--------|----------|
+| **Encoding** | UTF-8, Windows-1252, ISO-8859-1, Shift-JIS, BOM |
+| **Delimiter** | Comma, semicolon, tab, pipe |
+| **Dates** | DD-MM vs MM-DD resolved statistically; ISO, European, US formats |
+| **Numbers** | European (1.234,56) vs US (1,234.56); locale hint |
+| **Booleans** | Yes/No, Ja/Nein, Oui/Non, Sí/No, and more |
+| **Strings** | Preserves leading zeros (IDs like 00742) |
+
+Every transformation is recorded in the `.diagnosis` attribute so you can audit what was changed.
+
+## Installation
+
+```bash
+pip install csvmedic
+```
+
+Optional extras:
+
+- `pip install csvmedic[fast]` — better dialect detection (clevercsv)
+- `pip install csvmedic[excel]` — .xlsx support (openpyxl)
+- `pip install csvmedic[all]` — both
+
+## Configuration
+
+Override auto-detection when you know better:
+
+```python
+df = csvmedic.read(
+    "file.csv",
+    encoding="utf-8",
+    delimiter=";",
+    dayfirst=True,              # Force DD-MM dates
+    preserve_strings=["ID"],    # Never convert these columns
+    sample_rows=2000,           # Rows to use for detection
+    confidence_threshold=0.75,  # Min confidence to convert (0–1)
+)
+```
+
+## Analyze without converting
+
+```python
+profile = csvmedic.read_raw("file.csv")
+print(profile.summary())
+print(profile.columns["Date"].details)
+```
+
+## Schema pinning (recurring files)
+
+Save the detected schema after the first read and reuse it so the next read skips detection:
+
+```python
+df = csvmedic.read("monthly_export.csv")
+csvmedic.save_schema(df.attrs["diagnosis"].file_profile, "monthly_export.csvmedic.json")
+
+# Next time: same encoding, delimiter, and column types, no re-detection
+df2 = csvmedic.read("monthly_export.csv", schema="monthly_export.csvmedic.json")
+```
+
+## Batch read with consensus
+
+When reading many similar CSVs (e.g. one per month), use consensus so every file gets the same encoding and delimiter:
+
+```python
+dfs = csvmedic.read_batch(["jan.csv", "feb.csv", "mar.csv"], use_consensus=True)
+# Encoding and delimiter are chosen by majority across the three files.
+```
+
+## Diff: pandas vs csvmedic
+
+See exactly what pandas would have changed or corrupted vs what csvmedic preserves:
+
+```python
+result = csvmedic.diff("leading_zeros.csv")
+print(result.summary())           # Columns/rows that differ
+print(result.pandas_df)           # Default pandas read
+print(result.csvmedic_df)         # csvmedic read (e.g. keeps "00742" as string)
+print(result.sample_differences)  # Example (row, column, pandas_val, csvmedic_val)
+```
+
+## How disambiguation works
+
+For ambiguous dates like `03/04/2025` (March 4 or April 3?), csvmedic uses the data itself: if any value has a day > 12 (e.g. `25/03/2025`), the column is treated as day-first. It also uses cross-column inference, separator hints (e.g. period = European), and sequential order. If it still can’t decide, the column stays as string and is marked ambiguous in the diagnosis.
+
+## Documentation
+
+- [Quickstart](docs/quickstart.md)
+- [How it works](docs/how-it-works.md)
+- [API reference](docs/api-reference.md)
+- [FAQ](docs/faq.md)
+
+## License
+
+MIT

csvmedic-0.1.0/README.md

@@ -0,0 +1,107 @@
+# csvmedic
+
+Automatic locale-aware CSV and Excel reader. One line to clean messy data:
+
+```python
+import csvmedic
+
+df = csvmedic.read("messy_file.csv")
+print(df.diagnosis)  # See what was detected and converted
+```
+
+## What it does
+
+| Detects | Examples |
+|--------|----------|
+| **Encoding** | UTF-8, Windows-1252, ISO-8859-1, Shift-JIS, BOM |
+| **Delimiter** | Comma, semicolon, tab, pipe |
+| **Dates** | DD-MM vs MM-DD resolved statistically; ISO, European, US formats |
+| **Numbers** | European (1.234,56) vs US (1,234.56); locale hint |
+| **Booleans** | Yes/No, Ja/Nein, Oui/Non, Sí/No, and more |
+| **Strings** | Preserves leading zeros (IDs like 00742) |
+
+Every transformation is recorded in the `.diagnosis` attribute so you can audit what was changed.
+
+## Installation
+
+```bash
+pip install csvmedic
+```
+
+Optional extras:
+
+- `pip install csvmedic[fast]` — better dialect detection (clevercsv)
+- `pip install csvmedic[excel]` — .xlsx support (openpyxl)
+- `pip install csvmedic[all]` — both
+
+## Configuration
+
+Override auto-detection when you know better:
+
+```python
+df = csvmedic.read(
+    "file.csv",
+    encoding="utf-8",
+    delimiter=";",
+    dayfirst=True,              # Force DD-MM dates
+    preserve_strings=["ID"],    # Never convert these columns
+    sample_rows=2000,           # Rows to use for detection
+    confidence_threshold=0.75,  # Min confidence to convert (0–1)
+)
+```
+
+## Analyze without converting
+
+```python
+profile = csvmedic.read_raw("file.csv")
+print(profile.summary())
+print(profile.columns["Date"].details)
+```
+
+## Schema pinning (recurring files)
+
+Save the detected schema after the first read and reuse it so the next read skips detection:
+
+```python
+df = csvmedic.read("monthly_export.csv")
+csvmedic.save_schema(df.attrs["diagnosis"].file_profile, "monthly_export.csvmedic.json")
+
+# Next time: same encoding, delimiter, and column types, no re-detection
+df2 = csvmedic.read("monthly_export.csv", schema="monthly_export.csvmedic.json")
+```
+
+## Batch read with consensus
+
+When reading many similar CSVs (e.g. one per month), use consensus so every file gets the same encoding and delimiter:
+
+```python
+dfs = csvmedic.read_batch(["jan.csv", "feb.csv", "mar.csv"], use_consensus=True)
+# Encoding and delimiter are chosen by majority across the three files.
+```
+
+## Diff: pandas vs csvmedic
+
+See exactly what pandas would have changed or corrupted vs what csvmedic preserves:
+
+```python
+result = csvmedic.diff("leading_zeros.csv")
+print(result.summary())           # Columns/rows that differ
+print(result.pandas_df)           # Default pandas read
+print(result.csvmedic_df)         # csvmedic read (e.g. keeps "00742" as string)
+print(result.sample_differences)  # Example (row, column, pandas_val, csvmedic_val)
+```
+
+## How disambiguation works
+
+For ambiguous dates like `03/04/2025` (March 4 or April 3?), csvmedic uses the data itself: if any value has a day > 12 (e.g. `25/03/2025`), the column is treated as day-first. It also uses cross-column inference, separator hints (e.g. period = European), and sequential order. If it still can’t decide, the column stays as string and is marked ambiguous in the diagnosis.
+
+## Documentation
+
+- [Quickstart](docs/quickstart.md)
+- [How it works](docs/how-it-works.md)
+- [API reference](docs/api-reference.md)
+- [FAQ](docs/faq.md)
+
+## License
+
+MIT

csvmedic-0.1.0/pyproject.toml

@@ -0,0 +1,74 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "csvmedic"
+version = "0.1.0"
+description = "Automatic locale-aware CSV and Excel reader with encoding, delimiter, date format, and number locale detection."
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.9"
+authors = [
+    { name = "csvmedic contributors" },
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Information Analysis",
+]
+dependencies = [
+    "pandas>=1.5.0",
+    "charset-normalizer>=3.0.0",
+]
+
+[project.optional-dependencies]
+fast = ["clevercsv>=0.8.0"]
+excel = ["openpyxl>=3.1.0"]
+all = ["clevercsv>=0.8.0", "openpyxl>=3.1.0"]
+dev = [
+    "pytest>=7.0",
+    "pytest-cov>=4.0",
+    "ruff>=0.4.0",
+    "mypy>=1.8",
+    "pandas-stubs>=2.0",
+    "mkdocs-material>=9.0",
+    "mkdocstrings[python]>=0.24",
+]
+
+[project.urls]
+Homepage = "https://github.com/csvmedic/csvmedic"
+Documentation = "https://csvmedic.readthedocs.io"
+Repository = "https://github.com/csvmedic/csvmedic"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/csvmedic"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src/csvmedic"]
+
+[tool.ruff]
+line-length = 99
+target-version = "py39"
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "N", "W", "UP"]
+
+[tool.mypy]
+python_version = "3.9"
+strict = true
+warn_return_any = true
+warn_unused_ignores = true
+disallow_untyped_defs = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-v"
+filterwarnings = ["ignore::DeprecationWarning"]

csvmedic-0.1.0/src/csvmedic/__init__.py

@@ -0,0 +1,27 @@
+"""csvmedic — Automatic locale-aware CSV reading."""
+
+from csvmedic import accessor  # noqa: F401 — registers df.diagnosis accessor
+from csvmedic._version import __version__
+from csvmedic.batch import read_batch
+from csvmedic.diagnosis import Diagnosis, TransformationRecord
+from csvmedic.diff import DiffResult, diff
+from csvmedic.models import ColumnProfile, FileProfile
+from csvmedic.reader import MedicReader, read, read_raw
+from csvmedic.schema import load_schema, save_schema, schema_path_for_csv
+
+__all__ = [
+    "__version__",
+    "read",
+    "read_raw",
+    "read_batch",
+    "MedicReader",
+    "Diagnosis",
+    "TransformationRecord",
+    "ColumnProfile",
+    "FileProfile",
+    "save_schema",
+    "load_schema",
+    "schema_path_for_csv",
+    "diff",
+    "DiffResult",
+]

csvmedic-0.1.0/src/csvmedic/_version.py

@@ -0,0 +1,3 @@
+"""Single source of version string for csvmedic."""
+
+__version__ = "0.1.0"

csvmedic-0.1.0/src/csvmedic/accessor.py

@@ -0,0 +1,27 @@
+"""Pandas DataFrame accessor for .diagnosis attribute."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+
+
+@pd.api.extensions.register_dataframe_accessor("diagnosis")
+class DiagnosisAccessor:
+    """Accessor for df.diagnosis — returns the Diagnosis object from df.attrs."""
+
+    def __init__(self, pandas_obj: pd.DataFrame) -> None:
+        self._obj = pandas_obj
+
+    def __repr__(self) -> str:
+        d = self._obj.attrs.get("diagnosis")
+        if d is None:
+            return "No diagnosis available"
+        return repr(d)
+
+    def __getattr__(self, name: str) -> Any:
+        d = self._obj.attrs.get("diagnosis")
+        if d is None:
+            raise AttributeError("No diagnosis available")
+        return getattr(d, name)

csvmedic-0.1.0/src/csvmedic/batch.py

@@ -0,0 +1,102 @@
+"""
+Multi-file batch read with consensus detection.
+
+When reading multiple similar CSVs (e.g. monthly exports), run encoding and
+delimiter detection on each file's sample and use the majority result for all,
+so every file is read with the same settings.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from csvmedic.detectors.dialect import detect_dialect
+from csvmedic.detectors.encoding import detect_encoding
+from csvmedic.reader import _read_byte_sample, read
+
+
+def _consensus_encoding_dialect(
+    paths: list[Path],
+) -> tuple[str, str, bool]:
+    """Detect encoding/dialect per path; return (encoding, delimiter, has_header)."""
+    encodings: list[str] = []
+    delimiters: list[str] = []
+    headers: list[bool] = []
+    for path in paths:
+        try:
+            bytes_sample, _ = _read_byte_sample(path)
+            enc = detect_encoding(bytes_sample)
+            encodings.append(enc.encoding)
+            decoded = bytes_sample.decode(enc.encoding, errors="replace")
+            dialect = detect_dialect(None, enc.encoding, sample_text=decoded)
+            delimiters.append(dialect.delimiter)
+            headers.append(dialect.has_header)
+        except Exception:
+            continue
+    if not encodings:
+        return ("utf-8", ",", True)
+    enc_counter = Counter(encodings)
+    delim_counter = Counter(delimiters)
+    encoding = enc_counter.most_common(1)[0][0]
+    delimiter = delim_counter.most_common(1)[0][0]
+    has_header = sum(headers) > len(headers) / 2
+    return (encoding, delimiter, has_header)
+
+
+def read_batch(
+    paths: str | Path | list[str] | list[Path],
+    *,
+    encoding: str | None = None,
+    delimiter: str | None = None,
+    use_consensus: bool = True,
+    **read_kw: Any,
+) -> list[pd.DataFrame]:
+    """
+    Read multiple CSV files with optional consensus detection.
+
+    When use_consensus is True (default), encoding and delimiter are detected
+    on a sample from each file and the majority choice is used for every file,
+    so all DataFrames are read with the same settings. When use_consensus is
+    False or encoding/delimiter are provided, each file is read with read() and
+    its own detection (or the given overrides).
+
+    Parameters
+    ----------
+    paths : path or list of paths
+        One or more paths to CSV files.
+    encoding : str, optional
+        If set, overrides consensus and is used for all files.
+    delimiter : str, optional
+        If set, overrides consensus and is used for all files.
+    use_consensus : bool
+        If True, run detection on each file and use majority encoding/delimiter.
+    **read_kw
+        Passed through to read() (e.g. sample_rows, confidence_threshold).
+
+    Returns
+    -------
+    list of DataFrame
+        One DataFrame per path, in order.
+    """
+    if isinstance(paths, (str, Path)):
+        paths = [Path(paths)]
+    else:
+        paths = [Path(p) for p in paths]
+    if not paths:
+        return []
+
+    if use_consensus and encoding is None and delimiter is None:
+        enc, delim, has_header = _consensus_encoding_dialect(paths)
+        # Pass consensus to read(); has_header is not a read() kwarg so we rely on detection
+        read_kw["encoding"] = enc
+        read_kw["delimiter"] = delim
+    elif encoding is not None:
+        read_kw["encoding"] = encoding
+    if delimiter is not None:
+        read_kw["delimiter"] = delimiter
+
+    return [read(p, **read_kw) for p in paths]

csvmedic-0.1.0/src/csvmedic/confidence.py

@@ -0,0 +1,8 @@
+"""Confidence scoring for ambiguous detections."""
+
+from __future__ import annotations
+
+
+def clamp_confidence(value: float, min_val: float = 0.0, max_val: float = 1.0) -> float:
+    """Clamp a confidence score to [min_val, max_val]."""
+    return max(min_val, min(max_val, value))

csvmedic-0.1.0/src/csvmedic/detectors/__init__.py

@@ -0,0 +1 @@
+"""Detectors for encoding, dialect, dates, numbers, booleans, strings."""

csvmedic-0.1.0/src/csvmedic/detectors/booleans.py

@@ -0,0 +1,61 @@
+"""Boolean variant detection across locales."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+# (true_values, false_values) per locale
+BOOLEAN_MAP: list[tuple[list[str], list[str]]] = [
+    (["true", "yes", "y", "1", "on"], ["false", "no", "n", "0", "off"]),
+    (["ja", "j", "jaa"], ["nein", "n"]),
+    (["oui", "o", "vrai"], ["non", "faux"]),
+    (["sí", "si", "s"], ["no"]),
+    (["vero", "sì"], ["falso"]),
+    (["waar"], ["onwaar"]),
+]
+
+ALL_TRUE: set[str] = set()
+ALL_FALSE: set[str] = set()
+for trues, falses in BOOLEAN_MAP:
+    ALL_TRUE.update(trues)
+    ALL_FALSE.update(falses)
+
+
+@dataclass
+class BooleanDetectionResult:
+    """Result of boolean detection."""
+
+    is_boolean: bool
+    confidence: float
+    true_variants: list[str]
+    false_variants: list[str]
+
+
+def detect_boolean_column(values: list[str]) -> BooleanDetectionResult:
+    """Detect if column is boolean; require >=90% of non-null values to match."""
+    filtered = [str(v).strip().lower() for v in values if v is not None and str(v).strip()]
+    if not filtered:
+        return BooleanDetectionResult(False, 0.0, [], [])
+
+    unique = set(filtered)
+    if len(unique) > 2:
+        return BooleanDetectionResult(False, 0.0, [], [])
+
+    allowed = ALL_TRUE | ALL_FALSE
+    match_count = sum(1 for v in filtered if v in allowed)
+    ratio = match_count / len(filtered)
+
+    if ratio < 0.9:
+        return BooleanDetectionResult(False, ratio, [], [])
+
+    true_found = [v for v in unique if v in ALL_TRUE]
+    false_found = [v for v in unique if v in ALL_FALSE]
+    if not true_found or not false_found:
+        return BooleanDetectionResult(False, ratio, [], [])
+
+    return BooleanDetectionResult(
+        is_boolean=True,
+        confidence=min(1.0, ratio + 0.05),
+        true_variants=true_found,
+        false_variants=false_found,
+    )