valediction 1.0.0__tar.gz

valediction-1.0.0/.gitignore

@@ -0,0 +1,72 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+tests/outputs
+.ruff_cache
+.pytest_cache
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+poetry.lock
+uv.lock
+.coverage
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Secrets
+*.env
+
+# VSCode
+*.vscode
+*.code-workspace
+
+# Virtual Envrionments
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Other
+*.xlsx
+*.csv
+*.ipynb
+*.xltx
+
+# Project Specific
+notebooks/
+exports/
+
+# Project Keep
+!src/valediction/dictionary/template/PROJECT - Data Dictionary.xltx
+!src/valediction/demo/DEMO - Data Dictionary.xlsx
+!src/valediction/demo/DEMOGRAPHICS.csv
+!src/valediction/demo/DIAGNOSES.csv
+!src/valediction/demo/LAB_TESTS.csv
+!src/valediction/demo/VITALS.csv

valediction-1.0.0/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: valediction
+Version: 1.0.0
+Summary: Valediction is a convenience data validation package that allows generation, import, and constraint enforcement of user-defined data dictionaries against datasets.
+Author-email: Cai Davis <Cai.Davis@uhs.nhs.uk>
+Requires-Python: <4.0,>=3.11
+Requires-Dist: certifi<2025,>=2024.2.2
+Requires-Dist: mohawk<2,>=1.1.0
+Requires-Dist: openpyxl<4,>=3.1.5
+Requires-Dist: pandas<3,>=2.2.1
+Requires-Dist: pydantic<3,>=2.11.4
+Requires-Dist: requests<3,>=2.31.0
+Requires-Dist: tabulate<0.10,>=0.9.0
+Requires-Dist: tqdm>=4.67.1
+Requires-Dist: xlsxwriter<4,>=3.2.3

valediction-1.0.0/pyproject.toml

@@ -0,0 +1,57 @@
+[project]
+name = "valediction"
+version = "1.0.0"
+description = "Valediction is a convenience data validation package that allows generation, import, and constraint enforcement of user-defined data dictionaries against datasets."
+authors = [{ name = "Cai Davis", email = "Cai.Davis@uhs.nhs.uk" }]
+requires-python = ">=3.11,<4.0"
+dependencies = [
+    "pandas>=2.2.1,<3",
+    "requests>=2.31.0,<3",
+    "mohawk>=1.1.0,<2",
+    "certifi>=2024.2.2,<2025",
+    "openpyxl>=3.1.5,<4",
+    "xlsxwriter>=3.2.3,<4",
+    "pydantic>=2.11.4,<3",
+    "tabulate>=0.9.0,<0.10",
+    "tqdm>=4.67.1",
+]
+
+[dependency-groups]
+dev = [
+    "pytest>=7.3.1,<8",
+    "pytest-cov>=4.0.0,<5",
+    "commitizen>=3.13.0,<4",
+    "black>=23.12.1,<24",
+    "python-dotenv>=1.0.1,<2",
+    "ipykernel>=6.29.5,<7",
+    "faker>=37.4.0,<38",
+    "mypy>=1.18.2,<2",
+    "types-openpyxl (>=3.1.5.20250919,<4.0.0.0)",
+    "pandas-stubs (>=2.3.2.250827,<3.0.0.0)",
+    "pre-commit>=4.3.0",
+]
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "src/valediction",
+]
+
+[tool.hatch.build.targets.wheel]
+include = [
+    "src/valediction",
+]
+
+[tool.hatch.build.targets.wheel.sources]
+"src/valediction" = "valediction"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.commitizen]
+name = "cz_conventional_commits"
+tag_format = "v$major.$minor.$patch$prerelease"
+version_type = "pep440"
+version_provider = "pep621"
+update_changelog_on_bump = true
+major_version_zero = true

valediction-1.0.0/src/valediction/__init__.py

@@ -0,0 +1,8 @@
+from valediction.datasets.datasets import Dataset  # noqa
+from valediction.dictionary.importing import import_dictionary  # noqa
+from valediction.dictionary.exporting import export_dictionary  # noqa
+from valediction.dictionary.model import Dictionary, Table, Column  # noqa
+from valediction import demo  # noqa
+from valediction.convenience import validate  # noqa
+from valediction.integrity import get_config, reset_default_config, Config  # noqa
+from valediction.data_types.data_types import DataType  # noqa

valediction-1.0.0/src/valediction/convenience.py

@@ -0,0 +1,50 @@
+from pathlib import Path
+
+from pandas import DataFrame
+
+from valediction.datasets.datasets import Dataset
+from valediction.dictionary.importing import import_dictionary
+from valediction.dictionary.model import Dictionary
+
+
+def validate(
+    data: str | Path | dict[str, DataFrame] | Dictionary,
+    dictionary: Dictionary | str | Path,
+    *,
+    import_data: bool = False,
+    chunk_size: int | None = 10_000_000,
+    feedback: bool = True,
+) -> Dataset:
+    """Validate the dataset against the dictionary. Run dataset.check() afterwards to
+    raise Exception if issues.
+
+    Arguments:
+        dataset (str | Path | dict[str, DataFrame]): path to CSV, DataFrame, or dictionary of table names
+            to DataFrames
+        dictionary (Dictionary | str | Path): dictionary to validate against as a Dictionary object
+            or .xlsx filepath
+        import_data (bool, optional): whether to load all data into memory. Defaults to False.
+        chunk_size (int | None, optional): size of chunks for validating data to optimise RAM usage.
+            Defaults to 10_000_000.
+        feedback (bool, optional): whether to provide user feedback on progress. Defaults to True.
+
+    Returns:
+        Dataset: dataset, with or without Issues
+    """
+    dictionary = (
+        dictionary
+        if isinstance(dictionary, Dictionary)
+        else import_dictionary(dictionary)
+    )
+    data: Dataset = Dataset.create_from(data)
+    data.import_dictionary(dictionary)
+
+    if import_data:
+        data.import_data()
+
+    data.validate(
+        chunk_size=chunk_size,
+        feedback=feedback,
+    )
+
+    return data

valediction-1.0.0/src/valediction/data_types/data_type_helpers.py

@@ -0,0 +1,75 @@
+from pandas import NA, Series, to_datetime
+
+from valediction.data_types.data_types import DataType
+from valediction.integrity import get_config
+
+
+def infer_datetime_format(
+    series: Series,
+    slice_sample_size: int = 100,
+) -> str | None:
+    """Efficiently infers date/datetime format (or rules out) by looping over slices of
+    the column series, ruling out formats.
+
+    Args:
+        series (Series): Column data series
+        slice_sample_size (int, optional): Number of rows to test in each slice
+            loop. Defaults to 100.
+
+    Raises:
+        ValueError: No values, or ambiguous format after full scan.
+
+    Returns:
+        str | None: datetime format string, or None if no format matches.
+    """
+    datetime_formats = get_config().date_formats.keys()
+    values = series.str.strip().replace("", NA).dropna()
+    if values.empty:
+        raise ValueError("Series has no non-null values to test.")
+
+    start_i = 0
+    total = len(values)
+    last_ambiguous: list[str] | None = None
+    remaining = list(datetime_formats)
+
+    # loop over slices
+    while start_i < total:
+        end_i = min(start_i + slice_sample_size, total)
+        sample = values.iloc[start_i:end_i]
+
+        valid_formats: list[str] = []
+        for fmt in remaining:
+            try:
+                to_datetime(sample, format=fmt, errors="raise")
+                valid_formats.append(fmt)
+            except Exception:
+                pass
+
+        remaining = valid_formats
+
+        # Decision
+        current = valid_formats
+        if len(current) == 1:
+            return current[0]
+        elif len(current) == 0:
+            return None
+        else:
+            last_ambiguous = current
+            start_i = end_i  # advance to next slice
+
+    # all values scanned and format still ambiguous
+    raise ValueError(f"Ambiguous datetime format after scanning: {last_ambiguous}")
+
+
+def get_date_type(datetime_format: str) -> DataType | None:
+    """Identifies if a datetime format string corresponds to a Date or Datetime data
+    type.
+
+    Args:
+        datetime_format (str): datetime format string
+
+    Returns:
+        DataType | None: DataType of Date, Datetime, or None if not found.
+    """
+    config = get_config()
+    return config.date_formats.get(datetime_format)

valediction-1.0.0/src/valediction/data_types/data_types.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from enum import Enum
+
+
+class DataType(Enum):
+    TEXT = "Text"
+    INTEGER = "Integer"
+    FLOAT = "Float"
+    DATE = "Date"
+    DATETIME = "Datetime"
+    FILE = "File"
+
+    def __str__(self) -> str:
+        return self.value
+
+    def __repr__(self) -> str:
+        return self.value
+
+    @classmethod
+    def parse(cls, data_type: str) -> DataType:
+        """Case-insensitive, forgiving parser."""
+        normalised = (data_type or "").strip().lower()
+        aliases = {
+            "text": cls.TEXT,
+            "string": cls.TEXT,
+            "str": cls.TEXT,
+            "int": cls.INTEGER,
+            "integer": cls.INTEGER,
+            "float": cls.FLOAT,
+            "double": cls.FLOAT,
+            "number": cls.FLOAT,
+            "numeric": cls.FLOAT,
+            "date": cls.DATE,
+            "datetime": cls.DATETIME,
+            "datetime64": cls.DATETIME,
+            "timestamp": cls.DATETIME,
+            "file": cls.FILE,
+            "blob": cls.FILE,
+            "binary": cls.FILE,
+        }
+        try:
+            return aliases[normalised]
+        except KeyError as error:
+            raise ValueError(f"Unknown data type: {data_type!r}") from error
+
+    def allows_length(self) -> bool:
+        """Only TEXT should have a length attribute."""
+        return self in {DataType.TEXT}
+
+    def valid_for_primary_key(self) -> bool:
+        """PKs can only be Text, Integer, Date, Datetime."""
+        return self in {
+            DataType.TEXT,
+            DataType.INTEGER,
+            DataType.DATE,
+            DataType.DATETIME,
+        }