valediction 1.0.0__py3-none-any.whl

valediction/__init__.py

@@ -0,0 +1,8 @@
+from valediction.datasets.datasets import Dataset  # noqa
+from valediction.dictionary.importing import import_dictionary  # noqa
+from valediction.dictionary.exporting import export_dictionary  # noqa
+from valediction.dictionary.model import Dictionary, Table, Column  # noqa
+from valediction import demo  # noqa
+from valediction.convenience import validate  # noqa
+from valediction.integrity import get_config, reset_default_config, Config  # noqa
+from valediction.data_types.data_types import DataType  # noqa

valediction/convenience.py

@@ -0,0 +1,50 @@
+from pathlib import Path
+
+from pandas import DataFrame
+
+from valediction.datasets.datasets import Dataset
+from valediction.dictionary.importing import import_dictionary
+from valediction.dictionary.model import Dictionary
+
+
+def validate(
+    data: str | Path | dict[str, DataFrame] | Dictionary,
+    dictionary: Dictionary | str | Path,
+    *,
+    import_data: bool = False,
+    chunk_size: int | None = 10_000_000,
+    feedback: bool = True,
+) -> Dataset:
+    """Validate the dataset against the dictionary. Run dataset.check() afterwards to
+    raise Exception if issues.
+
+    Arguments:
+        dataset (str | Path | dict[str, DataFrame]): path to CSV, DataFrame, or dictionary of table names
+            to DataFrames
+        dictionary (Dictionary | str | Path): dictionary to validate against as a Dictionary object
+            or .xlsx filepath
+        import_data (bool, optional): whether to load all data into memory. Defaults to False.
+        chunk_size (int | None, optional): size of chunks for validating data to optimise RAM usage.
+            Defaults to 10_000_000.
+        feedback (bool, optional): whether to provide user feedback on progress. Defaults to True.
+
+    Returns:
+        Dataset: dataset, with or without Issues
+    """
+    dictionary = (
+        dictionary
+        if isinstance(dictionary, Dictionary)
+        else import_dictionary(dictionary)
+    )
+    data: Dataset = Dataset.create_from(data)
+    data.import_dictionary(dictionary)
+
+    if import_data:
+        data.import_data()
+
+    data.validate(
+        chunk_size=chunk_size,
+        feedback=feedback,
+    )
+
+    return data

valediction/data_types/data_type_helpers.py

@@ -0,0 +1,75 @@
+from pandas import NA, Series, to_datetime
+
+from valediction.data_types.data_types import DataType
+from valediction.integrity import get_config
+
+
+def infer_datetime_format(
+    series: Series,
+    slice_sample_size: int = 100,
+) -> str | None:
+    """Efficiently infers date/datetime format (or rules out) by looping over slices of
+    the column series, ruling out formats.
+
+    Args:
+        series (Series): Column data series
+        slice_sample_size (int, optional): Number of rows to test in each slice
+            loop. Defaults to 100.
+
+    Raises:
+        ValueError: No values, or ambiguous format after full scan.
+
+    Returns:
+        str | None: datetime format string, or None if no format matches.
+    """
+    datetime_formats = get_config().date_formats.keys()
+    values = series.str.strip().replace("", NA).dropna()
+    if values.empty:
+        raise ValueError("Series has no non-null values to test.")
+
+    start_i = 0
+    total = len(values)
+    last_ambiguous: list[str] | None = None
+    remaining = list(datetime_formats)
+
+    # loop over slices
+    while start_i < total:
+        end_i = min(start_i + slice_sample_size, total)
+        sample = values.iloc[start_i:end_i]
+
+        valid_formats: list[str] = []
+        for fmt in remaining:
+            try:
+                to_datetime(sample, format=fmt, errors="raise")
+                valid_formats.append(fmt)
+            except Exception:
+                pass
+
+        remaining = valid_formats
+
+        # Decision
+        current = valid_formats
+        if len(current) == 1:
+            return current[0]
+        elif len(current) == 0:
+            return None
+        else:
+            last_ambiguous = current
+            start_i = end_i  # advance to next slice
+
+    # all values scanned and format still ambiguous
+    raise ValueError(f"Ambiguous datetime format after scanning: {last_ambiguous}")
+
+
+def get_date_type(datetime_format: str) -> DataType | None:
+    """Identifies if a datetime format string corresponds to a Date or Datetime data
+    type.
+
+    Args:
+        datetime_format (str): datetime format string
+
+    Returns:
+        DataType | None: DataType of Date, Datetime, or None if not found.
+    """
+    config = get_config()
+    return config.date_formats.get(datetime_format)

valediction/data_types/data_types.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from enum import Enum
+
+
+class DataType(Enum):
+    TEXT = "Text"
+    INTEGER = "Integer"
+    FLOAT = "Float"
+    DATE = "Date"
+    DATETIME = "Datetime"
+    FILE = "File"
+
+    def __str__(self) -> str:
+        return self.value
+
+    def __repr__(self) -> str:
+        return self.value
+
+    @classmethod
+    def parse(cls, data_type: str) -> DataType:
+        """Case-insensitive, forgiving parser."""
+        normalised = (data_type or "").strip().lower()
+        aliases = {
+            "text": cls.TEXT,
+            "string": cls.TEXT,
+            "str": cls.TEXT,
+            "int": cls.INTEGER,
+            "integer": cls.INTEGER,
+            "float": cls.FLOAT,
+            "double": cls.FLOAT,
+            "number": cls.FLOAT,
+            "numeric": cls.FLOAT,
+            "date": cls.DATE,
+            "datetime": cls.DATETIME,
+            "datetime64": cls.DATETIME,
+            "timestamp": cls.DATETIME,
+            "file": cls.FILE,
+            "blob": cls.FILE,
+            "binary": cls.FILE,
+        }
+        try:
+            return aliases[normalised]
+        except KeyError as error:
+            raise ValueError(f"Unknown data type: {data_type!r}") from error
+
+    def allows_length(self) -> bool:
+        """Only TEXT should have a length attribute."""
+        return self in {DataType.TEXT}
+
+    def valid_for_primary_key(self) -> bool:
+        """PKs can only be Text, Integer, Date, Datetime."""
+        return self in {
+            DataType.TEXT,
+            DataType.INTEGER,
+            DataType.DATE,
+            DataType.DATETIME,
+        }