valediction 1.0.0__py3-none-any.whl

valediction/dictionary/generation.py

@@ -0,0 +1,357 @@
+# valediction/dictionary/generation.py
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import timedelta
+from pathlib import Path
+from typing import Iterable
+
+import pandas as pd
+from pandas import DataFrame
+
+from valediction.data_types.data_types import DataType
+from valediction.data_types.type_inference import (
+    COLUMN_STEPS,
+    ColumnState,
+    TypeInferer,
+)
+from valediction.datasets.datasets_helpers import DatasetItemLike
+from valediction.dictionary.model import Column, Dictionary, Table
+from valediction.io.csv_readers import (
+    CsvReadConfig,
+    iter_csv_chunks,
+    read_csv_headers,
+    read_csv_sample,
+)
+from valediction.progress import Progress
+from valediction.support import _normalise_name, calculate_runtime
+
+IMPORTING_DATA = "Importing data"
+CHUNK_STEPS = 1
+COLUMN_STEPS = COLUMN_STEPS
+
+
+@dataclass(slots=True)
+class GeneratorConfig:
+    chunk_size: int = 10_000_000
+    sample_rows: int | None = None
+    dayfirst: bool = True
+    infer_types: bool = True
+    infer_max_length: bool = True
+
+    def set_variables(
+        self,
+        chunk_size: int | None = None,
+        sample_rows: int | None = None,
+    ) -> None:
+        # Set user variables
+        self.chunk_size = chunk_size
+        self.sample_rows = sample_rows
+
+
+class Generator:
+    """
+    Summary:
+        Generator class for creating dictionaries from datasets.
+
+    Arguments:
+        feedback (bool): Provide user feedback on progress (default: True)
+        debug (bool): Enable debug mode, providing full log of data type inference and
+            reasoning (default: False)
+        chunk_size (int | None): Size of chunks for reading data to optimise RAM usage,
+            if reading from CSV (default: 10_000_000)
+        sample_rows (int | None): Number of rows to sample for data type inference. Note:
+            this overrides `chunk_size` and reads in a single chunk (default: None)
+
+    Raises:
+        DataDictionaryError: If there is an issue with the data dictionary
+    """
+
+    def __init__(
+        self,
+        feedback: bool = True,
+        debug: bool = False,
+        chunk_size: int | None = 10_000_000,
+        sample_rows: int | None = None,
+    ) -> None:
+        # User Config
+        self.config = GeneratorConfig()
+        self.config.set_variables(sample_rows=sample_rows, chunk_size=chunk_size)
+        self.feedback: bool = feedback
+        self.debug: bool = debug
+        self.csv_cfg: CsvReadConfig = CsvReadConfig()
+
+        # Progress
+        self.progress: Progress = None
+
+        # Setup
+        if sample_rows is not None:
+            self.config.sample_rows = int(sample_rows)
+        if chunk_size is not None:
+            self.config.chunk_size = int(chunk_size)
+
+    def __say(
+        self,
+        *values: object,
+        sep: str | None = " ",
+        end: str | None = "\n",
+    ) -> None:
+        if self.feedback:
+            print(*values, sep=sep, end=end)
+
+    def generate_dictionary(
+        self,
+        items: Iterable[DatasetItemLike],
+        dictionary_name: str | None = None,
+        primary_keys: dict[str, list[str]] | None = None,
+    ) -> Dictionary:
+        """
+        Summary:
+            Generate a dictionary from a Dataset.
+
+        Arguments:
+            items (Dataset): A list of DatasetItems to generate the dictionary from.
+            dictionary_name (str | None): The name of the dictionary to generate.
+                If None, will not be set.
+            primary_keys (dict[str, list[str]] | None): A dictionary of primary keys
+                to set on the generated dictionary. If None, will not be set.
+
+        Returns:
+            Dictionary: The generated dictionary.
+        """
+        dictionary = Dictionary(name=dictionary_name, imported=True)
+
+        self.__say(f"Generating dictionary for {len(items)} tables")
+        for item in items:
+            self.__progress_init(item)
+            table = Table(name=_normalise_name(item.name))
+            dictionary.add_table(table)
+
+            if item.is_path:
+                self._infer_from_csv_into_table(item, table)
+            else:
+                self._infer_from_dataframe_into_table(item.data, table)
+
+            item._dictionary_runtimes = self.__finish_generation_for_table()
+
+        dictionary.set_primary_keys(primary_keys or {})
+        self.__say("\n", end="")
+        return dictionary
+
+    # Generation Helpers
+    def _infer_from_csv_into_table(self, item: DatasetItemLike, table: Table) -> None:
+        self.__begin_step(step=IMPORTING_DATA)
+        csv_path = item.data
+        inferer = TypeInferer(
+            debug=self.debug,
+            dayfirst=self.config.dayfirst,
+            progress=self.progress,
+        )
+
+        # Read single sample
+        if self.config.sample_rows is not None:
+            self.__begin_step(step=IMPORTING_DATA)
+            df = read_csv_sample(
+                csv_path,
+                nrows=self.config.sample_rows,
+                cfg=self.csv_cfg,
+            ).df
+            self.__complete_step()
+
+            inferer.update_with_chunk(df)
+            self._create_or_update_columns(table, inferer)
+            return
+
+        # Read in chunks
+        first_chunk = True
+        columns_by_name: dict[str, Column] = {}
+        column_count = item.column_count
+        iterator = iter_csv_chunks(
+            path=Path(csv_path), chunk_size=self.config.chunk_size, cfg=self.csv_cfg
+        )
+
+        while True:
+            # Import chunk
+            try:
+                chunk = next(iterator)
+            except StopIteration:
+                break
+
+            est_chunk_count = chunk.estimate_chunk_count()
+            self.__progress_retarget_total(
+                est_chunk_count=est_chunk_count, column_count=column_count
+            )
+            self.__complete_step()
+
+            inferer.update_with_chunk(chunk.df)
+
+            self.__begin_step(step="Saving chunk data types")
+            if first_chunk:
+                ordered = list(inferer.states.keys())
+                for idx, col_name in enumerate(ordered, start=1):
+                    col_state = inferer.states[col_name]
+                    data_type, length = col_state.final_data_type_and_length()
+                    col = Column(
+                        name=_normalise_name(col_name),
+                        order=idx,
+                        data_type=data_type,
+                        length=length if data_type == DataType.TEXT else None,
+                        vocabulary=None,
+                        primary_key=None,
+                        foreign_key=None,
+                        description=None,
+                        enumerations=None,
+                    )
+
+                    self._set_datetime_format(column_state=col_state, column=col)
+                    table.add_column(col)
+                    columns_by_name[col_name] = col
+                first_chunk = False
+
+            else:
+                self._apply_state_to_existing_columns(table, inferer, columns_by_name)
+
+        if first_chunk:
+            empty = read_csv_headers(
+                csv_path,
+                cfg=self.csv_cfg,
+            )
+            inferer.update_with_chunk(empty)
+            self._create_or_update_columns(table, inferer)
+
+    def _infer_from_dataframe_into_table(self, df: pd.DataFrame, table: Table) -> None:
+        self.__begin_step(step=IMPORTING_DATA)
+        inferer = TypeInferer(
+            debug=self.debug,
+            dayfirst=self.config.dayfirst,
+            progress=self.progress,
+        )
+        self.__complete_step()
+
+        inferer.update_with_chunk(df)
+        self._create_or_update_columns(table, inferer)
+
+    # Emit/Update Helpers
+    def _create_or_update_columns(self, table: Table, inferer: TypeInferer) -> None:
+        if len(table):
+            for existing in table:
+                table.remove_column(existing.name)
+
+        ordered = list(inferer.states.keys())
+        for idx, col_name in enumerate(ordered, start=1):
+            col_state = inferer.states[col_name]
+            data_type, length = col_state.final_data_type_and_length()
+            col = Column(
+                name=_normalise_name(col_name),
+                order=idx,
+                data_type=data_type,
+                length=length if data_type == DataType.TEXT else None,
+                vocabulary=None,
+                primary_key=None,
+                foreign_key=None,
+                description=None,
+                enumerations=None,
+            )
+            self._set_datetime_format(column_state=col_state, column=col)
+
+            table.add_column(col)
+
+    def _set_datetime_format(self, column_state: ColumnState, column: Column) -> None:
+        if column.data_type in (DataType.DATE, DataType.DATETIME):
+            datetime_format = getattr(column_state, "cached_datetime_format", None)
+            if datetime_format and hasattr(column, "datetime_format"):
+                column.datetime_format = datetime_format
+
+        else:
+            if hasattr(column, "datetime_format"):
+                column.datetime_format = None
+
+    def _apply_state_to_existing_columns(
+        self,
+        table: Table,
+        inferer: TypeInferer,
+        columns_by_name: dict[str, Column],
+    ) -> None:
+        for col_name, col_state in inferer.states.items():
+            if col_name not in columns_by_name:
+                next_order = max((c.order or 0 for c in table), default=0) + 1
+                data_type, length = col_state.final_data_type_and_length()
+                new_col = Column(
+                    name=_normalise_name(col_name),
+                    order=next_order,
+                    data_type=data_type,
+                    length=length if data_type == DataType.TEXT else None,
+                    vocabulary=None,
+                    primary_key=None,
+                    foreign_key=None,
+                    description=None,
+                    enumerations=None,
+                )
+                self._set_datetime_format(column_state=col_state, column=new_col)
+                table.add_column(new_col)
+                columns_by_name[col_name] = new_col
+                continue
+
+            col = columns_by_name[col_name]
+            data_type, length = col_state.final_data_type_and_length()
+
+            if col.data_type != data_type:
+                col.data_type = data_type
+
+            if data_type == DataType.TEXT:
+                if length is not None and (col.length or 0) < length:
+                    col.length = int(length)
+            else:
+                col.length = None
+
+            self._set_datetime_format(column_state=col_state, column=col)
+
+    # Progress
+    def __progress_init(self, item: DatasetItemLike) -> None:
+        # Switch to debug mode
+        if self.debug:
+            self.progress = Progress(enabled=False)
+            return
+
+        # Switch to silent mode
+        if not self.feedback:
+            self.progress = Progress(enabled=False)
+            return
+
+        # Progress bars on
+        total_steps = (
+            (CHUNK_STEPS + (COLUMN_STEPS * item.column_count))
+            if (isinstance(item.data, DataFrame) or self.config.sample_rows)
+            else None
+        )
+        pad = " " * item._padding if item._padding else ""
+
+        self.progress = Progress(
+            desc=f"Generating {item.name}: {pad}",
+            starting_step=IMPORTING_DATA,
+            est_total=total_steps,
+            smoothing_steps=(COLUMN_STEPS * item.column_count),
+        )
+
+    def __progress_retarget_total(
+        self, est_chunk_count: int, column_count: int
+    ) -> None:
+        new_total = (CHUNK_STEPS * est_chunk_count) + (
+            COLUMN_STEPS * est_chunk_count * column_count
+        )
+        self.progress.retarget_total(new_total=new_total)
+
+    def __begin_step(self, step: str | None = None) -> None:
+        self.progress.begin_step(step=step)
+
+    def __complete_step(self) -> None:
+        self.progress.complete_step()
+
+    def __finish_generation_for_table(self) -> dict[str, timedelta]:
+        step = (
+            f"Completed ({calculate_runtime(start=self.progress.full_start).message})"
+        )
+        save_as = "Total"
+        self.progress.finish(postfix=step, save_as=save_as, good=True)
+        self.progress.close()
+        return self.progress.runtimes

valediction/dictionary/helpers.py

@@ -0,0 +1,174 @@
+import re
+from typing import Any, Literal
+
+from pandas import Series
+from pandas import isna as _pd_isna
+
+from valediction.data_types.data_types import DataType
+from valediction.exceptions import DataDictionaryImportError
+from valediction.integrity import get_config
+
+
+def _check_name(name: str, entity: Literal["table", "column"]) -> list[str]:
+    if entity not in ["table", "column"]:
+        raise ValueError("entity must be either 'table' or 'column'")
+
+    errors: list = []
+    config = get_config()
+    invalid_chars = (
+        config.invalid_name_pattern
+        if isinstance(config.invalid_name_pattern, re.Pattern)
+        else re.compile(config.invalid_name_pattern)
+    )
+    max_name_length = (
+        config.max_table_name_length
+        if entity == "table"
+        else config.max_column_name_length
+    )
+
+    if name != name.upper():  # name must be uppercase
+        errors.append("must be uppercase")
+
+    if invalid_chars.search(name):  # check invalid characters
+        bad = set(invalid_chars.findall(name))
+        errors.append(
+            f"invalid characters: '{''.join(sorted(bad))}'; "
+            "only A-Z, 0-9, and underscores are allowed with no whitespace"
+        )
+
+    if len(name) > max_name_length:  # max length 30
+        errors.append(f"exceeds max length of {max_name_length}")
+
+    if not name[0].isalpha():  # column starts with a letter
+        errors.append("must start with a letter")
+
+    if name.endswith("_"):  # column cannot end with an underscore
+        errors.append("cannot end with '_'")
+
+    if "__" in name:  # column cannot contain double underscores
+        errors.append("cannot contain double underscores '__'")
+
+    return errors
+
+
+def _check_order(order: int | None) -> list[str]:
+    errors: list = []
+    if order is None:  # presence
+        errors.append("order is required and must be an integer ≥ 1")
+        return errors
+
+    if not isinstance(order, int):  # type integer
+        errors.append("order must be an integer ≥ 1")
+        return errors
+
+    if order < 1:  # must be ≥ 1
+        errors.append("order must be ≥ 1")
+        return errors
+
+    return errors
+
+
+def _check_data_type(data_type: DataType, length: int | None) -> list[str]:
+    errors: list = []
+    if not isinstance(data_type, DataType):  # Ensure is a DataType
+        errors.append("data type is invalid; must be a DataType object")
+
+    if length is not None:  # length rules
+        if not isinstance(length, int):
+            errors.append("length must be an positive integer if provided")
+        if length <= 0:  # must be positive
+            errors.append("length must be an positive integer if provided")
+
+    if data_type == DataType.TEXT:  # required for DataType.TEXT
+        if length is None:
+            errors.append("length is required for TEXT columns")
+    else:
+        if length is not None:  # length not applicable
+            errors.append(f"length is not applicable to {data_type.value} columns")
+
+    return errors
+
+
+def _check_primary_key(primary_key: int | None, data_type: DataType) -> list[str]:
+    errors: list = []
+    if primary_key is None:
+        return errors
+
+    if (
+        not isinstance(primary_key, int)
+        or primary_key < 1
+        or primary_key > get_config().max_primary_keys
+    ):
+        errors.append(
+            "primary key order must be an integer between 1 and 7 if provided"
+        )
+
+    if (
+        hasattr(data_type, "valid_for_primary_key")
+        and not data_type.valid_for_primary_key()
+    ):
+        errors.append(
+            f"invalid data type '{data_type.value}' for primary key column; "
+            "primary keys must be Text, Integer, Date, or Datetime"
+        )
+
+    return errors
+
+
+def _normalise_name(name: str) -> str:
+    return name.upper().strip()
+
+
+def _norm_header_map(columns: list) -> dict:
+    mapping, _ = {}, set()
+    for c in columns:
+        k = str(c).strip().lower().replace(" ", "_").replace("-", "_")
+        if k in mapping:  # collision
+            raise DataDictionaryImportError(
+                f"Ambiguous headers after normalisation: {mapping[k]!r} and {c!r} both map to {k!r}"
+            )
+        mapping[k] = c
+    return mapping
+
+
+def _get_required_header(header_map: dict[str, str], key: str) -> str:
+    if key not in header_map:
+        raise DataDictionaryImportError(
+            f"Required Data Dictionary column '{key}' not found. Available: {list(header_map.keys())}"
+        )
+    return header_map[key]
+
+
+def _is_missing(val: Any) -> bool:
+    return _pd_isna(val) or (isinstance(val, str) and val.strip() == "")
+
+
+def _parse_truthy(val: Any) -> bool:
+    if isinstance(val, str):
+        return val.strip().lower() in {"y", "yes", "true", "1"}
+    if isinstance(val, (int, float)):
+        try:
+            return int(val) == 1
+        except Exception:
+            return False
+    return False
+
+
+def _row_is_blank(row: Series, keys: tuple[str, str]) -> bool:
+    a, b = keys
+    return _is_missing(row[a]) and _is_missing(row[b])
+
+
+def _parse_int(
+    value: Any, label: str, row_ctx: str, *, required: bool = True
+) -> int | None:
+    if _is_missing(value):
+        if required:
+            raise DataDictionaryImportError(f"{row_ctx}: {label} is required.")
+        return None
+    try:
+        return int(value)
+    except Exception as e:
+        raise DataDictionaryImportError(
+            f"{row_ctx}: {label} must be integer (got {value!r})."
+        ) from e