valediction 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

valediction/datasets/datasets.py

@@ -20,7 +20,8 @@ from valediction.io.csv_readers import (
 )
 from valediction.support import (
     _get_runtime_string,
-    _normalise_name,
+    _normalise,
+    _strip,
     list_as_bullets,
     print_bold_red,
     print_red,
@@ -437,16 +438,16 @@ class Dataset(list[DatasetItem]):
 
     # Getters
     def get(self, name: str, default: DatasetItem | None = None) -> DatasetItem | None:
-        name_key = name.strip()
+        name_key = _normalise(name)
         for item in self:
-            if item.name.lower() == name_key.lower():
+            if _normalise(item.name) == name_key:
                 return item
         return default
 
     def index_of(self, name: str) -> int | None:
-        name_key = name.strip()
+        name_key = _normalise(name)
         for i, item in enumerate(self):
-            if item.name == name_key:
+            if _normalise(item.name) == name_key:
                 return i
         return None
 
@@ -796,20 +797,21 @@ class Dataset(list[DatasetItem]):
         name: str | None,
         data: DataLike,
     ) -> DatasetItem:
-        """Normalise a (data, name) double into a DatasetItem."""
+        """Normalise a (name, data) double into a DatasetItem."""
         if isinstance(data, (str, Path)):
             path = Path(data)
             if not path.exists():
                 raise FileNotFoundError(f"File not found: {path}")
             if path.suffix.lower() != ".csv":
                 raise ValueError(f"Only .csv supported right now, got: {path}")
-            resolved_name = _normalise_name(name or path.stem)
+            resolved_name = _strip(name or path.stem)
             return DatasetItem(name=resolved_name, data=path.resolve())
 
         if isinstance(data, DataFrame):
             if not name:
                 raise ValueError("When providing a DataFrame, 'name' is required.")
-            resolved_name = _normalise_name(name)
+            resolved_name = _strip(name)
+            data.columns = [_strip(column) for column in data.columns]
             return DatasetItem(name=resolved_name, data=data)
 
         raise TypeError("data must be a Path/str to .csv or a pandas DataFrame.")
@@ -823,13 +825,11 @@ class Dataset(list[DatasetItem]):
         if p.is_file():
             if p.suffix.lower() != ".csv":
                 raise ValueError(f"Expected a .csv file, got: {p.suffix} ({p})")
-            return [DatasetItem(name=_normalise_name(p.stem), data=p.resolve())]
+            return [DatasetItem(name=_strip(p.stem), data=p.resolve())]
 
         if p.is_dir():
             return [
-                DatasetItem(
-                    name=_normalise_name(csv_path.stem), data=csv_path.resolve()
-                )
+                DatasetItem(name=_strip(csv_path.stem), data=csv_path.resolve())
                 for csv_path in p.glob("*.csv")
             ]
 

valediction/dictionary/generation.py

@@ -24,7 +24,7 @@ from valediction.io.csv_readers import (
     read_csv_sample,
 )
 from valediction.progress import Progress
-from valediction.support import _normalise_name, calculate_runtime
+from valediction.support import _strip, calculate_runtime
 
 IMPORTING_DATA = "Importing data"
 CHUNK_STEPS = 1
@@ -124,7 +124,7 @@ class Generator:
         self.__say(f"Generating dictionary for {len(items)} tables")
         for item in items:
             self.__progress_init(item)
-            table = Table(name=_normalise_name(item.name))
+            table = Table(name=_strip(item.name))
             dictionary.add_table(table)
 
             if item.is_path:
@@ -192,7 +192,7 @@ class Generator:
                     col_state = inferer.states[col_name]
                     data_type, length = col_state.final_data_type_and_length()
                     col = Column(
-                        name=_normalise_name(col_name),
+                        name=_strip(col_name),
                         order=idx,
                         data_type=data_type,
                         length=length if data_type == DataType.TEXT else None,
@@ -242,7 +242,7 @@ class Generator:
             col_state = inferer.states[col_name]
             data_type, length = col_state.final_data_type_and_length()
             col = Column(
-                name=_normalise_name(col_name),
+                name=_strip(col_name),
                 order=idx,
                 data_type=data_type,
                 length=length if data_type == DataType.TEXT else None,
@@ -277,7 +277,7 @@ class Generator:
                 next_order = max((c.order or 0 for c in table), default=0) + 1
                 data_type, length = col_state.final_data_type_and_length()
                 new_col = Column(
-                    name=_normalise_name(col_name),
+                    name=_strip(col_name),
                     order=next_order,
                     data_type=data_type,
                     length=length if data_type == DataType.TEXT else None,

valediction/dictionary/helpers.py

@@ -26,9 +26,6 @@ def _check_name(name: str, entity: Literal["table", "column"]) -> list[str]:
         else config.max_column_name_length
     )
 
-    if name != name.upper():  # name must be uppercase
-        errors.append("must be uppercase")
-
     if invalid_chars.search(name):  # check invalid characters
         bad = set(invalid_chars.findall(name))
         errors.append(
@@ -115,10 +112,6 @@ def _check_primary_key(primary_key: int | None, data_type: DataType) -> list[str
     return errors
 
 
-def _normalise_name(name: str) -> str:
-    return name.upper().strip()
-
-
 def _norm_header_map(columns: list) -> dict:
     mapping, _ = {}, set()
     for c in columns:

valediction/dictionary/importing.py

@@ -11,7 +11,6 @@ from valediction.dictionary.helpers import (
     _get_required_header,
     _is_missing,
     _norm_header_map,
-    _normalise_name,
     _parse_int,
     _parse_truthy,
     _row_is_blank,
@@ -19,7 +18,7 @@ from valediction.dictionary.helpers import (
 from valediction.dictionary.integrity import REQUIRED_SHEETS
 from valediction.dictionary.model import Column, Dictionary, Table
 from valediction.exceptions import DataDictionaryError, DataDictionaryImportError
-from valediction.support import list_as_bullets
+from valediction.support import _normalise, _strip, list_as_bullets
 
 
 @dataclass
@@ -80,6 +79,13 @@ class ExcelDataDictionary:
             raise error
 
     # Import & Helpers
+    def _resolve_table_name(self, name: str) -> str | None:
+        """Return the canonical table name as it appears in Tables sheet (or None)."""
+        target = _normalise(name)
+        return next(
+            (t for t in self.table_metadata.keys() if _normalise(t) == target), None
+        )
+
     def _open_workbook(self) -> None:
         if not self.path.exists():
             raise DataDictionaryImportError(f"File not found: {self.path}")
@@ -140,20 +146,27 @@ class ExcelDataDictionary:
         description_col_header = _get_required_header(header_map, "description")
 
         meta: dict[str, str | None] = {}
+        seen: set[str] = set()
+
         for _, row in tables_df.iterrows():
             if _is_missing(row[table_col_header]):
                 continue
-            table_name = _normalise_name(str(row[table_col_header]))
+
+            table_name = _strip(str(row[table_col_header]))
             table_description = (
                 None
                 if _is_missing(row[description_col_header])
                 else str(row[description_col_header])
             )
-            if table_name in meta:
+
+            key = _normalise(table_name)
+            if key in seen:
                 raise DataDictionaryImportError(
                     f"Duplicate table '{table_name}' in Tables sheet."
                 )
+            seen.add(key)
             meta[table_name] = table_description
+
         if not meta:
             raise DataDictionaryImportError(
                 "Data Dictionary sheet 'Tables' contains no table rows."
@@ -177,12 +190,13 @@ class ExcelDataDictionary:
                 or _is_missing(row[code_col_header])
             ):
                 continue
-            table_name = _normalise_name(str(row[table_col_header]))
-            column_name = _normalise_name(str(row[column_col_header]))
-            enum_map.setdefault((table_name, column_name), {})
-            enum_map[(table_name, column_name)][row[code_col_header]] = row[
-                name_col_header
-            ]
+            table_name = _strip(str(row[table_col_header]))
+            column_name = _strip(str(row[column_col_header]))
+            resolved_table = self._resolve_table_name(table_name) or table_name
+            enum_key = (_normalise(resolved_table), _normalise(column_name))
+            enum_map.setdefault(enum_key, {})
+            enum_map[enum_key][row[code_col_header]] = row[name_col_header]
+
         self.enumerations = enum_map
 
     # Parse Columns
@@ -234,7 +248,12 @@ class ExcelDataDictionary:
 
             self.table_columns[inputs.table_name].append(column_obj)
             if inputs.has_enumerations:
-                self.enum_flags.add((inputs.table_name, inputs.column_name))
+                self.enum_flags.add(
+                    (
+                        _normalise(inputs.table_name),
+                        _normalise(inputs.column_name),
+                    )
+                )
 
         if errors:
             raise DataDictionaryImportError(
@@ -279,7 +298,7 @@ class ExcelDataDictionary:
 
     # Validate Foreign Keys
     def _validate_foreign_keys(self) -> None:
-        name_to_table = {t.name: t for t in self.tables}
+        name_to_table = {_normalise(t.name): t for t in self.tables}
         errors: list[str] = []
         for table in self.tables:
             for column in table:
@@ -292,9 +311,9 @@ class ExcelDataDictionary:
                     )
                     continue
                 target_table_raw, target_column_raw = target.split(".", 1)
-                target_table_name = _normalise_name(target_table_raw)
-                target_column_name = _normalise_name(target_column_raw)
-                referenced_table = name_to_table.get(target_table_name)
+                target_table_name = _strip(target_table_raw)
+                target_column_name = _strip(target_column_raw)
+                referenced_table = name_to_table.get(_normalise(target_table_name))
                 if not referenced_table:
                     errors.append(
                         f"{table.name}.{column.name} references unknown table {target_table_name!r}."
@@ -392,13 +411,17 @@ class ExcelDataDictionary:
                 f"{row_context}: missing required field(s): {', '.join(missing_fields)}."
             )
 
-        table_name = _normalise_name(str(row[table_col_header]))
-        column_name = _normalise_name(str(row[column_col_header]))
-        if table_name not in self.table_metadata:
+        table_name_raw = _strip(str(row[table_col_header]))
+        column_name = _strip(str(row[column_col_header]))
+
+        resolved_table_name = self._resolve_table_name(table_name_raw)
+        if resolved_table_name is None:
             raise DataDictionaryImportError(
-                f"{row_context}: Table '{table_name}' not present in Tables sheet."
+                f"{row_context}: Table '{table_name_raw}' not present in Tables sheet."
             )
 
+        table_name = resolved_table_name
+
         order_int = _parse_int(row[order_col_header], "Order", row_context)
         length_int = (
             _parse_int(row[length_col_header], "Length", row_context, required=False)
@@ -461,7 +484,7 @@ class ExcelDataDictionary:
 
     def _make_column(self, inputs: _ColumnInputs) -> Column:
         enums_for_column = self.enumerations.get(
-            (inputs.table_name, inputs.column_name), {}
+            (_normalise(inputs.table_name), _normalise(inputs.column_name)), {}
         )
         return Column(
             name=inputs.column_name,

valediction/dictionary/model.py

@@ -9,10 +9,9 @@ from valediction.dictionary.helpers import (
     _check_name,
     _check_order,
     _check_primary_key,
-    _normalise_name,
 )
 from valediction.exceptions import DataDictionaryError
-from valediction.support import list_as_bullets
+from valediction.support import _normalise, _strip, list_as_bullets
 
 
 class Column:
@@ -44,7 +43,7 @@ class Column:
         description: str | None = None,
         datetime_format: str | None = None,
     ):
-        self.name = _normalise_name(name)
+        self.name = _strip(name)
         self.order = int(order) if order is not None else None
         self.data_type: DataType = None
         self.length = int(length) if length is not None else None
@@ -127,7 +126,7 @@ class Table(list[Column]):
         columns: list[Column] | None = None,
     ):
         super().__init__()
-        self.name = _normalise_name(name)
+        self.name = _strip(name)
         self.description = description
         for column in columns or []:
             self.add_column(column)
@@ -139,24 +138,28 @@ class Table(list[Column]):
         )
         return f"Table(name={self.name!r}, description={self.description!r}{cols_str})"
 
+    def __key(self, name: str) -> str:
+        return _normalise(name)
+
     def __getitem__(self, key: int | str) -> Column:
         if isinstance(key, int):
             return super().__getitem__(key)
-        target = _normalise_name(key)
-        found = next((c for c in self if c.name == target), None)
+
+        target_key = self.__key(key)
+        found = next((c for c in self if self.__key(c.name) == target_key), None)
         if not found:
             raise KeyError(f"Column {key!r} not found in table {self.name!r}.")
         return found
 
     def __get(self, name: str, default: Column | None = None) -> Column | None:
-        target = _normalise_name(name)
-        return next((c for c in self if c.name == target), default)
+        target_key = self.__key(name)
+        return next((c for c in self if self.__key(c.name) == target_key), default)
 
     # Getters
     def index_of(self, name: str) -> int | None:
-        target = _normalise_name(name)
+        target_key = self.__key(name)
         for i, c in enumerate(self):
-            if c.name == target:
+            if self.__key(c.name) == target_key:
                 return i
         return None
 
@@ -303,16 +306,17 @@ class Table(list[Column]):
         if not isinstance(column, Column):
             raise DataDictionaryError("Only Column objects can be added to a Table.")
 
-        if column.name in self.get_column_names():
-            conflict = self.get_column(column.name)
+        incoming_key = self.__key(column.name)
+        conflict = next((c for c in self if self.__key(c.name) == incoming_key), None)
+        if conflict is not None:
             raise DataDictionaryError(
-                f"Column {column.name!r} already exists (order={conflict.order!r})"
+                f"Column {column.name!r} already exists (order={conflict.order!r}, as {conflict.name!r})."
             )
 
         if column.order in self.get_column_orders():
-            conflict = self.get_column(column.order)
+            conflict_by_order = self.get_column(column.order)
             raise DataDictionaryError(
-                f"Order {column.order!r} already exists (name={conflict.name!r})"
+                f"Order {column.order!r} already exists (name={conflict_by_order.name!r})"
             )
 
         if column.primary_key is not None:
@@ -339,10 +343,7 @@ class Table(list[Column]):
         Raises:
             DataDictionaryError: if the column does not exist
         """
-        if isinstance(column, str):
-            name = self.get_column(column).name
-        else:
-            name = self.get_column(column).name  # by order
+        name = self.get_column(column).name
         remaining = [c for c in self if c.name != name]
         self.clear()
         super().extend(remaining)
@@ -367,16 +368,17 @@ class Table(list[Column]):
         for col in self:
             col.primary_key = None
 
-        # Resolve and dedupe
+        # Resolve and deduplicate
         resolved: list[Column] = []
         seen: set[str] = set()
         for key in primary_keys:
             col = self.get_column(key)
-            if col.name in seen:
+            col_key = self.__key(col.name)
+            if col_key in seen:
                 raise DataDictionaryError(
                     f"Duplicate column {col.name!r} provided for table {self.name!r}."
                 )
-            seen.add(col.name)
+            seen.add(col_key)
             resolved.append(col)
 
         # Assign ordinals 1..N
@@ -416,14 +418,20 @@ class Dictionary(list[Table]):
     ):
         super().__init__()
         self.name = name
+
+        if isinstance(tables, Table):
+            tables = [tables]
+
         for t in tables or []:
             self.add_table(t)
+
         self.organisations = organisations
         self.version = version
         self.version_notes = version_notes
         self.inclusion_criteria = inclusion_criteria
         self.exclusion_criteria = exclusion_criteria
         self.imported = imported
+        self.__check_variables()
 
     # Properties
     @property
@@ -439,24 +447,85 @@ class Dictionary(list[Table]):
         tables = list_as_bullets(elements=[str(t) for t in self], bullet="\n- ")
         return f"Dictionary(name={self.name!r}, imported={self.imported!r}, {tables})"
 
+    def __key(self, name: str) -> str:
+        return _normalise(name)
+
     def __getitem__(self, key: int | str) -> Table:
         if isinstance(key, int):
             return super().__getitem__(key)
-        target = _normalise_name(key)
-        found = next((t for t in self if t.name == target), None)
+
+        target_key = self.__key(key)
+        found = next((t for t in self if self.__key(t.name) == target_key), None)
         if not found:
             raise KeyError(f"Table {key!r} not found in Dictionary.")
         return found
 
-    # Getters
     def __get(self, name: str, default: Table | None = None) -> Table | None:
-        target = _normalise_name(name)
-        return next((t for t in self if t.name == target), default)
+        target_key = self.__key(name)
+        return next((t for t in self if self.__key(t.name) == target_key), default)
+
+    # Checkers
+    def __check_variables(self) -> None:
+        self.__check_name()
+        self.__check_organisations()
+        self.__check_version()
+        self.__check_version_notes()
+        self.__check_criteria()
+
+    def __check_name(self) -> None:
+        # Check name
+        if self.name is not None:
+            if not isinstance(self.name, str):
+                raise DataDictionaryError("Dictionary `name` must be a string.")
+
+    def __check_organisations(self) -> None:
+        # Check organisations
+        if self.organisations is not None:
+            if not isinstance(self.organisations, str):
+                raise DataDictionaryError(
+                    "Dictionary `organisations` must be a string."
+                )
+
+    def __check_version(self) -> None:
+        # Check version
+        if self.version is not None:
+            if not isinstance(self.version, (str, int, float)):
+                raise DataDictionaryError(
+                    "Dictionary `version` must be a string, int, or float."
+                )
+
+            if isinstance(self.version, (int, float)):
+                self.version = str(self.version)
+
+        # Check version_notes
 
+    def __check_version_notes(self) -> None:
+        if self.version_notes is not None:
+            if not isinstance(self.version_notes, str):
+                raise DataDictionaryError(
+                    "Dictionary `version_notes` must be a string."
+                )
+
+    def __check_criteria(self) -> None:
+        # Check inclusion_criteria
+        if self.inclusion_criteria is not None:
+            if not isinstance(self.inclusion_criteria, str):
+                raise DataDictionaryError(
+                    "Dictionary `inclusion_criteria` must be a string."
+                )
+
+        # Check exclusion_criteria
+        if self.exclusion_criteria is not None:
+            if not isinstance(self.exclusion_criteria, str):
+                raise DataDictionaryError(
+                    "Dictionary exclusion_criteria must be a string."
+                )
+
+    # Getters
     def index_of(self, name: str) -> int | None:
-        target = _normalise_name(name)
+        target_key = self.__key(name)
         for i, t in enumerate(self):
-            if t.name == target:
+            if self.__key(t.name) == target_key:
                 return i
         return None
 
@@ -484,12 +553,9 @@ class Dictionary(list[Table]):
         Raises:
             KeyError: If the table is not found in the dictionary.
         """
-        target = _normalise_name(table)
-        found = next((t for t in self if t.name == target), None)
-
-        if not found:
+        found = self.__get(table)
+        if found is None:
             raise KeyError(f"Table {table!r} not found in Dictionary.")
-
         return found
 
     # Manipulation
@@ -508,8 +574,14 @@ class Dictionary(list[Table]):
             raise DataDictionaryError(
                 "Only Table objects can be added to a Dictionary."
             )
-        if table.name in self.get_table_names():
-            raise DataDictionaryError(f"Table {table.name!r} already exists.")
+
+        incoming_key = self.__key(table.name)
+        conflict = next((t for t in self if self.__key(t.name) == incoming_key), None)
+        if conflict is not None:
+            raise DataDictionaryError(
+                f"Table {table.name!r} already exists (as {conflict.name!r})."
+            )
+
         super().append(table)
 
     def remove_table(self, table: str) -> None:

valediction/integrity.py

@@ -1,6 +1,10 @@
+from __future__ import annotations
+
 import re
+from copy import deepcopy
 from pathlib import Path
 from re import Pattern
+from typing import Any
 
 from valediction.data_types.data_types import DataType
 from valediction.support import list_as_bullets
@@ -12,13 +16,58 @@ TEMPLATE_DATA_DICTIONARY_PATH = (
 )
 
 
+externally_injected_variables: dict[
+    str, Any
+] = {}  # External injection store for package wrapping (any keys, always included)
+
+
+def reset_injected_config_variables() -> None:
+    global externally_injected_variables
+    externally_injected_variables = {}
+
+
+def inject_config_variables(variables: dict[str, Any]) -> None:
+    """Injects variables into the Valediction Config, which will always be incorporated
+    as overrides, regardless of Config calling method (default, session-scoped, or
+    contextual).
+
+    Args:
+        variables (dict[str, Any]): Dictionary of config variables.
+    """
+    global externally_injected_variables, session_config
+
+    # check type allows
+    if not isinstance(variables, dict):
+        raise TypeError(
+            f"Config injection variables must be a dictionary, not {type(variables)}"
+        )
+    problematic_keys = []
+    for variable_name in variables.keys():
+        if not isinstance(variable_name, str):
+            problematic_keys.append(variable_name)
+
+    if problematic_keys:
+        raise TypeError("Config injection variables accepts only string keys.")
+
+    externally_injected_variables = dict(variables or {})
+
+    # Apply immediately to the current session config (if it exists)
+    if session_config is not None:
+        _apply_external_injections(session_config)
+
+
+def _apply_external_injections(config: Config) -> None:
+    for variable_name, variable_value in externally_injected_variables.items():
+        setattr(config, variable_name, deepcopy(variable_value))
+
+
 class Config:
     def __init__(self):
         self.template_data_dictionary_path: Path = TEMPLATE_DATA_DICTIONARY_PATH
         self.max_table_name_length: int = 63
         self.max_column_name_length: int = 30
         self.max_primary_keys: int = 7
-        self.invalid_name_pattern: str | Pattern = re.compile(r"[^A-Z0-9_]")
+        self.invalid_name_pattern: str | Pattern = re.compile(r"[^A-Za-z0-9_]")
         self.null_values: list[str] = ["", "null", "none"]
         self.forbidden_characters: list[str] = []
         self.date_formats: dict[str, DataType] = {
@@ -42,6 +91,7 @@ class Config:
         }
         self.enforce_no_null_columns: bool = True
         self.enforce_primary_keys: bool = True
+        _apply_external_injections(self)
 
     def __repr__(self):
         date_list = list_as_bullets(
@@ -65,33 +115,37 @@ class Config:
 
     # Context Wrapper With Reset
     def __enter__(self):
-        global default_config
-        default_config = self
+        global session_config
+
+        _apply_external_injections(self)
+
+        session_config = self
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
-        global default_config
-        default_config = Config()
+        global session_config
+        session_config = Config()
 
 
-default_config: Config = None
+session_config: Config = None
 
 
 def get_config() -> Config:
-    """Gets the current `default_config` instance. Changing attributes will set them
-    globally.
+    """Gets the current `session_config` instance. Changing attributes will set them
+    globally for the python session. Use `reset_default_config()` to reset to original
+    defaults.
 
     Returns:
-        Config: The current default configuration.
+        Config: The current session configuration.
     """
-    global default_config
-    return default_config
+    global session_config
+    return session_config
 
 
 def reset_default_config() -> None:
     """Resets `default_config` settings globally to original defaults."""
-    global default_config
-    default_config = Config()
+    global session_config
+    session_config = Config()
 
 
 reset_default_config()

valediction/io/csv_readers.py

@@ -11,7 +11,7 @@ import pandas as pd
 from pandas import DataFrame
 from pandas.errors import ParserError
 
-from valediction.support import _normalise_name
+from valediction.support import _strip
 
 
 class FrameChunk(NamedTuple):
@@ -34,7 +34,7 @@ class FrameChunk(NamedTuple):
     total_chunks_seen: int | None
 
     def estimate_chunk_count(self) -> int:
-        # Buffers (accounting for CSV tails/bytes innacuracy)
+        # Buffers (accounting for CSV tails/bytes inaccuracy)
         EPS_ABS = 4096  # Fixed
         EPS_REL = 0.05  # 5% tail buffer
 
@@ -93,7 +93,7 @@ def _post_read_processing(df: DataFrame, cfg: CsvReadConfig) -> DataFrame:
     """Apply header normalisation and vectorised value stripping after reading."""
     cfg = cfg or CsvReadConfig()
     if cfg.normalise_headers:
-        df = df.rename(columns={c: _normalise_name(c) for c in df.columns})
+        df = df.rename(columns={c: _strip(c) for c in df.columns})
     if cfg.strip_values:
         str_cols = df.select_dtypes(include=["string"]).columns
         if len(str_cols) > 0:

valediction/support.py

@@ -35,10 +35,14 @@ def list_as_bullets(elements: list, bullet: str = "\n - ") -> str:
     return bullet + bullet.join(elements)
 
 
-def _normalise_name(name: str) -> str:
+def _normalise(name: str) -> str:
     return name.strip().upper()
 
 
+def _strip(name: str) -> str:
+    return name.strip()
+
+
 def _get_runtime_string(runtime: timedelta) -> str:
     total_seconds = runtime.total_seconds()
     hours = trunc(total_seconds / 3600)

valediction/validation/helpers.py

@@ -10,6 +10,7 @@ from pandas.util import hash_pandas_object
 from valediction.data_types.data_types import DataType
 from valediction.dictionary.model import Table
 from valediction.integrity import get_config
+from valediction.support import _normalise
 from valediction.validation.issues import Range
 
 
@@ -17,11 +18,14 @@ from valediction.validation.issues import Range
 def _set_nulls(df: DataFrame) -> DataFrame:
     null_values = get_config().null_values
     token_set = {str(t).strip().casefold() for t in null_values}
-    columns = df.select_dtypes(include=["string", "object"]).columns
+    columns = df.select_dtypes(include=["string", "object", "category"]).columns
     for column in columns:
         series = df[column]
-        mask = series.notna() & series.str.casefold().isin(token_set)
-        df[column] = series.mask(mask, NA)
+
+        s_txt = series.astype("string", copy=False)  # dtype safe
+        mask = s_txt.notna() & s_txt.str.strip().str.casefold().isin(token_set)
+        if mask.any():
+            df[column] = series.mask(mask, NA)
 
     return df
 
@@ -68,37 +72,24 @@ def create_pk_hashes(
     Returns:
         Series: Pandas Series with hashes or Nulls.
     """
-    hash_col_name = "PK_HASH"
+    HASH_COL_NAME = "PK_HASH"
     if df_primaries.empty or df_primaries.shape[1] == 0:
-        return Series([], dtype=object, name=hash_col_name)
+        return Series([], dtype=object, name=HASH_COL_NAME)
 
-    # Any NA in row => invalid PK -> None
+    # Check Nulls
     null_rows = df_primaries.isna().any(axis=1)
 
-    # First Hash
-    hash_1 = hash_pandas_object(df_primaries, index=False)  # uint64
-
-    # Second Hash (rows backwards if single row, else salt)
-    if df_primaries.shape[1] > 1:
-        df_primaries_backwards = df_primaries.iloc[:, ::-1]
-    else:
-        s = df_primaries.iloc[:, 0]
-        salt = Series(["§"] * len(s), index=s.index, dtype="string")
-        df_primaries_backwards = DataFrame(
-            {
-                "_a": s,
-                "_b": s.str.cat(salt),
-            }
-        )
-
-    hash_2 = hash_pandas_object(df_primaries_backwards, index=False)  # uint64
+    # Two independent 64-bit hashes with 16 byte keys
+    hash_1 = hash_pandas_object(df_primaries, index=False, hash_key="valediction_pk1!")
+    hash_2 = hash_pandas_object(df_primaries, index=False, hash_key="valediction_pk2!")
 
+    # Combine into 128-bit integer keys
     a1 = hash_1.to_numpy(dtype="uint64", copy=False).astype(object)
     a2 = hash_2.to_numpy(dtype="uint64", copy=False).astype(object)
-
     combined = (a1 << 64) | a2
+
     hashes = Series(
-        combined, index=df_primaries.index, name=hash_col_name, dtype=object
+        combined, index=df_primaries.index, name=HASH_COL_NAME, dtype=object
     )
     hashes[null_rows] = None
     return hashes
@@ -167,8 +158,9 @@ def pk_contains_whitespace_mask(df_primaries: DataFrame) -> Series:
     if df_primaries.empty or df_primaries.shape[1] == 0:
         return Series(False, index=df_primaries.index)
 
-    col_masks = df_primaries.apply(lambda s: s.str.contains(r"\s", na=False))
-
+    col_masks = df_primaries.apply(
+        lambda s: s.astype("string", copy=False).str.contains(r"\s", na=False)
+    )
     return col_masks.any(axis=1)
 
 
@@ -261,7 +253,9 @@ def invalid_mask_text_too_long(column: Series, max_len: int) -> Series:
         return Series(False, index=column.index)
 
     notnull = column.notna()
-    lens = column.str.len()
+    s_txt = column.astype("string", copy=False)
+    lens = s_txt.str.len()
+
     return notnull & (lens > max_len)
 
 
@@ -270,20 +264,23 @@ def invalid_mask_text_forbidden_characters(column: Series) -> Series:
     if not forbidden:
         return column.notna() & False
 
-    pattern = "[" + re.escape("".join(forbidden)) + "]"
+    pattern = "[" + re.escape("".join([str(s) for s in forbidden])) + "]"
     notnull = column.notna()
-    has_forbidden = column.str.contains(pattern, regex=True, na=False)
+
+    s_txt = column.astype("string", copy=False)
+    has_forbidden = s_txt.str.contains(pattern, regex=True, na=False)
+
     return notnull & has_forbidden
 
 
 # Apply Data Types #
 def apply_data_types(df: DataFrame, table_dictionary: Table) -> DataFrame:
     # name -> column object
-    column_dictionary = {column.name: column for column in table_dictionary}
+    column_dictionary = {_normalise(column.name): column for column in table_dictionary}
 
     for col in df.columns:
-        data_type = column_dictionary.get(col).data_type
-        datetime_format = column_dictionary.get(col).datetime_format
+        data_type = column_dictionary.get(_normalise(col)).data_type
+        datetime_format = column_dictionary.get(_normalise(col)).datetime_format
 
         if data_type in (DataType.TEXT, DataType.FILE):
             df[col] = df[col].astype("string")

valediction/validation/issues.py

@@ -8,7 +8,7 @@ from pandas import DataFrame, concat
 
 from valediction.datasets.datasets_helpers import DatasetItemLike
 from valediction.io.csv_readers import CsvReadConfig, read_csv_ranges
-from valediction.support import _normalise_name, list_as_bullets
+from valediction.support import _strip, list_as_bullets
 
 
 class IssueType(Enum):
@@ -107,6 +107,7 @@ class Issue:
         merged.append(cur)
         self.ranges = merged
 
+    # Inspect
     def inspect(
         self,
         additional_columns: bool | str | list[str] | None = None,
@@ -132,9 +133,9 @@ class Issue:
             ValueError: if the issue has no parent DatasetItem
         """
         # Guard
-        if not self.parent:
-            raise ValueError("Issue has no parent DatasetItem")
+        self.__guard_parent()
         header = self.__repr__() if print_header else ""
+
         # Not applicable
         if self.type in APPLIES_WHOLE_COLUMN:
             print(f"{header}: applies to whole column")
@@ -143,22 +144,8 @@ class Issue:
         # Column Inclusion
         if print_header:
             print(f"{header}:")
-        if additional_columns is True:
-            columns = None
-        else:
-            additional_columns = (
-                [additional_columns]
-                if isinstance(additional_columns, str)
-                else additional_columns
-            )
-            base = (
-                set(self.parent.primary_keys)
-                if self.type in PRIMARY_KEY_ISSUES
-                else {self.column}
-            )
-            base |= set(additional_columns or [])
-            base.discard(None)
-            columns = list(base) if base else None
+
+        columns = self.__select_columns(additional_columns)
 
         if not self.ranges:
             return DataFrame(columns=columns) if columns else DataFrame()
@@ -194,6 +181,31 @@ class Issue:
 
         return out if columns is None else out.loc[:, columns]
 
+    # Inspect Helpers
+    def __guard_parent(self):
+        if not self.parent:
+            raise ValueError("Issue has no parent DatasetItem")
+
+    def __select_columns(self, additional_columns: bool | str | list[str]) -> list:
+        if additional_columns is True:
+            columns = None
+        else:
+            additional_columns = (
+                [additional_columns]
+                if isinstance(additional_columns, str)
+                else additional_columns
+            )
+            base = (
+                set(self.parent.primary_keys)
+                if self.type in PRIMARY_KEY_ISSUES
+                else {self.column}
+            )
+            base |= set(additional_columns or [])
+            base.discard(None)
+            columns = list(base) if base else None
+
+        return columns
+
 
 @dataclass
 class Issues:
@@ -235,8 +247,8 @@ class Issues:
         parent: DatasetItemLike | None = None,
     ) -> Issue:
         key = (
-            _normalise_name(table),
-            _normalise_name(column) if column is not None else None,
+            _strip(table),
+            _strip(column) if column is not None else None,
             issue_type,
         )
         issue = self._index.get(key)
@@ -255,8 +267,8 @@ class Issues:
         issue_type: IssueType | None = None,
     ) -> list[Issue]:
         """Case-insensitive filter; any arg can be None to act as a wildcard."""
-        table = _normalise_name(table)
-        column = _normalise_name(column) if column is not None else None
+        table = _strip(table)
+        column = _strip(column) if column is not None else None
         output: list[Issue] = []
         if issue_type is not None:
             # direct index lookup where possible
@@ -268,9 +280,9 @@ class Issues:
 
         # otherwise scan (still cheap; we maintain a compact list)
         for item in self._items:
-            if _normalise_name(item.table) != table:
+            if _strip(item.table) != table:
                 continue
-            if column is not None and (_normalise_name(item.column) or "") != column:
+            if column is not None and (_strip(item.column) or "") != column:
                 continue
             output.append(item)
         return output

valediction/validation/validation.py

@@ -20,7 +20,7 @@ from valediction.io.csv_readers import (
     iter_csv_chunks,
 )
 from valediction.progress import Progress
-from valediction.support import _get_runtime_string, calculate_runtime
+from valediction.support import _get_runtime_string, _normalise, calculate_runtime
 from valediction.validation.helpers import (
     _column_has_values,
     _set_nulls,
@@ -86,7 +86,9 @@ class Validator:
         self._dt_needs_infer: set[str] = set()
 
         #  Helpers
-        self._column_names: set = set(self.table_dictionary.get_column_names())
+        self._column_names: set[str] = {
+            _normalise(n) for n in self.table_dictionary.get_column_names()
+        }
 
         # Progress Tracking
         self.progress: Progress | None = None
@@ -155,6 +157,20 @@ class Validator:
                 if not datetime_format:
                     self._dt_needs_infer.add(name)
 
+    # Column Scanning
+    def _resolve_df_col(self, df: DataFrame, name: str) -> str | None:
+        """Return the actual df column label matching name case-insensitively."""
+        target = _normalise(name)
+        return next((c for c in df.columns if _normalise(str(c)) == target), None)
+
+    def _resolve_df_cols(self, df: DataFrame, names: list[str]) -> list[str]:
+        resolved: list[str] = []
+        for n in names:
+            c = self._resolve_df_col(df, n)
+            if c is not None:
+                resolved.append(c)
+        return resolved
+
     # Validate
     def validate(self):
         """
@@ -272,28 +288,45 @@ class Validator:
     # Validation: Start Helpers
     def _check_for_missing_columns(self, df: DataFrame):
         self.__begin_step(step="Checking for missing columns")
-        missing = self._column_names - set(df.columns)
-        if missing:
-            for column in missing:
-                self.issues.add(
-                    issue_type=IssueType.MISSING_COLUMN,
-                    table=self.table_name,
-                    column=column,
-                    parent=self.dataset_item,
-                )
+
+        dict_names = self.table_dictionary.get_column_names()
+        dict_keys = {_normalise(name) for name in dict_names}
+
+        df_keys = {_normalise(str(column)) for column in df.columns}
+
+        missing_keys = dict_keys - df_keys
+        if missing_keys:
+            for name in dict_names:
+                if _normalise(name) in missing_keys:
+                    self.issues.add(
+                        issue_type=IssueType.MISSING_COLUMN,
+                        table=self.table_name,
+                        column=name,
+                        parent=self.dataset_item,
+                    )
+
         self.__complete_step()
 
     def _check_for_extra_columns(self, df: DataFrame):
         self.__begin_step(step="Checking for extra columns")
-        extra = set(df.columns) - self._column_names
-        if extra:
-            for column in extra:
-                self.issues.add(
-                    issue_type=IssueType.EXTRA_COLUMN,
-                    table=self.table_name,
-                    column=column,
-                    parent=self.dataset_item,
-                )
+
+        dict_keys = {
+            _normalise(name) for name in self.table_dictionary.get_column_names()
+        }
+        df_cols = [str(column) for column in df.columns]
+        df_keys = {_normalise(column) for column in df_cols}
+
+        extra_keys = df_keys - dict_keys
+        if extra_keys:
+            for col in df_cols:
+                if _normalise(col) in extra_keys:
+                    self.issues.add(
+                        issue_type=IssueType.EXTRA_COLUMN,
+                        table=self.table_name,
+                        column=col,  # report the actual df label
+                        parent=self.dataset_item,
+                    )
+
         self.__complete_step()
 
     # Validation: Chunk Helpers
@@ -319,13 +352,16 @@ class Validator:
 
         # Check for whitespace (text cols only)
         self.__begin_step(step="Checking for primary key whitespace")
-        pk_cols_text = []
-        for column in self.table_dictionary:
-            if column.name in pk_cols and column.data_type in [DataType.TEXT]:
-                pk_cols_text.append(column.name)
+        pk_keys = {_normalise(p) for p in pk_cols}
+        pk_cols_text = [
+            column.name
+            for column in self.table_dictionary
+            if _normalise(column.name) in pk_keys and column.data_type is DataType.TEXT
+        ]
 
         if pk_cols_text:
-            space_mask = pk_contains_whitespace_mask(df[pk_cols_text])
+            pk_cols_text_df = self._resolve_df_cols(df, pk_cols_text)
+            space_mask = pk_contains_whitespace_mask(df[pk_cols_text_df])
             if space_mask.any():
                 self.issues.add(
                     issue_type=IssueType.PK_WHITESPACE,
@@ -343,7 +379,9 @@ class Validator:
 
         # Create primary key hashes
         self.__begin_step(step="Creating primary key hashes")
-        pk_hashes = create_pk_hashes(df[pk_cols])
+        pk_cols_df = self._resolve_df_cols(df, pk_cols)
+        pk_hashes = create_pk_hashes(df[pk_cols_df])
+
         self.__complete_step()
 
         # Primary Key Nulls
@@ -437,44 +475,51 @@ class Validator:
             self.__complete_step()
             return
 
-        columns = [col for col in self._dt_needs_infer if col in df.columns]
-        if not columns:
+        cols = [
+            (dict_col, df_col)
+            for dict_col in self._dt_needs_infer
+            if (df_col := self._resolve_df_col(df, dict_col)) is not None
+        ]
+        if not cols:
             self.__complete_step()
             return
 
-        for column in columns:
-            series = df[column].astype("string", copy=False).str.strip()
-            unique = series.dropna().unique()
+        from valediction.validation.helpers import _allowed_formats_for
+
+        for dict_col, df_col in cols:
+            unique = (
+                df[df_col].astype("string", copy=False).str.strip().dropna().unique()
+            )
             if len(unique) == 0:
                 continue
 
             try:
-                fmt_or_false = infer_datetime_format(Series(unique, dtype="string"))
+                fmt = infer_datetime_format(Series(unique, dtype="string"))
             except ValueError:
-                # ambiguous - try again in later chunk
                 continue
 
-            if fmt_or_false and fmt_or_false is not False:
-                col_dtype = self._find_data_type(column)
-                from valediction.validation.helpers import _allowed_formats_for
-
-                allowed = _allowed_formats_for(col_dtype)
-                if fmt_or_false in allowed:
-                    self._dt_format_cache[column] = fmt_or_false
-                    self._dt_needs_infer.discard(column)
-
-                    # Persist in the dictionary
-                    try:
-                        self.table_dictionary.get_column(
-                            column
-                        ).datetime_format = fmt_or_false
-                    except Exception:
-                        pass
+            if not fmt or fmt is False:
+                continue
+
+            col_dtype = self._find_data_type(dict_col)  # case-insensitive getter
+            if fmt not in _allowed_formats_for(col_dtype):
+                continue
+
+            self._dt_format_cache[dict_col] = fmt
+            self._dt_needs_infer.discard(dict_col)
+
+            try:
+                self.table_dictionary.get_column(dict_col).datetime_format = fmt
+            except Exception:
+                pass
+
         self.__complete_step()
 
     def _check_column_types(self, df: DataFrame, start_row: int) -> None:
         self.__begin_step(step="Checking column types")
-        present = [col for col in df.columns if col in self._column_names]
+        present = [
+            col for col in df.columns if _normalise(str(col)) in self._column_names
+        ]
         for col in present:
             dtype = self._find_data_type(col)
             if dtype == DataType.TEXT:
@@ -506,7 +551,9 @@ class Validator:
 
     def _check_text_lengths(self, df: DataFrame, start_row: int) -> None:
         self.__begin_step(step="Checking text lengths")
-        present = [col for col in df.columns if col in self._column_names]
+        present = [
+            col for col in df.columns if _normalise(str(col)) in self._column_names
+        ]
         for col in present:
             if self._find_data_type(col) != DataType.TEXT:
                 continue
@@ -524,7 +571,9 @@ class Validator:
 
     def _check_text_forbidden_chars(self, df: DataFrame, start_row: int) -> None:
         self.__begin_step(step="Checking for forbidden characters")
-        present = [col for col in df.columns if col in self._column_names]
+        present = [
+            col for col in df.columns if _normalise(str(col)) in self._column_names
+        ]
         for col in present:
             if self._find_data_type(col) != DataType.TEXT:
                 continue

{valediction-1.1.0.dist-info → valediction-1.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: valediction
-Version: 1.1.0
+Version: 1.2.0
 Summary: Valediction is a convenience data validation package that allows generation, import, and constraint enforcement of user-defined data dictionaries against datasets.
 Author-email: Cai Davis <Cai.Davis@uhs.nhs.uk>
 Requires-Python: <4.0,>=3.11

{valediction-1.1.0.dist-info → valediction-1.2.0.dist-info}/RECORD

@@ -1,15 +1,15 @@
 valediction/__init__.py,sha256=HJy57qHyaeENZ0xGf5-jkkal-P92n242UU6vIqtsnaw,511
 valediction/convenience.py,sha256=gDSNcI_T9VKO3Lk1Van4YQCt6hp_fqPyJnUJD8QNP_o,1438
 valediction/exceptions.py,sha256=OtAq_ShVCZeoNx0hWCzJVlVdl3Gm55l72IP1KrKUMR0,748
-valediction/integrity.py,sha256=_4kTywJFSoQIkQ3tFbxMJH1---N8ML9SY0C7j2ZXA-c,3352
+valediction/integrity.py,sha256=2x1xpz1J3dmXCPRSGHPpnbLEvdlJDUuQ9B1y0baZ-mk,5151
 valediction/progress.py,sha256=fXld7BRhp8kk7xPCG50PbRPXvF8RV7Br2hENHuOUlbo,5974
-valediction/support.py,sha256=Kmjopx62dGUnLcIcq-kMAk-YRFx6ip2pCHaCwBFxBIk,2156
+valediction/support.py,sha256=dhKwhtL6dgG709T6fkGaLZDvjYtnxIO9cMmgz477m-I,2207
 valediction/data_types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 valediction/data_types/data_type_helpers.py,sha256=iqcpSPBoFZybkTMHBmxrlv56ZRg8PbqSLtVsuJXC2G0,2282
 valediction/data_types/data_types.py,sha256=MJv_io_hvbLo0G0N38kwj71goXlAo0isPFyS3TU05II,1605
 valediction/data_types/type_inference.py,sha256=11SGYgpvfAfwrDwyOewVIwvfA6pQtDo6i94_xfebYM8,19952
 valediction/datasets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-valediction/datasets/datasets.py,sha256=uDDzmLLAWbzKwXhT_RtLaVH-2f_QEhxAFxNFDotTLdY,30069
+valediction/datasets/datasets.py,sha256=uweSdVkk5-zDBzL8M-cHnC6LETZXnubakajFi0J6L_c,30089
 valediction/datasets/datasets_helpers.py,sha256=AdB3ws5MYFpiXTmHXmSsdm2wZVwDXkXDOtYLvSYhs4I,1159
 valediction/demo/DEMO - Data Dictionary.xlsx,sha256=wj1JG8dHgdALVwV0zSSYnyWMomMTzrHxGFRm491wM_A,45308
 valediction/demo/DEMOGRAPHICS.csv,sha256=ochj8tiHdRByvK2YbZTw5UR3UxbjYxA2xVbA1BfUlbU,3437
@@ -21,18 +21,18 @@ valediction/demo/demo_dictionary.py,sha256=OQcmKpKuRmLQuidYr2KIVF3_78crki5HU8E6R
 valediction/dictionary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 valediction/dictionary/exporting.py,sha256=uRcnVzY5Uj0Yfju4y8sWsjTSP7ywATV9CQDYZMmnws0,18617
 valediction/dictionary/exporting_helpers.py,sha256=O2pOMAHsRIyhLzxrSFrHIHJEf0wEteJguSI2JK9Rxcw,12434
-valediction/dictionary/generation.py,sha256=eN8zBkvppp3fz7winWRx8wW0rhG5Ti88Z4QpXp7wJvE,12350
-valediction/dictionary/helpers.py,sha256=5QLt0isARdkoegVopWmHvTju9JOtz3yLBx7ID67QyHw,5460
-valediction/dictionary/importing.py,sha256=ucpTxAD1rwNdnUAFDeYkyLMppgNsM-LIByCmtAcAvjQ,18841
+valediction/dictionary/generation.py,sha256=CsmQcW6joFGQe-PkhzOXj0uv97hiq7rN3kZKliqox9A,12305
+valediction/dictionary/helpers.py,sha256=DtEbtn__CSs9LWi0oVZD3DtZonRjHeoQA9de-xQ1z3I,5288
+valediction/dictionary/importing.py,sha256=PbE5gLv-y29MAl540ClBCzoTGT8U4Ss0xbzG_GJOpzo,19557
 valediction/dictionary/integrity.py,sha256=k0RLRyNs8dsHyOivl2WCS6jxlhPsW9wfXB48owyokfs,787
-valediction/dictionary/model.py,sha256=7p7FhTicudq16oXzCc7CgK1iF24AoKIjqgpBcGUJrtU,19107
+valediction/dictionary/model.py,sha256=WtTGb5gZAtg7JiurvaWuD1K4DnNkygU-PoEVTZIgBCc,21617
 valediction/dictionary/template/PROJECT - Data Dictionary.xltx,sha256=ZsWmJsSBHvh3ADfrntmeVMWI9Vp_q7zqrTgp7rGd-AI,41721
 valediction/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-valediction/io/csv_readers.py,sha256=6MAOZFZexvNwqiZz-8Er8Me8L4n9NVjLIj6oZOxzZ64,9830
+valediction/io/csv_readers.py,sha256=sKYP_xtOuxwm6ce2eDrphQ_wagxP0RYMXiMlEtkybBg,9812
 valediction/validation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-valediction/validation/helpers.py,sha256=qSs_rlJzZ29TPO0pHHFFg9XqVdl1XRxu048NWDl78I0,10180
-valediction/validation/issues.py,sha256=bvHawjehs_e_2xQGoSvPdOt4t8dfASES3KBKHO3zFZo,9122
-valediction/validation/validation.py,sha256=ahYhWlY6wjwL7blYOfNhNIoRCyXCWnNrYn10G0AQGjU,21555
-valediction-1.1.0.dist-info/METADATA,sha256=tg9HnCfKQhYplzno5mOrw_hSICZZL3FyXPhNedYglOE,612
-valediction-1.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-valediction-1.1.0.dist-info/RECORD,,
+valediction/validation/helpers.py,sha256=rJl_t0XCwt_X3LZAEz5pMihsAKoAIn-qkANNcm4lLf0,10195
+valediction/validation/issues.py,sha256=fBeGjbGGmwGg5XfENU4FtsYhvFfwdqAFSYGNF3UBEI8,9327
+valediction/validation/validation.py,sha256=XgYnRslQTCZCpAHi_AYUkZw4mXM7yjwBw6-iYJXhcao,22961
+valediction-1.2.0.dist-info/METADATA,sha256=vDocxg062EAic20YXBd3rQmfk95hkybDdnpLDQvV0W0,612
+valediction-1.2.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+valediction-1.2.0.dist-info/RECORD,,