valediction 1.1.0__py3-none-any.whl → 1.5.0__py3-none-any.whl

valediction/dictionary/model.py

@@ -9,10 +9,9 @@ from valediction.dictionary.helpers import (
     _check_name,
     _check_order,
     _check_primary_key,
-    _normalise_name,
 )
 from valediction.exceptions import DataDictionaryError
-from valediction.support import list_as_bullets
+from valediction.support import _normalise, _strip, list_as_bullets
 
 
 class Column:
@@ -44,7 +43,7 @@ class Column:
         description: str | None = None,
         datetime_format: str | None = None,
     ):
-        self.name = _normalise_name(name)
+        self.name = _strip(name)
         self.order = int(order) if order is not None else None
         self.data_type: DataType = None
         self.length = int(length) if length is not None else None
@@ -127,7 +126,7 @@ class Table(list[Column]):
         columns: list[Column] | None = None,
     ):
         super().__init__()
-        self.name = _normalise_name(name)
+        self.name = _strip(name)
         self.description = description
         for column in columns or []:
             self.add_column(column)
@@ -139,24 +138,28 @@ class Table(list[Column]):
         )
         return f"Table(name={self.name!r}, description={self.description!r}{cols_str})"
 
+    def __key(self, name: str) -> str:
+        return _normalise(name)
+
     def __getitem__(self, key: int | str) -> Column:
         if isinstance(key, int):
             return super().__getitem__(key)
-        target = _normalise_name(key)
-        found = next((c for c in self if c.name == target), None)
+
+        target_key = self.__key(key)
+        found = next((c for c in self if self.__key(c.name) == target_key), None)
         if not found:
             raise KeyError(f"Column {key!r} not found in table {self.name!r}.")
         return found
 
     def __get(self, name: str, default: Column | None = None) -> Column | None:
-        target = _normalise_name(name)
-        return next((c for c in self if c.name == target), default)
+        target_key = self.__key(name)
+        return next((c for c in self if self.__key(c.name) == target_key), default)
 
     # Getters
     def index_of(self, name: str) -> int | None:
-        target = _normalise_name(name)
+        target_key = self.__key(name)
         for i, c in enumerate(self):
-            if c.name == target:
+            if self.__key(c.name) == target_key:
                 return i
         return None
 
@@ -303,16 +306,17 @@ class Table(list[Column]):
         if not isinstance(column, Column):
             raise DataDictionaryError("Only Column objects can be added to a Table.")
 
-        if column.name in self.get_column_names():
-            conflict = self.get_column(column.name)
+        incoming_key = self.__key(column.name)
+        conflict = next((c for c in self if self.__key(c.name) == incoming_key), None)
+        if conflict is not None:
             raise DataDictionaryError(
-                f"Column {column.name!r} already exists (order={conflict.order!r})"
+                f"Column {column.name!r} already exists (order={conflict.order!r}, as {conflict.name!r})."
             )
 
         if column.order in self.get_column_orders():
-            conflict = self.get_column(column.order)
+            conflict_by_order = self.get_column(column.order)
             raise DataDictionaryError(
-                f"Order {column.order!r} already exists (name={conflict.name!r})"
+                f"Order {column.order!r} already exists (name={conflict_by_order.name!r})"
             )
 
         if column.primary_key is not None:
@@ -339,10 +343,7 @@ class Table(list[Column]):
         Raises:
             DataDictionaryError: if the column does not exist
         """
-        if isinstance(column, str):
-            name = self.get_column(column).name
-        else:
-            name = self.get_column(column).name  # by order
+        name = self.get_column(column).name
         remaining = [c for c in self if c.name != name]
         self.clear()
         super().extend(remaining)
@@ -367,16 +368,17 @@ class Table(list[Column]):
         for col in self:
             col.primary_key = None
 
-        # Resolve and dedupe
+        # Resolve and deduplicate
         resolved: list[Column] = []
         seen: set[str] = set()
         for key in primary_keys:
             col = self.get_column(key)
-            if col.name in seen:
+            col_key = self.__key(col.name)
+            if col_key in seen:
                 raise DataDictionaryError(
                     f"Duplicate column {col.name!r} provided for table {self.name!r}."
                 )
-            seen.add(col.name)
+            seen.add(col_key)
             resolved.append(col)
 
         # Assign ordinals 1..N
@@ -416,14 +418,20 @@ class Dictionary(list[Table]):
     ):
         super().__init__()
         self.name = name
+
+        if isinstance(tables, Table):
+            tables = [tables]
+
         for t in tables or []:
             self.add_table(t)
+
         self.organisations = organisations
         self.version = version
         self.version_notes = version_notes
         self.inclusion_criteria = inclusion_criteria
         self.exclusion_criteria = exclusion_criteria
         self.imported = imported
+        self.__check_variables()
 
     # Properties
     @property
@@ -439,24 +447,85 @@ class Dictionary(list[Table]):
         tables = list_as_bullets(elements=[str(t) for t in self], bullet="\n- ")
         return f"Dictionary(name={self.name!r}, imported={self.imported!r}, {tables})"
 
+    def __key(self, name: str) -> str:
+        return _normalise(name)
+
     def __getitem__(self, key: int | str) -> Table:
         if isinstance(key, int):
             return super().__getitem__(key)
-        target = _normalise_name(key)
-        found = next((t for t in self if t.name == target), None)
+
+        target_key = self.__key(key)
+        found = next((t for t in self if self.__key(t.name) == target_key), None)
         if not found:
             raise KeyError(f"Table {key!r} not found in Dictionary.")
         return found
 
-    # Getters
     def __get(self, name: str, default: Table | None = None) -> Table | None:
-        target = _normalise_name(name)
-        return next((t for t in self if t.name == target), default)
+        target_key = self.__key(name)
+        return next((t for t in self if self.__key(t.name) == target_key), default)
+
+    # Checkers
+    def __check_variables(self) -> None:
+        self.__check_name()
+        self.__check_organisations()
+        self.__check_version()
+        self.__check_version_notes()
+        self.__check_criteria()
+
+    def __check_name(self) -> None:
+        # Check name
+        if self.name is not None:
+            if not isinstance(self.name, str):
+                raise DataDictionaryError("Dictionary `name` must be a string.")
+
+    def __check_organisations(self) -> None:
+        # Check organisations
+        if self.organisations is not None:
+            if not isinstance(self.organisations, str):
+                raise DataDictionaryError(
+                    "Dictionary `organisations` must be a string."
+                )
+
+    def __check_version(self) -> None:
+        # Check version
+        if self.version is not None:
+            if not isinstance(self.version, (str, int, float)):
+                raise DataDictionaryError(
+                    "Dictionary `version` must be a string, int, or float."
+                )
+
+            if isinstance(self.version, (int, float)):
+                self.version = str(self.version)
+
+        # Check version_notes
 
+    def __check_version_notes(self) -> None:
+        if self.version_notes is not None:
+            if not isinstance(self.version_notes, str):
+                raise DataDictionaryError(
+                    "Dictionary `version_notes` must be a string."
+                )
+
+    def __check_criteria(self) -> None:
+        # Check inclusion_criteria
+        if self.inclusion_criteria is not None:
+            if not isinstance(self.inclusion_criteria, str):
+                raise DataDictionaryError(
+                    "Dictionary `inclusion_criteria` must be a string."
+                )
+
+        # Check exclusion_criteria
+        if self.exclusion_criteria is not None:
+            if not isinstance(self.exclusion_criteria, str):
+                raise DataDictionaryError(
+                    "Dictionary exclusion_criteria must be a string."
+                )
+
+    # Getters
     def index_of(self, name: str) -> int | None:
-        target = _normalise_name(name)
+        target_key = self.__key(name)
         for i, t in enumerate(self):
-            if t.name == target:
+            if self.__key(t.name) == target_key:
                 return i
         return None
 
@@ -484,12 +553,9 @@ class Dictionary(list[Table]):
         Raises:
             KeyError: If the table is not found in the dictionary.
         """
-        target = _normalise_name(table)
-        found = next((t for t in self if t.name == target), None)
-
-        if not found:
+        found = self.__get(table)
+        if found is None:
             raise KeyError(f"Table {table!r} not found in Dictionary.")
-
         return found
 
     # Manipulation
@@ -508,8 +574,14 @@ class Dictionary(list[Table]):
             raise DataDictionaryError(
                 "Only Table objects can be added to a Dictionary."
             )
-        if table.name in self.get_table_names():
-            raise DataDictionaryError(f"Table {table.name!r} already exists.")
+
+        incoming_key = self.__key(table.name)
+        conflict = next((t for t in self if self.__key(t.name) == incoming_key), None)
+        if conflict is not None:
+            raise DataDictionaryError(
+                f"Table {table.name!r} already exists (as {conflict.name!r})."
+            )
+
         super().append(table)
 
     def remove_table(self, table: str) -> None:

valediction/integrity.py

@@ -1,6 +1,10 @@
+from __future__ import annotations
+
 import re
+from copy import deepcopy
 from pathlib import Path
 from re import Pattern
+from typing import Any
 
 from valediction.data_types.data_types import DataType
 from valediction.support import list_as_bullets
@@ -12,13 +16,58 @@ TEMPLATE_DATA_DICTIONARY_PATH = (
 )
 
 
+externally_injected_variables: dict[
+    str, Any
+] = {}  # External injection store for package wrapping (any keys, always included)
+
+
+def reset_injected_config_variables() -> None:
+    global externally_injected_variables
+    externally_injected_variables = {}
+
+
+def inject_config_variables(variables: dict[str, Any]) -> None:
+    """Injects variables into the Valediction Config, which will always be incorporated
+    as overrides, regardless of Config calling method (default, session-scoped, or
+    contextual).
+
+    Args:
+        variables (dict[str, Any]): Dictionary of config variables.
+    """
+    global externally_injected_variables, session_config
+
+    # check type allows
+    if not isinstance(variables, dict):
+        raise TypeError(
+            f"Config injection variables must be a dictionary, not {type(variables)}"
+        )
+    problematic_keys = []
+    for variable_name in variables.keys():
+        if not isinstance(variable_name, str):
+            problematic_keys.append(variable_name)
+
+    if problematic_keys:
+        raise TypeError("Config injection variables accepts only string keys.")
+
+    externally_injected_variables = dict(variables or {})
+
+    # Apply immediately to the current session config (if it exists)
+    if session_config is not None:
+        _apply_external_injections(session_config)
+
+
+def _apply_external_injections(config: Config) -> None:
+    for variable_name, variable_value in externally_injected_variables.items():
+        setattr(config, variable_name, deepcopy(variable_value))
+
+
 class Config:
     def __init__(self):
         self.template_data_dictionary_path: Path = TEMPLATE_DATA_DICTIONARY_PATH
         self.max_table_name_length: int = 63
         self.max_column_name_length: int = 30
         self.max_primary_keys: int = 7
-        self.invalid_name_pattern: str | Pattern = re.compile(r"[^A-Z0-9_]")
+        self.invalid_name_pattern: str | Pattern = re.compile(r"[^A-Za-z0-9_]")
         self.null_values: list[str] = ["", "null", "none"]
         self.forbidden_characters: list[str] = []
         self.date_formats: dict[str, DataType] = {
@@ -28,20 +77,22 @@ class Config:
             "%d-%m-%Y": DataType.DATE,
             "%m/%d/%Y": DataType.DATE,
             "%m-%d-%Y": DataType.DATE,
-            "%Y-%m-%d %H:%M:%S": DataType.DATETIME,
-            "%Y-%m-%d %H:%M": DataType.DATETIME,
-            "%d/%m/%Y %H:%M:%S": DataType.DATETIME,
-            "%d/%m/%Y %H:%M": DataType.DATETIME,
-            "%m/%d/%Y %H:%M:%S": DataType.DATETIME,
-            "%Y-%m-%dT%H:%M:%S": DataType.DATETIME,
-            "%Y-%m-%dT%H:%M:%S.%f": DataType.DATETIME,
-            "%Y-%m-%dT%H:%M:%S%z": DataType.DATETIME,
-            "%Y-%m-%dT%H:%M:%S.%f%z": DataType.DATETIME,
-            "%Y-%m-%dT%H:%M:%SZ": DataType.DATETIME,
-            "%Y-%m-%dT%H:%M:%S.%fZ": DataType.DATETIME,
+            "%Y-%m-%d %H:%M:%S": DataType.TIMESTAMP,
+            "%Y-%m-%d %H:%M": DataType.TIMESTAMP,
+            "%d/%m/%Y %H:%M:%S": DataType.TIMESTAMP,
+            "%d/%m/%Y %H:%M": DataType.TIMESTAMP,
+            "%m/%d/%Y %H:%M:%S": DataType.TIMESTAMP,
+            "%Y-%m-%dT%H:%M:%S": DataType.TIMESTAMP,
+            "%Y-%m-%dT%H:%M:%S.%f": DataType.TIMESTAMP,
+            "%Y-%m-%dT%H:%M:%S%z": DataType.TIMESTAMP,
+            "%Y-%m-%dT%H:%M:%S.%f%z": DataType.TIMESTAMP,
+            "%Y-%m-%dT%H:%M:%SZ": DataType.TIMESTAMP,
+            "%Y-%m-%dT%H:%M:%S.%fZ": DataType.TIMESTAMP,
         }
         self.enforce_no_null_columns: bool = True
         self.enforce_primary_keys: bool = True
+        self.allow_bigint: bool = True
+        _apply_external_injections(self)
 
     def __repr__(self):
         date_list = list_as_bullets(
@@ -60,38 +111,43 @@ class Config:
             f" - default_null_values={self.null_values}\n"
             f" - forbidden_characters={self.forbidden_characters}\n"
             f" - date_formats=[{date_list}\n  ]\n"
+            f" - allow_bigint={self.allow_bigint}\n"
             ")"
         )
 
     # Context Wrapper With Reset
     def __enter__(self):
-        global default_config
-        default_config = self
+        global session_config
+
+        _apply_external_injections(self)
+
+        session_config = self
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
-        global default_config
-        default_config = Config()
+        global session_config
+        session_config = Config()
 
 
-default_config: Config = None
+session_config: Config = None
 
 
 def get_config() -> Config:
-    """Gets the current `default_config` instance. Changing attributes will set them
-    globally.
+    """Gets the current `session_config` instance. Changing attributes will set them
+    globally for the python session. Use `reset_default_config()` to reset to original
+    defaults.
 
     Returns:
-        Config: The current default configuration.
+        Config: The current session configuration.
     """
-    global default_config
-    return default_config
+    global session_config
+    return session_config
 
 
 def reset_default_config() -> None:
     """Resets `default_config` settings globally to original defaults."""
-    global default_config
-    default_config = Config()
+    global session_config
+    session_config = Config()
 
 
 reset_default_config()

valediction/io/csv_readers.py

@@ -11,7 +11,7 @@ import pandas as pd
 from pandas import DataFrame
 from pandas.errors import ParserError
 
-from valediction.support import _normalise_name
+from valediction.support import _strip
 
 
 class FrameChunk(NamedTuple):
@@ -34,7 +34,7 @@ class FrameChunk(NamedTuple):
     total_chunks_seen: int | None
 
     def estimate_chunk_count(self) -> int:
-        # Buffers (accounting for CSV tails/bytes innacuracy)
+        # Buffers (accounting for CSV tails/bytes inaccuracy)
         EPS_ABS = 4096  # Fixed
         EPS_REL = 0.05  # 5% tail buffer
 
@@ -93,7 +93,7 @@ def _post_read_processing(df: DataFrame, cfg: CsvReadConfig) -> DataFrame:
     """Apply header normalisation and vectorised value stripping after reading."""
     cfg = cfg or CsvReadConfig()
     if cfg.normalise_headers:
-        df = df.rename(columns={c: _normalise_name(c) for c in df.columns})
+        df = df.rename(columns={c: _strip(c) for c in df.columns})
     if cfg.strip_values:
         str_cols = df.select_dtypes(include=["string"]).columns
         if len(str_cols) > 0:

valediction/support.py

@@ -35,10 +35,14 @@ def list_as_bullets(elements: list, bullet: str = "\n - ") -> str:
     return bullet + bullet.join(elements)
 
 
-def _normalise_name(name: str) -> str:
+def _normalise(name: str) -> str:
     return name.strip().upper()
 
 
+def _strip(name: str) -> str:
+    return name.strip()
+
+
 def _get_runtime_string(runtime: timedelta) -> str:
     total_seconds = runtime.total_seconds()
     hours = trunc(total_seconds / 3600)

valediction/validation/helpers.py

@@ -10,6 +10,7 @@ from pandas.util import hash_pandas_object
 from valediction.data_types.data_types import DataType
 from valediction.dictionary.model import Table
 from valediction.integrity import get_config
+from valediction.support import _normalise
 from valediction.validation.issues import Range
 
 
@@ -17,11 +18,14 @@ from valediction.validation.issues import Range
 def _set_nulls(df: DataFrame) -> DataFrame:
     null_values = get_config().null_values
     token_set = {str(t).strip().casefold() for t in null_values}
-    columns = df.select_dtypes(include=["string", "object"]).columns
+    columns = df.select_dtypes(include=["string", "object", "category"]).columns
     for column in columns:
         series = df[column]
-        mask = series.notna() & series.str.casefold().isin(token_set)
-        df[column] = series.mask(mask, NA)
+
+        s_txt = series.astype("string", copy=False)  # dtype safe
+        mask = s_txt.notna() & s_txt.str.strip().str.casefold().isin(token_set)
+        if mask.any():
+            df[column] = series.mask(mask, NA)
 
     return df
 
@@ -68,37 +72,24 @@ def create_pk_hashes(
     Returns:
         Series: Pandas Series with hashes or Nulls.
     """
-    hash_col_name = "PK_HASH"
+    HASH_COL_NAME = "PK_HASH"
     if df_primaries.empty or df_primaries.shape[1] == 0:
-        return Series([], dtype=object, name=hash_col_name)
+        return Series([], dtype=object, name=HASH_COL_NAME)
 
-    # Any NA in row => invalid PK -> None
+    # Check Nulls
     null_rows = df_primaries.isna().any(axis=1)
 
-    # First Hash
-    hash_1 = hash_pandas_object(df_primaries, index=False)  # uint64
-
-    # Second Hash (rows backwards if single row, else salt)
-    if df_primaries.shape[1] > 1:
-        df_primaries_backwards = df_primaries.iloc[:, ::-1]
-    else:
-        s = df_primaries.iloc[:, 0]
-        salt = Series(["§"] * len(s), index=s.index, dtype="string")
-        df_primaries_backwards = DataFrame(
-            {
-                "_a": s,
-                "_b": s.str.cat(salt),
-            }
-        )
-
-    hash_2 = hash_pandas_object(df_primaries_backwards, index=False)  # uint64
+    # Two independent 64-bit hashes with 16 byte keys
+    hash_1 = hash_pandas_object(df_primaries, index=False, hash_key="valediction_pk1!")
+    hash_2 = hash_pandas_object(df_primaries, index=False, hash_key="valediction_pk2!")
 
+    # Combine into 128-bit integer keys
     a1 = hash_1.to_numpy(dtype="uint64", copy=False).astype(object)
     a2 = hash_2.to_numpy(dtype="uint64", copy=False).astype(object)
-
     combined = (a1 << 64) | a2
+
     hashes = Series(
-        combined, index=df_primaries.index, name=hash_col_name, dtype=object
+        combined, index=df_primaries.index, name=HASH_COL_NAME, dtype=object
     )
     hashes[null_rows] = None
     return hashes
@@ -167,8 +158,9 @@ def pk_contains_whitespace_mask(df_primaries: DataFrame) -> Series:
     if df_primaries.empty or df_primaries.shape[1] == 0:
         return Series(False, index=df_primaries.index)
 
-    col_masks = df_primaries.apply(lambda s: s.str.contains(r"\s", na=False))
-
+    col_masks = df_primaries.apply(
+        lambda s: s.astype("string", copy=False).str.contains(r"\s", na=False)
+    )
     return col_masks.any(axis=1)
 
 
@@ -249,7 +241,7 @@ def invalid_mask_datetime(column: Series, fmt: str | None) -> Series:
         ok = parsed.notna()
         return notnull & (~ok)
 
-    allowed = _allowed_formats_for(DataType.DATETIME)
+    allowed = _allowed_formats_for(DataType.TIMESTAMP)
     ok_any = _parse_ok_any(column, allowed)
     return notnull & (~ok_any)
 
@@ -261,7 +253,9 @@ def invalid_mask_text_too_long(column: Series, max_len: int) -> Series:
         return Series(False, index=column.index)
 
     notnull = column.notna()
-    lens = column.str.len()
+    s_txt = column.astype("string", copy=False)
+    lens = s_txt.str.len()
+
     return notnull & (lens > max_len)
 
 
@@ -270,20 +264,23 @@ def invalid_mask_text_forbidden_characters(column: Series) -> Series:
     if not forbidden:
         return column.notna() & False
 
-    pattern = "[" + re.escape("".join(forbidden)) + "]"
+    pattern = "[" + re.escape("".join([str(s) for s in forbidden])) + "]"
     notnull = column.notna()
-    has_forbidden = column.str.contains(pattern, regex=True, na=False)
+
+    s_txt = column.astype("string", copy=False)
+    has_forbidden = s_txt.str.contains(pattern, regex=True, na=False)
+
     return notnull & has_forbidden
 
 
 # Apply Data Types #
 def apply_data_types(df: DataFrame, table_dictionary: Table) -> DataFrame:
     # name -> column object
-    column_dictionary = {column.name: column for column in table_dictionary}
+    column_dictionary = {_normalise(column.name): column for column in table_dictionary}
 
     for col in df.columns:
-        data_type = column_dictionary.get(col).data_type
-        datetime_format = column_dictionary.get(col).datetime_format
+        data_type = column_dictionary.get(_normalise(col)).data_type
+        datetime_format = column_dictionary.get(_normalise(col)).datetime_format
 
         if data_type in (DataType.TEXT, DataType.FILE):
             df[col] = df[col].astype("string")
@@ -303,7 +300,7 @@ def apply_data_types(df: DataFrame, table_dictionary: Table) -> DataFrame:
             )
             df[col] = dtv.dt.normalize()  # midnight
 
-        elif data_type == DataType.DATETIME:
+        elif data_type == DataType.TIMESTAMP:
             df[col] = to_datetime(
                 df[col], format=datetime_format, errors="raise", utc=False
             )
@@ -313,3 +310,62 @@ def apply_data_types(df: DataFrame, table_dictionary: Table) -> DataFrame:
             df[col] = df[col].astype("string")
 
     return df
+
+
+# Bigint Checks
+_PG_INT4_MIN_STR_ABS = "2147483648"  # abs(-2147483648)
+_PG_INT4_MAX_STR_ABS = "2147483647"
+_PG_INT4_MIN_LEN = len(_PG_INT4_MIN_STR_ABS)
+_PG_INT4_MAX_LEN = len(_PG_INT4_MAX_STR_ABS)
+
+
+def invalid_mask_integer_out_of_range(
+    series: Series,
+    invalid_integer_mask: Series | None = None,
+) -> Series:
+    """
+    Returns a boolean mask for values that:
+      - are integer-like under Valediction's integer rules, AND
+      - fall outside PostgreSQL INTEGER (int4) range.
+    """
+
+    # Start with all-False mask
+    out = series.isna() & False
+
+    # Use caller-provided invalid mask to avoid recomputing if available
+    if invalid_integer_mask is None:
+        from valediction.validation.helpers import invalid_mask_integer  # avoid cycles
+
+        invalid_integer_mask = invalid_mask_integer(series)
+
+    # We only check range for values that already pass integer validation
+    valid = (~invalid_integer_mask) & series.notna()
+    if not valid.any():
+        return out
+
+    # String-normalise for safe compare (works for object/int dtype)
+    s = series[valid].astype("string", copy=False).str.strip()
+
+    # Sign handling
+    neg = s.str.startswith("-")
+    abs_str = s.str.lstrip("+-")
+
+    # Lengths
+    abs_len = abs_str.str.len()
+
+    # Positive overflow:
+    #   abs_len > 10 OR (abs_len == 10 AND abs_str > 2147483647)
+    pos = ~neg
+    pos_over = (abs_len > _PG_INT4_MAX_LEN) | (
+        (abs_len == _PG_INT4_MAX_LEN) & (abs_str > _PG_INT4_MAX_STR_ABS)
+    )
+
+    # Negative overflow (too small):
+    #   abs_len > 10 OR (abs_len == 10 AND abs_str > 2147483648)
+    neg_over = (abs_len > _PG_INT4_MIN_LEN) | (
+        (abs_len == _PG_INT4_MIN_LEN) & (abs_str > _PG_INT4_MIN_STR_ABS)
+    )
+
+    # Combine back into the full index
+    out.loc[valid] = (pos & pos_over) | (neg & neg_over)
+    return out