valediction 1.1.0__py3-none-any.whl → 1.5.0__py3-none-any.whl

valediction/data_types/data_type_helpers.py

@@ -62,14 +62,14 @@ def infer_datetime_format(
 
 
 def get_date_type(datetime_format: str) -> DataType | None:
-    """Identifies if a datetime format string corresponds to a Date or Datetime data
+    """Identifies if a datetime format string corresponds to a Date or Timestamp data
     type.
 
     Args:
         datetime_format (str): datetime format string
 
     Returns:
-        DataType | None: DataType of Date, Datetime, or None if not found.
+        DataType | None: DataType of Date, Timestamp, or None if not found.
     """
     config = get_config()
     return config.date_formats.get(datetime_format)

valediction/data_types/data_types.py

@@ -8,7 +8,7 @@ class DataType(Enum):
     INTEGER = "Integer"
     FLOAT = "Float"
     DATE = "Date"
-    DATETIME = "Datetime"
+    TIMESTAMP = "Timestamp"
     FILE = "File"
 
     def __str__(self) -> str:
@@ -32,9 +32,9 @@ class DataType(Enum):
             "number": cls.FLOAT,
             "numeric": cls.FLOAT,
             "date": cls.DATE,
-            "datetime": cls.DATETIME,
-            "datetime64": cls.DATETIME,
-            "timestamp": cls.DATETIME,
+            "datetime": cls.TIMESTAMP,
+            "datetime64": cls.TIMESTAMP,
+            "timestamp": cls.TIMESTAMP,
             "file": cls.FILE,
             "blob": cls.FILE,
             "binary": cls.FILE,
@@ -49,10 +49,10 @@ class DataType(Enum):
         return self in {DataType.TEXT}
 
     def valid_for_primary_key(self) -> bool:
-        """PKs can only be Text, Integer, Date, Datetime."""
+        """PKs can only be Text, Integer, Date, Timestamp."""
         return self in {
             DataType.TEXT,
             DataType.INTEGER,
             DataType.DATE,
-            DataType.DATETIME,
+            DataType.TIMESTAMP,
         }

valediction/data_types/type_inference.py

@@ -4,6 +4,7 @@ import re
 import warnings
 
 import pandas as pd
+from pandas.api.types import is_object_dtype, is_string_dtype
 
 from valediction.data_types.data_type_helpers import infer_datetime_format
 from valediction.data_types.data_types import DataType
@@ -53,8 +54,8 @@ class ColumnState:
             return DataType.FLOAT, None
         if self.data_type == DataType.DATE:
             return DataType.DATE, None
-        if self.data_type == DataType.DATETIME:
-            return DataType.DATETIME, None
+        if self.data_type == DataType.TIMESTAMP:
+            return DataType.TIMESTAMP, None
 
         return DataType.TEXT, _len1()
 
@@ -123,7 +124,7 @@ class TypeInferer:
             _handling_function: callable = {
                 DataType.TEXT: self._handle_state_text,
                 DataType.DATE: self._handle_state_date,
-                DataType.DATETIME: self._handle_state_datetime,
+                DataType.TIMESTAMP: self._handle_state_datetime,
                 DataType.INTEGER: self._handle_state_integer,
                 DataType.FLOAT: self._handle_state_float,
             }.get(state.data_type, self._handle_state_text)
@@ -141,20 +142,31 @@ class TypeInferer:
         self, s: pd.Series
     ) -> tuple[pd.Series, pd.Series, pd.Series, int | None]:
         self.__begin_step(step="Trimming whitespace")
-        trimmed = s.str.strip()
+        is_text = is_string_dtype(s) or is_object_dtype(s)
+
+        if is_text:
+            trimmed = s.astype("string").str.strip()
+        else:
+            trimmed = s
         self.__complete_step()
 
         self.__begin_step(step="Checking nulls")
-        nulls = trimmed.isna() | trimmed.str.lower().isin(self.null_tokens)
+        if is_text:
+            nulls = trimmed.isna() | trimmed.str.lower().isin(self.null_tokens)
+        else:
+            nulls = trimmed.isna()
         self.__complete_step()
 
         self.__begin_step(step="Checking max length")
-        lengths = s.str.len()
-        max_len = int(lengths.max(skipna=True)) if lengths.notna().any() else None
+        if is_text:
+            lengths = trimmed.str.len()
+            max_len = int(lengths.max(skipna=True)) if lengths.notna().any() else None
+        else:
+            max_len = None
         self.__complete_step()
 
         self.__begin_step(step="Setting non-null mask")
-        nonnull_mask = (~nulls) & s.notna()
+        nonnull_mask = (~nulls) & trimmed.notna()
         self.__complete_step()
 
         return trimmed, nulls, nonnull_mask, max_len
@@ -193,7 +205,7 @@ class TypeInferer:
             if ok.all():
                 self._transition(
                     st,
-                    DataType.DATETIME if has_time.any() else DataType.DATE,
+                    DataType.TIMESTAMP if has_time.any() else DataType.DATE,
                     f"cached datetime format={st.cached_datetime_format!r}",
                 )
                 self.__complete_step()
@@ -210,7 +222,7 @@ class TypeInferer:
                     st.cached_datetime_format = fmt
                     self._transition(
                         st,
-                        DataType.DATETIME if has_time.any() else DataType.DATE,
+                        DataType.TIMESTAMP if has_time.any() else DataType.DATE,
                         f"explicit datetime format={fmt!r}",
                     )
                     self.__complete_step()
@@ -276,7 +288,7 @@ class TypeInferer:
             st.lock_text_permanent = True
             self._transition(st, DataType.TEXT, "datetime parse failures")
         elif has_time.any():
-            self._transition(st, DataType.DATETIME, "time component detected")
+            self._transition(st, DataType.TIMESTAMP, "time component detected")
 
         self.__complete_step()
 
@@ -334,7 +346,7 @@ class TypeInferer:
             if ok.all():
                 self._transition(
                     st,
-                    DataType.DATETIME if has_time.any() else DataType.DATE,
+                    DataType.TIMESTAMP if has_time.any() else DataType.DATE,
                     f"cached datetime format={st.cached_datetime_format!r}",
                 )
                 return True
@@ -377,7 +389,7 @@ class TypeInferer:
         if ok.all():
             self._transition(
                 st,
-                DataType.DATETIME if has_time.any() else DataType.DATE,
+                DataType.TIMESTAMP if has_time.any() else DataType.DATE,
                 f"explicit datetime format={st.cached_datetime_format!r}",
             )
             return True

valediction/datasets/datasets.py

@@ -20,7 +20,8 @@ from valediction.io.csv_readers import (
 )
 from valediction.support import (
     _get_runtime_string,
-    _normalise_name,
+    _normalise,
+    _strip,
     list_as_bullets,
     print_bold_red,
     print_red,
@@ -437,16 +438,16 @@ class Dataset(list[DatasetItem]):
 
     # Getters
     def get(self, name: str, default: DatasetItem | None = None) -> DatasetItem | None:
-        name_key = name.strip()
+        name_key = _normalise(name)
         for item in self:
-            if item.name.lower() == name_key.lower():
+            if _normalise(item.name) == name_key:
                 return item
         return default
 
     def index_of(self, name: str) -> int | None:
-        name_key = name.strip()
+        name_key = _normalise(name)
         for i, item in enumerate(self):
-            if item.name == name_key:
+            if _normalise(item.name) == name_key:
                 return i
         return None
 
@@ -796,20 +797,21 @@ class Dataset(list[DatasetItem]):
         name: str | None,
         data: DataLike,
     ) -> DatasetItem:
-        """Normalise a (data, name) double into a DatasetItem."""
+        """Normalise a (name, data) double into a DatasetItem."""
         if isinstance(data, (str, Path)):
             path = Path(data)
             if not path.exists():
                 raise FileNotFoundError(f"File not found: {path}")
             if path.suffix.lower() != ".csv":
                 raise ValueError(f"Only .csv supported right now, got: {path}")
-            resolved_name = _normalise_name(name or path.stem)
+            resolved_name = _strip(name or path.stem)
             return DatasetItem(name=resolved_name, data=path.resolve())
 
         if isinstance(data, DataFrame):
             if not name:
                 raise ValueError("When providing a DataFrame, 'name' is required.")
-            resolved_name = _normalise_name(name)
+            resolved_name = _strip(name)
+            data.columns = [_strip(column) for column in data.columns]
             return DatasetItem(name=resolved_name, data=data)
 
         raise TypeError("data must be a Path/str to .csv or a pandas DataFrame.")
@@ -823,13 +825,11 @@ class Dataset(list[DatasetItem]):
         if p.is_file():
             if p.suffix.lower() != ".csv":
                 raise ValueError(f"Expected a .csv file, got: {p.suffix} ({p})")
-            return [DatasetItem(name=_normalise_name(p.stem), data=p.resolve())]
+            return [DatasetItem(name=_strip(p.stem), data=p.resolve())]
 
         if p.is_dir():
             return [
-                DatasetItem(
-                    name=_normalise_name(csv_path.stem), data=csv_path.resolve()
-                )
+                DatasetItem(name=_strip(csv_path.stem), data=csv_path.resolve())
                 for csv_path in p.glob("*.csv")
             ]
 

valediction/demo/demo_dictionary.py

@@ -103,7 +103,7 @@ def demo_dictionary() -> Dictionary:
                 foreign_key="DEMOGRAPHICS.PATIENT_HASH",
             ),
             Column(
-                name="OBSERVATION_TIME", order=2, data_type="datetime", primary_key=2
+                name="OBSERVATION_TIME", order=2, data_type="timestamp", primary_key=2
             ),
             Column(
                 name="OBSERVATION_TYPE",

valediction/dictionary/generation.py

@@ -24,7 +24,7 @@ from valediction.io.csv_readers import (
     read_csv_sample,
 )
 from valediction.progress import Progress
-from valediction.support import _normalise_name, calculate_runtime
+from valediction.support import _strip, calculate_runtime
 
 IMPORTING_DATA = "Importing data"
 CHUNK_STEPS = 1
@@ -124,7 +124,7 @@ class Generator:
         self.__say(f"Generating dictionary for {len(items)} tables")
         for item in items:
             self.__progress_init(item)
-            table = Table(name=_normalise_name(item.name))
+            table = Table(name=_strip(item.name))
             dictionary.add_table(table)
 
             if item.is_path:
@@ -192,7 +192,7 @@ class Generator:
                     col_state = inferer.states[col_name]
                     data_type, length = col_state.final_data_type_and_length()
                     col = Column(
-                        name=_normalise_name(col_name),
+                        name=_strip(col_name),
                         order=idx,
                         data_type=data_type,
                         length=length if data_type == DataType.TEXT else None,
@@ -242,7 +242,7 @@ class Generator:
             col_state = inferer.states[col_name]
             data_type, length = col_state.final_data_type_and_length()
             col = Column(
-                name=_normalise_name(col_name),
+                name=_strip(col_name),
                 order=idx,
                 data_type=data_type,
                 length=length if data_type == DataType.TEXT else None,
@@ -257,7 +257,7 @@ class Generator:
             table.add_column(col)
 
     def _set_datetime_format(self, column_state: ColumnState, column: Column) -> None:
-        if column.data_type in (DataType.DATE, DataType.DATETIME):
+        if column.data_type in (DataType.DATE, DataType.TIMESTAMP):
             datetime_format = getattr(column_state, "cached_datetime_format", None)
             if datetime_format and hasattr(column, "datetime_format"):
                 column.datetime_format = datetime_format
@@ -277,7 +277,7 @@ class Generator:
                 next_order = max((c.order or 0 for c in table), default=0) + 1
                 data_type, length = col_state.final_data_type_and_length()
                 new_col = Column(
-                    name=_normalise_name(col_name),
+                    name=_strip(col_name),
                     order=next_order,
                     data_type=data_type,
                     length=length if data_type == DataType.TEXT else None,

valediction/dictionary/helpers.py

@@ -26,9 +26,6 @@ def _check_name(name: str, entity: Literal["table", "column"]) -> list[str]:
         else config.max_column_name_length
     )
 
-    if name != name.upper():  # name must be uppercase
-        errors.append("must be uppercase")
-
     if invalid_chars.search(name):  # check invalid characters
         bad = set(invalid_chars.findall(name))
         errors.append(
@@ -109,16 +106,12 @@ def _check_primary_key(primary_key: int | None, data_type: DataType) -> list[str
     ):
         errors.append(
             f"invalid data type '{data_type.value}' for primary key column; "
-            "primary keys must be Text, Integer, Date, or Datetime"
+            "primary keys must be Text, Integer, Date, or Timestamp"
         )
 
     return errors
 
 
-def _normalise_name(name: str) -> str:
-    return name.upper().strip()
-
-
 def _norm_header_map(columns: list) -> dict:
     mapping, _ = {}, set()
     for c in columns:

valediction/dictionary/importing.py

@@ -11,7 +11,6 @@ from valediction.dictionary.helpers import (
     _get_required_header,
     _is_missing,
     _norm_header_map,
-    _normalise_name,
     _parse_int,
     _parse_truthy,
     _row_is_blank,
@@ -19,7 +18,7 @@ from valediction.dictionary.helpers import (
 from valediction.dictionary.integrity import REQUIRED_SHEETS
 from valediction.dictionary.model import Column, Dictionary, Table
 from valediction.exceptions import DataDictionaryError, DataDictionaryImportError
-from valediction.support import list_as_bullets
+from valediction.support import _normalise, _strip, list_as_bullets
 
 
 @dataclass
@@ -80,6 +79,13 @@ class ExcelDataDictionary:
             raise error
 
     # Import & Helpers
+    def _resolve_table_name(self, name: str) -> str | None:
+        """Return the canonical table name as it appears in Tables sheet (or None)."""
+        target = _normalise(name)
+        return next(
+            (t for t in self.table_metadata.keys() if _normalise(t) == target), None
+        )
+
     def _open_workbook(self) -> None:
         if not self.path.exists():
             raise DataDictionaryImportError(f"File not found: {self.path}")
@@ -140,20 +146,27 @@ class ExcelDataDictionary:
         description_col_header = _get_required_header(header_map, "description")
 
         meta: dict[str, str | None] = {}
+        seen: set[str] = set()
+
         for _, row in tables_df.iterrows():
             if _is_missing(row[table_col_header]):
                 continue
-            table_name = _normalise_name(str(row[table_col_header]))
+
+            table_name = _strip(str(row[table_col_header]))
             table_description = (
                 None
                 if _is_missing(row[description_col_header])
                 else str(row[description_col_header])
             )
-            if table_name in meta:
+
+            key = _normalise(table_name)
+            if key in seen:
                 raise DataDictionaryImportError(
                     f"Duplicate table '{table_name}' in Tables sheet."
                 )
+            seen.add(key)
             meta[table_name] = table_description
+
         if not meta:
             raise DataDictionaryImportError(
                 "Data Dictionary sheet 'Tables' contains no table rows."
@@ -177,12 +190,13 @@ class ExcelDataDictionary:
                 or _is_missing(row[code_col_header])
             ):
                 continue
-            table_name = _normalise_name(str(row[table_col_header]))
-            column_name = _normalise_name(str(row[column_col_header]))
-            enum_map.setdefault((table_name, column_name), {})
-            enum_map[(table_name, column_name)][row[code_col_header]] = row[
-                name_col_header
-            ]
+            table_name = _strip(str(row[table_col_header]))
+            column_name = _strip(str(row[column_col_header]))
+            resolved_table = self._resolve_table_name(table_name) or table_name
+            enum_key = (_normalise(resolved_table), _normalise(column_name))
+            enum_map.setdefault(enum_key, {})
+            enum_map[enum_key][row[code_col_header]] = row[name_col_header]
+
         self.enumerations = enum_map
 
     # Parse Columns
@@ -234,7 +248,12 @@ class ExcelDataDictionary:
 
             self.table_columns[inputs.table_name].append(column_obj)
             if inputs.has_enumerations:
-                self.enum_flags.add((inputs.table_name, inputs.column_name))
+                self.enum_flags.add(
+                    (
+                        _normalise(inputs.table_name),
+                        _normalise(inputs.column_name),
+                    )
+                )
 
         if errors:
             raise DataDictionaryImportError(
@@ -279,7 +298,7 @@ class ExcelDataDictionary:
 
     # Validate Foreign Keys
     def _validate_foreign_keys(self) -> None:
-        name_to_table = {t.name: t for t in self.tables}
+        name_to_table = {_normalise(t.name): t for t in self.tables}
         errors: list[str] = []
         for table in self.tables:
             for column in table:
@@ -292,9 +311,9 @@ class ExcelDataDictionary:
                     )
                     continue
                 target_table_raw, target_column_raw = target.split(".", 1)
-                target_table_name = _normalise_name(target_table_raw)
-                target_column_name = _normalise_name(target_column_raw)
-                referenced_table = name_to_table.get(target_table_name)
+                target_table_name = _strip(target_table_raw)
+                target_column_name = _strip(target_column_raw)
+                referenced_table = name_to_table.get(_normalise(target_table_name))
                 if not referenced_table:
                     errors.append(
                         f"{table.name}.{column.name} references unknown table {target_table_name!r}."
@@ -345,7 +364,7 @@ class ExcelDataDictionary:
         enumeration_flag_col_header = header_map.get("enumerations")
         primary_key_col_header = header_map.get("primary_key")
         foreign_key_col_header = header_map.get("foreign_key_target")
-        description_col_header = header_map.get("description")
+        description_col_header = header_map.get("column_description")
         return (
             table_col_header,
             column_col_header,
@@ -392,13 +411,17 @@ class ExcelDataDictionary:
                 f"{row_context}: missing required field(s): {', '.join(missing_fields)}."
             )
 
-        table_name = _normalise_name(str(row[table_col_header]))
-        column_name = _normalise_name(str(row[column_col_header]))
-        if table_name not in self.table_metadata:
+        table_name_raw = _strip(str(row[table_col_header]))
+        column_name = _strip(str(row[column_col_header]))
+
+        resolved_table_name = self._resolve_table_name(table_name_raw)
+        if resolved_table_name is None:
             raise DataDictionaryImportError(
-                f"{row_context}: Table '{table_name}' not present in Tables sheet."
+                f"{row_context}: Table '{table_name_raw}' not present in Tables sheet."
             )
 
+        table_name = resolved_table_name
+
         order_int = _parse_int(row[order_col_header], "Order", row_context)
         length_int = (
             _parse_int(row[length_col_header], "Length", row_context, required=False)
@@ -461,7 +484,7 @@ class ExcelDataDictionary:
 
     def _make_column(self, inputs: _ColumnInputs) -> Column:
         enums_for_column = self.enumerations.get(
-            (inputs.table_name, inputs.column_name), {}
+            (_normalise(inputs.table_name), _normalise(inputs.column_name)), {}
         )
         return Column(
             name=inputs.column_name,