valediction 1.0.3__tar.gz → 1.2.0__tar.gz

{valediction-1.0.3 → valediction-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: valediction
-Version: 1.0.3
+Version: 1.2.0
 Summary: Valediction is a convenience data validation package that allows generation, import, and constraint enforcement of user-defined data dictionaries against datasets.
 Author-email: Cai Davis <Cai.Davis@uhs.nhs.uk>
 Requires-Python: <4.0,>=3.11

{valediction-1.0.3 → valediction-1.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "valediction"
-version = "1.0.3"
+version = "1.2.0"
 description = "Valediction is a convenience data validation package that allows generation, import, and constraint enforcement of user-defined data dictionaries against datasets."
 authors = [{ name = "Cai Davis", email = "Cai.Davis@uhs.nhs.uk" }]
 requires-python = ">=3.11,<4.0"

{valediction-1.0.3 → valediction-1.2.0}/src/valediction/convenience.py

@@ -3,12 +3,11 @@ from pathlib import Path
 from pandas import DataFrame
 
 from valediction.datasets.datasets import Dataset
-from valediction.dictionary.importing import import_dictionary
 from valediction.dictionary.model import Dictionary
 
 
 def validate(
-    data: str | Path | dict[str, DataFrame] | Dictionary,
+    dataset: str | Path | dict[str, DataFrame],
     dictionary: Dictionary | str | Path,
     *,
     import_data: bool = False,
@@ -31,20 +30,16 @@ def validate(
     Returns:
         Dataset: dataset, with or without Issues
     """
-    dictionary = (
-        dictionary
-        if isinstance(dictionary, Dictionary)
-        else import_dictionary(dictionary)
-    )
-    data: Dataset = Dataset.create_from(data)
-    data.import_dictionary(dictionary)
+
+    _dataset: Dataset = Dataset.create_from(dataset)
+    _dataset.import_dictionary(dictionary)
 
     if import_data:
-        data.import_data()
+        _dataset.import_data()
 
-    data.validate(
+    _dataset.validate(
         chunk_size=chunk_size,
         feedback=feedback,
     )
 
-    return data
+    return _dataset

{valediction-1.0.3 → valediction-1.2.0}/src/valediction/datasets/datasets.py

@@ -20,7 +20,8 @@ from valediction.io.csv_readers import (
 )
 from valediction.support import (
     _get_runtime_string,
-    _normalise_name,
+    _normalise,
+    _strip,
     list_as_bullets,
     print_bold_red,
     print_red,
@@ -135,16 +136,16 @@ class DatasetItem:
     # Validation
     def validate(
         self,
-        chunk_size: int = None,
+        chunk_size: int | None = 10_000_000,
         feedback: bool = True,
-    ):
+    ) -> None:
         """
         Summary:
             Validates the dataset item against the dictionary.
             Warns if there are issues with the integrity of the data.
 
         Arguments:
-            chunk_size (int): Size of chunks for validating data to optimise RAM usage,
+            chunk_size (int | None): Size of chunks for validating data to optimise RAM usage,
                 if reading from CSV (default: 10_000_000)
             feedback (bool): Provide user feedback on progress (default: True)
 
@@ -437,16 +438,16 @@ class Dataset(list[DatasetItem]):
 
     # Getters
     def get(self, name: str, default: DatasetItem | None = None) -> DatasetItem | None:
-        name_key = name.strip()
+        name_key = _normalise(name)
         for item in self:
-            if item.name.lower() == name_key.lower():
+            if _normalise(item.name) == name_key:
                 return item
         return default
 
     def index_of(self, name: str) -> int | None:
-        name_key = name.strip()
+        name_key = _normalise(name)
         for i, item in enumerate(self):
-            if item.name == name_key:
+            if _normalise(item.name) == name_key:
                 return i
         return None
 
@@ -710,7 +711,7 @@ class Dataset(list[DatasetItem]):
     # Validation
     def validate(
         self,
-        chunk_size: int = None,
+        chunk_size: int | None = 10_000_000,
         feedback: bool = True,
     ) -> None:
         """
@@ -751,7 +752,7 @@ class Dataset(list[DatasetItem]):
         if feedback:
             print("\n", end="")
 
-    def __reattach_issues(self):
+    def __reattach_issues(self) -> None:
         self.issues = Issues()
         for item in self:
             self.issues.extend(item.issues)
@@ -796,20 +797,21 @@ class Dataset(list[DatasetItem]):
         name: str | None,
         data: DataLike,
     ) -> DatasetItem:
-        """Normalise a (data, name) double into a DatasetItem."""
+        """Normalise a (name, data) double into a DatasetItem."""
         if isinstance(data, (str, Path)):
             path = Path(data)
             if not path.exists():
                 raise FileNotFoundError(f"File not found: {path}")
             if path.suffix.lower() != ".csv":
                 raise ValueError(f"Only .csv supported right now, got: {path}")
-            resolved_name = _normalise_name(name or path.stem)
+            resolved_name = _strip(name or path.stem)
             return DatasetItem(name=resolved_name, data=path.resolve())
 
         if isinstance(data, DataFrame):
             if not name:
                 raise ValueError("When providing a DataFrame, 'name' is required.")
-            resolved_name = _normalise_name(name)
+            resolved_name = _strip(name)
+            data.columns = [_strip(column) for column in data.columns]
             return DatasetItem(name=resolved_name, data=data)
 
         raise TypeError("data must be a Path/str to .csv or a pandas DataFrame.")
@@ -823,13 +825,11 @@ class Dataset(list[DatasetItem]):
         if p.is_file():
             if p.suffix.lower() != ".csv":
                 raise ValueError(f"Expected a .csv file, got: {p.suffix} ({p})")
-            return [DatasetItem(name=_normalise_name(p.stem), data=p.resolve())]
+            return [DatasetItem(name=_strip(p.stem), data=p.resolve())]
 
         if p.is_dir():
             return [
-                DatasetItem(
-                    name=_normalise_name(csv_path.stem), data=csv_path.resolve()
-                )
+                DatasetItem(name=_strip(csv_path.stem), data=csv_path.resolve())
                 for csv_path in p.glob("*.csv")
             ]
 

{valediction-1.0.3 → valediction-1.2.0}/src/valediction/dictionary/generation.py

@@ -24,7 +24,7 @@ from valediction.io.csv_readers import (
     read_csv_sample,
 )
 from valediction.progress import Progress
-from valediction.support import _normalise_name, calculate_runtime
+from valediction.support import _strip, calculate_runtime
 
 IMPORTING_DATA = "Importing data"
 CHUNK_STEPS = 1
@@ -124,7 +124,7 @@ class Generator:
         self.__say(f"Generating dictionary for {len(items)} tables")
         for item in items:
             self.__progress_init(item)
-            table = Table(name=_normalise_name(item.name))
+            table = Table(name=_strip(item.name))
             dictionary.add_table(table)
 
             if item.is_path:
@@ -192,7 +192,7 @@ class Generator:
                     col_state = inferer.states[col_name]
                     data_type, length = col_state.final_data_type_and_length()
                     col = Column(
-                        name=_normalise_name(col_name),
+                        name=_strip(col_name),
                         order=idx,
                         data_type=data_type,
                         length=length if data_type == DataType.TEXT else None,
@@ -242,7 +242,7 @@ class Generator:
             col_state = inferer.states[col_name]
             data_type, length = col_state.final_data_type_and_length()
             col = Column(
-                name=_normalise_name(col_name),
+                name=_strip(col_name),
                 order=idx,
                 data_type=data_type,
                 length=length if data_type == DataType.TEXT else None,
@@ -277,7 +277,7 @@ class Generator:
                 next_order = max((c.order or 0 for c in table), default=0) + 1
                 data_type, length = col_state.final_data_type_and_length()
                 new_col = Column(
-                    name=_normalise_name(col_name),
+                    name=_strip(col_name),
                     order=next_order,
                     data_type=data_type,
                     length=length if data_type == DataType.TEXT else None,

{valediction-1.0.3 → valediction-1.2.0}/src/valediction/dictionary/helpers.py

@@ -26,9 +26,6 @@ def _check_name(name: str, entity: Literal["table", "column"]) -> list[str]:
         else config.max_column_name_length
     )
 
-    if name != name.upper():  # name must be uppercase
-        errors.append("must be uppercase")
-
     if invalid_chars.search(name):  # check invalid characters
         bad = set(invalid_chars.findall(name))
         errors.append(
@@ -115,10 +112,6 @@ def _check_primary_key(primary_key: int | None, data_type: DataType) -> list[str
     return errors
 
 
-def _normalise_name(name: str) -> str:
-    return name.upper().strip()
-
-
 def _norm_header_map(columns: list) -> dict:
     mapping, _ = {}, set()
     for c in columns:

{valediction-1.0.3 → valediction-1.2.0}/src/valediction/dictionary/importing.py

@@ -11,7 +11,6 @@ from valediction.dictionary.helpers import (
     _get_required_header,
     _is_missing,
     _norm_header_map,
-    _normalise_name,
     _parse_int,
     _parse_truthy,
     _row_is_blank,
@@ -19,7 +18,7 @@ from valediction.dictionary.helpers import (
 from valediction.dictionary.integrity import REQUIRED_SHEETS
 from valediction.dictionary.model import Column, Dictionary, Table
 from valediction.exceptions import DataDictionaryError, DataDictionaryImportError
-from valediction.support import list_as_bullets
+from valediction.support import _normalise, _strip, list_as_bullets
 
 
 @dataclass
@@ -80,6 +79,13 @@ class ExcelDataDictionary:
             raise error
 
     # Import & Helpers
+    def _resolve_table_name(self, name: str) -> str | None:
+        """Return the canonical table name as it appears in Tables sheet (or None)."""
+        target = _normalise(name)
+        return next(
+            (t for t in self.table_metadata.keys() if _normalise(t) == target), None
+        )
+
     def _open_workbook(self) -> None:
         if not self.path.exists():
             raise DataDictionaryImportError(f"File not found: {self.path}")
@@ -140,20 +146,27 @@ class ExcelDataDictionary:
         description_col_header = _get_required_header(header_map, "description")
 
         meta: dict[str, str | None] = {}
+        seen: set[str] = set()
+
         for _, row in tables_df.iterrows():
             if _is_missing(row[table_col_header]):
                 continue
-            table_name = _normalise_name(str(row[table_col_header]))
+
+            table_name = _strip(str(row[table_col_header]))
             table_description = (
                 None
                 if _is_missing(row[description_col_header])
                 else str(row[description_col_header])
             )
-            if table_name in meta:
+
+            key = _normalise(table_name)
+            if key in seen:
                 raise DataDictionaryImportError(
                     f"Duplicate table '{table_name}' in Tables sheet."
                 )
+            seen.add(key)
             meta[table_name] = table_description
+
         if not meta:
             raise DataDictionaryImportError(
                 "Data Dictionary sheet 'Tables' contains no table rows."
@@ -177,12 +190,13 @@ class ExcelDataDictionary:
                 or _is_missing(row[code_col_header])
             ):
                 continue
-            table_name = _normalise_name(str(row[table_col_header]))
-            column_name = _normalise_name(str(row[column_col_header]))
-            enum_map.setdefault((table_name, column_name), {})
-            enum_map[(table_name, column_name)][row[code_col_header]] = row[
-                name_col_header
-            ]
+            table_name = _strip(str(row[table_col_header]))
+            column_name = _strip(str(row[column_col_header]))
+            resolved_table = self._resolve_table_name(table_name) or table_name
+            enum_key = (_normalise(resolved_table), _normalise(column_name))
+            enum_map.setdefault(enum_key, {})
+            enum_map[enum_key][row[code_col_header]] = row[name_col_header]
+
         self.enumerations = enum_map
 
     # Parse Columns
@@ -234,7 +248,12 @@ class ExcelDataDictionary:
 
             self.table_columns[inputs.table_name].append(column_obj)
             if inputs.has_enumerations:
-                self.enum_flags.add((inputs.table_name, inputs.column_name))
+                self.enum_flags.add(
+                    (
+                        _normalise(inputs.table_name),
+                        _normalise(inputs.column_name),
+                    )
+                )
 
         if errors:
             raise DataDictionaryImportError(
@@ -279,7 +298,7 @@ class ExcelDataDictionary:
 
     # Validate Foreign Keys
     def _validate_foreign_keys(self) -> None:
-        name_to_table = {t.name: t for t in self.tables}
+        name_to_table = {_normalise(t.name): t for t in self.tables}
         errors: list[str] = []
         for table in self.tables:
             for column in table:
@@ -292,9 +311,9 @@ class ExcelDataDictionary:
                     )
                     continue
                 target_table_raw, target_column_raw = target.split(".", 1)
-                target_table_name = _normalise_name(target_table_raw)
-                target_column_name = _normalise_name(target_column_raw)
-                referenced_table = name_to_table.get(target_table_name)
+                target_table_name = _strip(target_table_raw)
+                target_column_name = _strip(target_column_raw)
+                referenced_table = name_to_table.get(_normalise(target_table_name))
                 if not referenced_table:
                     errors.append(
                         f"{table.name}.{column.name} references unknown table {target_table_name!r}."
@@ -392,13 +411,17 @@ class ExcelDataDictionary:
                 f"{row_context}: missing required field(s): {', '.join(missing_fields)}."
             )
 
-        table_name = _normalise_name(str(row[table_col_header]))
-        column_name = _normalise_name(str(row[column_col_header]))
-        if table_name not in self.table_metadata:
+        table_name_raw = _strip(str(row[table_col_header]))
+        column_name = _strip(str(row[column_col_header]))
+
+        resolved_table_name = self._resolve_table_name(table_name_raw)
+        if resolved_table_name is None:
             raise DataDictionaryImportError(
-                f"{row_context}: Table '{table_name}' not present in Tables sheet."
+                f"{row_context}: Table '{table_name_raw}' not present in Tables sheet."
             )
 
+        table_name = resolved_table_name
+
         order_int = _parse_int(row[order_col_header], "Order", row_context)
         length_int = (
             _parse_int(row[length_col_header], "Length", row_context, required=False)
@@ -461,7 +484,7 @@ class ExcelDataDictionary:
 
     def _make_column(self, inputs: _ColumnInputs) -> Column:
         enums_for_column = self.enumerations.get(
-            (inputs.table_name, inputs.column_name), {}
+            (_normalise(inputs.table_name), _normalise(inputs.column_name)), {}
         )
         return Column(
             name=inputs.column_name,

{valediction-1.0.3 → valediction-1.2.0}/src/valediction/dictionary/model.py

@@ -9,10 +9,9 @@ from valediction.dictionary.helpers import (
     _check_name,
     _check_order,
     _check_primary_key,
-    _normalise_name,
 )
 from valediction.exceptions import DataDictionaryError
-from valediction.support import list_as_bullets
+from valediction.support import _normalise, _strip, list_as_bullets
 
 
 class Column:
@@ -44,7 +43,7 @@ class Column:
         description: str | None = None,
         datetime_format: str | None = None,
     ):
-        self.name = _normalise_name(name)
+        self.name = _strip(name)
         self.order = int(order) if order is not None else None
         self.data_type: DataType = None
         self.length = int(length) if length is not None else None
@@ -127,7 +126,7 @@ class Table(list[Column]):
         columns: list[Column] | None = None,
     ):
         super().__init__()
-        self.name = _normalise_name(name)
+        self.name = _strip(name)
         self.description = description
         for column in columns or []:
             self.add_column(column)
@@ -139,24 +138,28 @@ class Table(list[Column]):
         )
         return f"Table(name={self.name!r}, description={self.description!r}{cols_str})"
 
+    def __key(self, name: str) -> str:
+        return _normalise(name)
+
     def __getitem__(self, key: int | str) -> Column:
         if isinstance(key, int):
             return super().__getitem__(key)
-        target = _normalise_name(key)
-        found = next((c for c in self if c.name == target), None)
+
+        target_key = self.__key(key)
+        found = next((c for c in self if self.__key(c.name) == target_key), None)
         if not found:
             raise KeyError(f"Column {key!r} not found in table {self.name!r}.")
         return found
 
     def __get(self, name: str, default: Column | None = None) -> Column | None:
-        target = _normalise_name(name)
-        return next((c for c in self if c.name == target), default)
+        target_key = self.__key(name)
+        return next((c for c in self if self.__key(c.name) == target_key), default)
 
     # Getters
     def index_of(self, name: str) -> int | None:
-        target = _normalise_name(name)
+        target_key = self.__key(name)
         for i, c in enumerate(self):
-            if c.name == target:
+            if self.__key(c.name) == target_key:
                 return i
         return None
 
@@ -303,16 +306,17 @@ class Table(list[Column]):
         if not isinstance(column, Column):
             raise DataDictionaryError("Only Column objects can be added to a Table.")
 
-        if column.name in self.get_column_names():
-            conflict = self.get_column(column.name)
+        incoming_key = self.__key(column.name)
+        conflict = next((c for c in self if self.__key(c.name) == incoming_key), None)
+        if conflict is not None:
             raise DataDictionaryError(
-                f"Column {column.name!r} already exists (order={conflict.order!r})"
+                f"Column {column.name!r} already exists (order={conflict.order!r}, as {conflict.name!r})."
             )
 
         if column.order in self.get_column_orders():
-            conflict = self.get_column(column.order)
+            conflict_by_order = self.get_column(column.order)
             raise DataDictionaryError(
-                f"Order {column.order!r} already exists (name={conflict.name!r})"
+                f"Order {column.order!r} already exists (name={conflict_by_order.name!r})"
             )
 
         if column.primary_key is not None:
@@ -339,10 +343,7 @@ class Table(list[Column]):
         Raises:
             DataDictionaryError: if the column does not exist
         """
-        if isinstance(column, str):
-            name = self.get_column(column).name
-        else:
-            name = self.get_column(column).name  # by order
+        name = self.get_column(column).name
         remaining = [c for c in self if c.name != name]
         self.clear()
         super().extend(remaining)
@@ -367,16 +368,17 @@ class Table(list[Column]):
         for col in self:
             col.primary_key = None
 
-        # Resolve and dedupe
+        # Resolve and deduplicate
         resolved: list[Column] = []
         seen: set[str] = set()
         for key in primary_keys:
             col = self.get_column(key)
-            if col.name in seen:
+            col_key = self.__key(col.name)
+            if col_key in seen:
                 raise DataDictionaryError(
                     f"Duplicate column {col.name!r} provided for table {self.name!r}."
                 )
-            seen.add(col.name)
+            seen.add(col_key)
             resolved.append(col)
 
         # Assign ordinals 1..N
@@ -416,14 +418,20 @@ class Dictionary(list[Table]):
     ):
         super().__init__()
         self.name = name
+
+        if isinstance(tables, Table):
+            tables = [tables]
+
         for t in tables or []:
             self.add_table(t)
+
         self.organisations = organisations
         self.version = version
         self.version_notes = version_notes
         self.inclusion_criteria = inclusion_criteria
         self.exclusion_criteria = exclusion_criteria
         self.imported = imported
+        self.__check_variables()
 
     # Properties
     @property
@@ -439,24 +447,85 @@ class Dictionary(list[Table]):
         tables = list_as_bullets(elements=[str(t) for t in self], bullet="\n- ")
         return f"Dictionary(name={self.name!r}, imported={self.imported!r}, {tables})"
 
+    def __key(self, name: str) -> str:
+        return _normalise(name)
+
     def __getitem__(self, key: int | str) -> Table:
         if isinstance(key, int):
             return super().__getitem__(key)
-        target = _normalise_name(key)
-        found = next((t for t in self if t.name == target), None)
+
+        target_key = self.__key(key)
+        found = next((t for t in self if self.__key(t.name) == target_key), None)
         if not found:
             raise KeyError(f"Table {key!r} not found in Dictionary.")
         return found
 
-    # Getters
     def __get(self, name: str, default: Table | None = None) -> Table | None:
-        target = _normalise_name(name)
-        return next((t for t in self if t.name == target), default)
+        target_key = self.__key(name)
+        return next((t for t in self if self.__key(t.name) == target_key), default)
+
+    # Checkers
+    def __check_variables(self) -> None:
+        self.__check_name()
+        self.__check_organisations()
+        self.__check_version()
+        self.__check_version_notes()
+        self.__check_criteria()
+
+    def __check_name(self) -> None:
+        # Check name
+        if self.name is not None:
+            if not isinstance(self.name, str):
+                raise DataDictionaryError("Dictionary `name` must be a string.")
+
+    def __check_organisations(self) -> None:
+        # Check organisations
+        if self.organisations is not None:
+            if not isinstance(self.organisations, str):
+                raise DataDictionaryError(
+                    "Dictionary `organisations` must be a string."
+                )
+
+    def __check_version(self) -> None:
+        # Check version
+        if self.version is not None:
+            if not isinstance(self.version, (str, int, float)):
+                raise DataDictionaryError(
+                    "Dictionary `version` must be a string, int, or float."
+                )
+
+            if isinstance(self.version, (int, float)):
+                self.version = str(self.version)
+
+        # Check version_notes
 
+    def __check_version_notes(self) -> None:
+        if self.version_notes is not None:
+            if not isinstance(self.version_notes, str):
+                raise DataDictionaryError(
+                    "Dictionary `version_notes` must be a string."
+                )
+
+    def __check_criteria(self) -> None:
+        # Check inclusion_criteria
+        if self.inclusion_criteria is not None:
+            if not isinstance(self.inclusion_criteria, str):
+                raise DataDictionaryError(
+                    "Dictionary `inclusion_criteria` must be a string."
+                )
+
+        # Check exclusion_criteria
+        if self.exclusion_criteria is not None:
+            if not isinstance(self.exclusion_criteria, str):
+                raise DataDictionaryError(
+                    "Dictionary exclusion_criteria must be a string."
+                )
+
+    # Getters
     def index_of(self, name: str) -> int | None:
-        target = _normalise_name(name)
+        target_key = self.__key(name)
         for i, t in enumerate(self):
-            if t.name == target:
+            if self.__key(t.name) == target_key:
                 return i
         return None
 
@@ -484,12 +553,9 @@ class Dictionary(list[Table]):
         Raises:
             KeyError: If the table is not found in the dictionary.
         """
-        target = _normalise_name(table)
-        found = next((t for t in self if t.name == target), None)
-
-        if not found:
+        found = self.__get(table)
+        if found is None:
             raise KeyError(f"Table {table!r} not found in Dictionary.")
-
         return found
 
     # Manipulation
@@ -508,8 +574,14 @@ class Dictionary(list[Table]):
             raise DataDictionaryError(
                 "Only Table objects can be added to a Dictionary."
             )
-        if table.name in self.get_table_names():
-            raise DataDictionaryError(f"Table {table.name!r} already exists.")
+
+        incoming_key = self.__key(table.name)
+        conflict = next((t for t in self if self.__key(t.name) == incoming_key), None)
+        if conflict is not None:
+            raise DataDictionaryError(
+                f"Table {table.name!r} already exists (as {conflict.name!r})."
+            )
+
         super().append(table)
 
     def remove_table(self, table: str) -> None: