thds.tabularasa 0.13.0__py3-none-any.whl

thds/tabularasa/data_dependencies/sqlite.py

@@ -0,0 +1,286 @@
+import datetime
+import io
+import warnings
+from logging import getLogger
+from pathlib import Path
+from typing import Callable, Mapping, Optional, Union
+
+import pandas as pd
+
+from thds.tabularasa.loaders.parquet_util import TypeCheckLevel, pandas_maybe
+from thds.tabularasa.loaders.sqlite_util import bulk_write_connection, sqlite_preprocessor_for_type
+from thds.tabularasa.loaders.util import PandasParquetLoader
+from thds.tabularasa.schema.compilation.sqlite import render_sql_index_schema, render_sql_table_schema
+from thds.tabularasa.schema.metaschema import Schema, Table, is_build_time_package_table
+from thds.tabularasa.sqlite3_compat import sqlite3
+
+from .util import hash_file
+
+TABLE_METADATA_TABLE_NAME = "_table_metadata"
+TABLE_METADATA_DDL = f"""CREATE TABLE IF NOT EXISTS {TABLE_METADATA_TABLE_NAME}(
+    table_name TEXT PRIMARY KEY,
+    n_rows INTEGER NOT NULL,
+    data_hash TEXT NOT NULL,
+    table_ddl_hash TEXT NOT NULL ,
+    index_ddl_hash TEXT
+);"""
+
+
+def insert_table(
+    con: sqlite3.Connection,
+    table: Table,
+    package: Optional[str],
+    data_dir: str,
+    filename: Optional[str] = None,
+    validate: bool = False,
+    check_hash: bool = True,
+    type_check: Optional[TypeCheckLevel] = None,
+    cast: bool = False,
+):
+    """Insert data for a schema table but only if the table doesn't exist in the database OR it exists
+    and the hashes of DDL scripts and data recorded there do not match those derived from auto-generated
+    code and data from the schema
+    """
+    _LOGGER = getLogger(__name__)
+    table_ddl = render_sql_table_schema(table)
+    index_ddl = render_sql_index_schema(table)
+
+    table_ddl_hash_expected = hash_file(io.BytesIO(table_ddl.strip().encode()))
+    if index_ddl is None:
+        index_ddl_hash_expected = None
+    else:
+        index_ddl_hash_expected = hash_file(io.BytesIO(index_ddl.strip().encode()))
+
+    loader = PandasParquetLoader.from_schema_table(
+        table,
+        package=package or None,
+        data_dir=data_dir,
+        filename=filename,
+        derive_schema=validate,
+    )
+    data_hash_expected = loader.file_hash()
+
+    _ensure_metadata_table(con)
+
+    if (not check_hash) or (not table_populated(con, table)):
+        insert_data = True
+    else:
+        row = con.execute(
+            f"SELECT data_hash, table_ddl_hash, index_ddl_hash FROM {TABLE_METADATA_TABLE_NAME} "
+            "WHERE table_name = ?",
+            (table.snake_case_name,),
+        ).fetchone()
+        if row is None:
+            insert_data = True
+        else:
+            data_hash, table_ddl_hash, index_ddl_hash = row
+            if table_ddl_hash != table_ddl_hash_expected:
+                _LOGGER.info(
+                    f"Hash of table DDL doesn't match that recorded in the database; reinserting data "
+                    f"for {table.name}"
+                )
+                insert_data = True
+            elif index_ddl_hash != index_ddl_hash_expected:
+                _LOGGER.info(
+                    f"Hash of index DDL doesn't match that recorded in the database; reinserting data "
+                    f"for {table.name}"
+                )
+                insert_data = True
+            elif data_hash != data_hash_expected:
+                _LOGGER.info(
+                    f"Hash of source data doesn't match that recorded in the database; reinserting data "
+                    f"for {table.name}"
+                )
+                insert_data = True
+            else:
+                _LOGGER.info(
+                    f"Hashes of table DDL, index DDL, and source data match those recorded in the "
+                    f"database; skipping data insert for {table.name}"
+                )
+                insert_data = False
+
+    if insert_data:
+        _LOGGER.info(f"Inserting data for {table.name}")
+        with con:
+            _LOGGER.debug(f"Dropping existing table {table.name}")
+            con.execute(f"DROP TABLE IF EXISTS {table.snake_case_name}")
+            con.execute(
+                f"DELETE FROM {TABLE_METADATA_TABLE_NAME} WHERE table_name = ?",
+                (table.snake_case_name,),
+            )
+
+        with con:
+            _LOGGER.debug(f"Executing table DDL for {table.name}:\n{table_ddl}")
+            con.execute(table_ddl)
+            # `postprocess=True` implies that all collection-valued columns will have values that are
+            # instances of builtin python types (dicts, lists), and thus that they are handleable by
+            # `cattrs` and thus ultimately JSON-serializable
+            try:
+                batches = loader.load_batched(
+                    validate=validate, postprocess=True, type_check=type_check, cast=cast
+                )
+            except KeyError as key_error:
+                _LOGGER.error(
+                    "Column names in parquet file may not match "
+                    "schema. Try deleting derived files and building again."
+                )
+                raise Exception(key_error)
+
+            _LOGGER.debug(f"Inserting data for table {table.name}")
+            for df in batches:
+                df = _prepare_for_sqlite(df, table)
+
+                with warnings.catch_warnings():
+                    warnings.filterwarnings("ignore", "pandas only supports SQLAlchemy connectable")
+                    df.to_sql(table.snake_case_name, con, if_exists="append", index=False)
+
+        if index_ddl:
+            with con:
+                _LOGGER.debug(f"Executing index DDL for table {table.name}:\n{index_ddl}")
+                con.executescript(index_ddl)
+
+        with con:
+            # only insert hashes once all table and index creation is complete
+            _LOGGER.debug(f"Inserting metadata for {table.name}")
+            num_rows = loader.num_rows()
+            con.execute(
+                f"INSERT INTO {TABLE_METADATA_TABLE_NAME}"
+                "(table_name, n_rows, data_hash, table_ddl_hash, index_ddl_hash) VALUES (?, ?, ?, ?, ?)",
+                (
+                    table.snake_case_name,
+                    num_rows,
+                    data_hash_expected,
+                    table_ddl_hash_expected,
+                    index_ddl_hash_expected,
+                ),
+            )
+
+
+def _prepare_for_sqlite(df: pd.DataFrame, table: Table) -> pd.DataFrame:
+    """Only meant for use right before `df` is inserted into a sqlite table. Mutates `df` by converting
+    collection-valued columns to JSON literals"""
+    if table.primary_key:
+        df.reset_index(inplace=True, drop=False)
+
+    for column in table.columns:
+        name = column.snake_case_name
+        pytype = column.type.python
+        if pytype is datetime.date:
+            # pandas has only one type for dates/datetimes and uses a datetime ISO format for it, whereas
+            # our sqlite adapters use ISO formats specific to dates and datetimes; just cast to regular
+            # `datetime.date` here and the sqlite API will handle the conversion to ISO format since
+            # we're using `detect_types=sqlite3.PARSE_DECLTYPES`
+            df[name] = df[name].dt.date
+        else:
+            preproc = sqlite_preprocessor_for_type(pytype)  # type: ignore
+            if preproc is not None:
+                preproc_ = pandas_maybe(preproc) if column.nullable else preproc
+                df[name] = df[name].apply(preproc_)
+
+    return df
+
+
+def _ensure_metadata_table(con: sqlite3.Connection):
+    if not table_exists(con, TABLE_METADATA_TABLE_NAME):
+        con.execute(TABLE_METADATA_DDL)
+
+
+def table_populated(con: sqlite3.Connection, table: Table) -> bool:
+    """Return True if a table named `table` exists in the database represented by `con` and has any rows,
+    False otherwise"""
+    try:
+        cur = con.execute(
+            f"SELECT {', '.join(c.snake_case_name for c in table.columns)} FROM {table.snake_case_name} limit 1"
+        )
+    except sqlite3.Error:
+        return False
+    else:
+        results = list(cur)
+        return bool(results)
+
+
+def table_exists(con: sqlite3.Connection, table: Union[str, Table]) -> bool:
+    table_name = table if isinstance(table, str) else table.snake_case_name
+    try:
+        con.execute(f"SELECT * FROM {table_name} limit 1")
+    except sqlite3.Error:
+        return False
+    return True
+
+
+def populate_sqlite_db(
+    schema: Schema,
+    db_package: Optional[str],
+    db_path: str,
+    data_package: Optional[str],
+    data_dir: str,
+    transient_data_dir: str,
+    validate: bool = False,
+    check_hash: bool = True,
+    type_check: Optional[TypeCheckLevel] = None,
+    cast: bool = False,
+    table_predicate: Callable[[Table], bool] = is_build_time_package_table,
+    data_path_overrides: Optional[Mapping[str, Path]] = None,
+):
+    """Populate a sqlite database with data for a set of tables from a `reference_data.schema.Schema`
+
+    :param schema: the `reference_data.schema.Schema` object defining the data to be inserted
+    :param db_package: name of the package where the database file is stored, if any. In case `None` is
+      passed, `db_path` refers to an ordinary file.
+    :param db_path: path to the sqlite database archive file in which the data will be inserted
+    :param data_package: optional package name, to be used if the data files are distributed as package
+      data
+    :param data_dir: path to the directory where the table parquet files are stored (relative to
+      `data_package` if it is passed). Ignored for any tables specified in `data_path_overrides` - see
+      below
+    :param transient_data_dir: path to the directory where transient table parquet files are stored
+      (relative to `data_package` if it is passed). Ignored for any tables specified in
+      `data_path_overrides` - see below
+    :param validate: if True, validate tables against their pandera schemas on load before inserting the
+      data into the database
+    :param check_hash: if True, skip tables whose data has already been inserted into the database, as
+      indicated by a hash of file contents and DDL statements in the _table_metadata table in the
+      database
+    :param type_check: optional `reference_data.loaders.parquet_util.TypeCheckLevel`. If given, the
+      arrow schemas of the files to be loaded will be checked against their expected arrow schemas as
+      derived from `schema` before read. This is a very efficient check as it requires no data to be
+      read. Useful for loading tables at run time as a quick validity check. This is passed to the same
+      keyword argument of `reference_data.loaders.util.PandasParquetLoader.__call__`.
+    :param cast: indicates that a safe cast of the parquet data should be performed on load using
+      `pyarrow`, in case the file arrow schema doesn't match the expected one exactly. This is passed to
+      the same keyword argument of `reference_data.loaders.util.PandasParquetLoader.__call__`.
+    :param table_predicate: Optional predicate indicating which tables from `schema.tables` should be
+      inserted into the database. If not given, all tables in the schema will be inserted.
+    :param data_path_overrides: Optional mapping from table name to the file path where the parquet
+      data for the table is to be loaded from. Any table whose name is a key in this mapping will be
+      loaded from the associated file path as a normal file (`data_package` and `data_dir` will be
+      ignored). This is useful for specifying dynamic run-time-installed tables.
+    """
+    # gather all tables before executing any I/O
+    insert_tables = [table for table in schema.filter_tables(table_predicate) if table.has_indexes]
+
+    with bulk_write_connection(db_path, db_package, close=True) as con:
+        for table in insert_tables:
+            table_filename: Optional[str]
+            table_package: Optional[str]
+            if data_path_overrides and table.name in data_path_overrides:
+                data_path = Path(data_path_overrides[table.name]).absolute()
+                table_package = None
+                table_data_dir = str(data_path.parent)
+                table_filename = data_path.name
+            else:
+                table_package = data_package
+                table_data_dir = transient_data_dir if table.transient else data_dir
+                table_filename = None
+
+            insert_table(
+                con=con,
+                table=table,
+                package=table_package,
+                data_dir=table_data_dir,
+                filename=table_filename,
+                validate=validate,
+                check_hash=check_hash,
+                type_check=type_check,
+                cast=cast,
+            )

thds/tabularasa/data_dependencies/tabular.py

@@ -0,0 +1,167 @@
+from typing import AbstractSet, Any, Callable, Dict, List, Optional, TypeVar, cast
+
+import pandas as pd
+import pandera as pa
+
+from thds.tabularasa.loaders.sqlite_util import sqlite_postprocessor_for_type
+from thds.tabularasa.schema import metaschema
+from thds.tabularasa.schema.dtypes import DType, PyType
+from thds.tabularasa.schema.files import TabularFileSource
+
+from .util import check_categorical_values
+
+T = TypeVar("T")
+K = TypeVar("K", bound=PyType)
+V = TypeVar("V", bound=PyType)
+
+BOOL_CONSTANTS = {
+    "true": True,
+    "false": False,
+    "t": True,
+    "f": False,
+    "yes": True,
+    "no": False,
+    "y": True,
+    "n": False,
+    "1": True,
+    "0": False,
+}
+JSON_NULL = "null"
+
+
+class PandasCSVLoader:
+    """Base interface for loading package data CSV files as pandas.DataFrames
+    This is only for use at build time"""
+
+    def __init__(self, table: metaschema.Table, schema: Optional[pa.DataFrameSchema] = None):
+        if not isinstance(table.dependencies, TabularFileSource):
+            raise ValueError(
+                f"Table '{table.name}' has no single tablular text file source of truth; it depends on "
+                f"{table.dependencies}"
+            )
+
+        self.schema = schema
+        self.table = table
+        self.header = [column.header_name for column in self.table.columns]
+        self.rename = {column.header_name: column.name for column in self.table.columns}
+        # set the primary key as the index
+        self.index_cols: List[str] = list(self.table.primary_key) if self.table.primary_key else []
+        self.cols = [c.name for c in self.table.columns if c.name not in self.index_cols]
+        # pass these to the csv parser as parse_dates
+        self.parse_date_cols = [
+            column.header_name
+            for column in self.table.columns
+            if column.dtype in (DType.DATE, DType.DATETIME)
+        ]
+        self.na_values = table.csv_na_values
+        self.dtypes = {}
+        self.dtypes_for_csv_read = {}
+        self.converters: Dict[str, Callable[[str], Any]] = {}
+        for column in self.table.columns:
+            dtype = column.pandas(index=column.name in self.index_cols)
+            self.dtypes[column.name] = dtype
+            na_values_for_col = self.na_values.get(column.header_name)
+            if column.dtype == DType.BOOL:
+                # we have a custom converter (parser) for boolean values.
+                # converters override na_values in pandas.read_csv, so we have to specify them here.
+                self.converters[column.header_name] = (
+                    parse_optional(parse_bool, na_values_for_col) if na_values_for_col else parse_bool
+                )
+            elif isinstance(column.dtype, (metaschema.ArrayType, metaschema.MappingType)):
+                # parse as json - again a custom converter which overrides na_values
+                converter = sqlite_postprocessor_for_type(column.dtype.python)
+                assert converter is not None  # converter for a structured type will not be None
+                self.converters[column.header_name] = cast(
+                    Callable[[str], Any],
+                    parse_optional(converter, na_values_for_col) if na_values_for_col else converter,
+                )
+            elif column.header_name not in self.parse_date_cols:
+                # read_csv requires passing `parse_dates` for determining date-typed columns
+                # also do NOT tell pandas.read_csv you want an enum; it will mangle unknown values to null!
+                self.dtypes_for_csv_read[column.header_name] = (
+                    dtype
+                    if column.type.enum is None
+                    else column.dtype.pandas(
+                        nullable=column.nullable, index=column.name in self.index_cols
+                    )
+                )
+
+    def __call__(self, validate: bool = False) -> pd.DataFrame:
+        if validate and self.schema is None:
+            raise ValueError(f"Can't validate table {self.table.name} with no schema")
+
+        df = self.read()
+        df = self.postprocess(df)
+        # schema not None only to make mypy happy - error thrown above in case it's required
+        return self.schema.validate(df) if (validate and self.schema is not None) else df
+
+    def read(self):
+        # make mypy happy; this is checked in __init__
+        deps = self.table.dependencies
+        assert isinstance(deps, TabularFileSource)
+        with deps.file_handle as f:
+            df = pd.read_csv(  # type: ignore
+                f,
+                usecols=self.header,
+                dtype=self.dtypes_for_csv_read,
+                parse_dates=self.parse_date_cols,
+                converters=self.converters,
+                skiprows=deps.skiprows or 0,
+                encoding=deps.encoding,
+                dialect=deps.csv_dialect,
+                na_values={k: sorted(v) for k, v in self.na_values.items()},
+                # without this, pandas adds in its own extensive set of strings to interpret as null.
+                # we force the user to be explicit about the values they want to parse as null.
+                keep_default_na=False,
+            )
+
+        return df
+
+    def postprocess(
+        self,
+        df: pd.DataFrame,
+    ):
+        """Ensure correct column names, column order, dtypes and index. Mutates `df` in-place"""
+        df.rename(columns=self.rename, inplace=True)
+
+        # pandas silently nullifies values not matching a categorical dtype!
+        # so we have to do this ourselves before we coerce with .astype below
+        for col in self.table.columns:
+            name = col.name
+            dtype = self.dtypes[name]
+            if isinstance(dtype, pd.CategoricalDtype):
+                check_categorical_values(df[name], dtype)
+
+        df = df.astype(self.dtypes, copy=False)
+        if self.index_cols:
+            df.set_index(self.index_cols, inplace=True)
+
+        if list(df.columns) != self.cols:
+            df = df[self.cols]
+
+        return df
+
+
+# CSV parsing for complex types
+
+
+def identity(x):
+    return x
+
+
+def parse_optional(
+    func: Callable[[str], V], null_values: AbstractSet[str] = frozenset([""])
+) -> Callable[[str], Optional[V]]:
+    """Turn a csv parser for a type V into a parser for Optional[V] by treating the empty string as a
+    null value"""
+
+    def parse(s: str) -> Optional[V]:
+        if s in null_values:
+            return None
+        return func(s)
+
+    return parse
+
+
+def parse_bool(s: str) -> bool:
+    return BOOL_CONSTANTS[s.lower()]

thds/tabularasa/data_dependencies/util.py

@@ -0,0 +1,209 @@
+import hashlib
+import multiprocessing
+import multiprocessing.connection
+import os
+import warnings
+from functools import partial, wraps
+from logging import getLogger
+from pathlib import Path
+from typing import IO, Callable, Dict, List, Optional, TypeVar, Union, cast
+
+import pandas as pd
+import pkg_resources
+import pyarrow
+
+from thds.tabularasa.data_dependencies.adls import ADLSDownloadResult
+from thds.tabularasa.loaders.parquet_util import pandas_maybe, preprocessor_for_pyarrow_type
+from thds.tabularasa.schema.files import LocalDataSpec
+from thds.tabularasa.schema.metaschema import Table
+from thds.tabularasa.schema.util import Identifier, import_func
+
+PARQUET_FORMAT_VERSION = "2.4"
+HASH_FILE_BUFFER_SIZE = 2**16
+FILENAME_YEAR_REGEX = r"(?:[^\d])(20\d{2})(?:[^\d])"
+FILENAME_QUARTER_REGEX = r"(?:[^\d])([Qq]\d{1})(?:[^\d])"
+
+MaterializedPackageDataDeps = Dict[Identifier, pd.DataFrame]
+SyncedADLSDeps = Dict[Identifier, List[ADLSDownloadResult]]
+RawLocalDataDeps = Dict[Identifier, LocalDataSpec]
+DataPreprocessor = Callable[
+    [MaterializedPackageDataDeps, SyncedADLSDeps, RawLocalDataDeps], pd.DataFrame
+]
+F = TypeVar("F", bound=Callable)
+
+
+def package_data_file_size(package: str, path: str) -> int:
+    os_path = pkg_resources.resource_filename(package, path)
+    return os.stat(os_path).st_size
+
+
+def run_in_subprocess(func: F) -> F:
+    """Decorator to cause a side-effect-producing routine to run in a subprocess. Isolates memory
+    consumption of the function call to the subprocess to ensure that all memory resources are reclaimed
+    at the end of the call. For example, `pyarrow` is known to be quite aggressive with memory allocation
+    and reluctant to free consumed memory. For example, a routine that simply reads a parquet file using
+    `pyarrow` and then writes the result somewhere else would benefit from use of this decorator.
+    """
+
+    @wraps(func)
+    def subprocess_func(*args, _subprocess: bool = True, **kwargs):
+        # extra _subprocess arg required to avoid trying to send the wrapped `func` to a subprocess,
+        # which in python 3.8+ results in a pickling error since it isn't the same object as the
+        # function importable at its own module/name (it's been replaced by `subprocess_func`)
+        if _subprocess:
+            recv_con, send_con = multiprocessing.Pipe(duplex=False)
+            proc = multiprocessing.Process(
+                target=SubprocessFunc(subprocess_func),
+                args=(send_con, *args),
+                kwargs=dict(_subprocess=False, **kwargs),
+            )
+            proc.start()
+            try:
+                result, exc = recv_con.recv()
+                proc.join()
+            except Exception as e:
+                # communication error, e.g. unpicklable return value
+                raise e
+            else:
+                if exc is not None:
+                    raise exc
+            finally:
+                proc.close()
+        else:
+            result = func(*args, **kwargs)
+
+        return result
+
+    return cast(F, subprocess_func)
+
+
+class SubprocessFunc:
+    def __init__(self, func):
+        self.func = func
+
+    def __call__(self, con: multiprocessing.connection.Connection, *args, **kwargs):
+        exc: Optional[Exception]
+        try:
+            result = self.func(*args, **kwargs)
+        except Exception as e:
+            exc = e
+            result = None
+        else:
+            exc = None
+        con.send((result, exc))
+
+
+def hash_file(file: Union[Path, str, IO[bytes]]) -> str:
+    """MD5 hash of the contents of a file (specified by path or passed directly as a handle)"""
+    io: IO[bytes]
+    if isinstance(file, (str, Path)):
+        io = open(file, "rb")
+        close = True
+    else:
+        io = file
+        close = False
+
+    hash_ = hashlib.md5()
+    for bytes_ in iter(partial(io.read, HASH_FILE_BUFFER_SIZE), b""):
+        hash_.update(bytes_)
+
+    if close:
+        io.close()
+
+    return hash_.hexdigest()
+
+
+def import_data_preprocessor(path: str) -> DataPreprocessor:
+    return import_func(path)
+
+
+def arrow_table_for_parquet_write(df: pd.DataFrame, table: Table) -> pyarrow.Table:
+    """Preprocess a dataframe with possibly complex object types in preparation to write to a parquet
+    file. Casts dicts to lists since pyarrow expects lists or arrays of key-value tuples as the
+    represenation of mapping types. Also casts any types with different kinds - e.g. if a float column is
+    expected but an int column is passed. Possibly mutates input as this should only be called on a table
+    which is about to be saved as a parquet file and then garbage-collected."""
+    logger = getLogger(__name__)
+
+    if any(df.index.names):
+        df.reset_index(inplace=True)
+
+    table_columns = {c.name for c in table.columns}
+    extra_columns = [c for c in df.columns if c not in table_columns]
+    if extra_columns:
+        logger.warning(
+            f"Extra columns {extra_columns!r} in dataframe but not in schema of table {table.name!r} "
+            "will be dropped on parquet write"
+        )
+
+    for column in table.columns:
+        field = column.parquet_field
+        name = column.name
+        pproc = preprocessor_for_pyarrow_type(field.type)
+        if pproc is not None:
+            if field.nullable:
+                pproc = pandas_maybe(pproc)
+            df[name] = df[name].apply(pproc)
+
+        if (enum_constraint := column.dtype.enum) is not None:
+            try:
+                check_categorical_values(df[name], pd.CategoricalDtype(enum_constraint.enum))
+            except ValueError as e:
+                # only warn on write since the data may in fact be correct while only the schema needs
+                # updating, potentially saving the developer an expensive derivation
+                warnings.warn(str(e))
+
+    if table.primary_key:
+        df.sort_values(list(table.primary_key), inplace=True)
+
+    arrow = pyarrow.Table.from_pandas(df, table.parquet_schema, safe=True)
+    # we remove the pandas-related metadata to ensure that insignificant changes e.g. to pandas/pyarrow
+    # versions do not effect file hashes. We can safely do this since we don't rely on pandas to infer
+    # data types on load, instead using the parquet/arrow schemas directly on load (pyarrow uses the
+    # 'ARROW:schema' metadata key to document arrow schemas in a serialized binary format so we don't
+    # lose that information by discarding the pandas information)
+    meta = arrow.schema.metadata
+    meta.pop(b"pandas")
+    return arrow.replace_schema_metadata(meta)
+
+
+def check_categorical_values(col: pd.Series, dtype: pd.CategoricalDtype):
+    """Check that values in a column match an expected categorical dtype prior to a write or cast operation.
+    This exists to preempt the unfortunate behavior of pandas wherein a cast silently nullifies any values
+    which are not in the categories of the target `CategoricalDtype`, resulting in confusing errors (or
+    worse - no errors in case null values are tolerated) downstream.
+
+    :raises TypeError: when the underlying data type of the `series` has a different kind than the categories of the `dtype`
+    :raises ValueError: when any values in the `series` are outside the expected set of categories of the `dtype`
+    """
+    current_dtype = col.dtype
+    expected_dtype = dtype.categories.dtype
+
+    if isinstance(current_dtype, pd.CategoricalDtype):
+        current_dtype_kind = current_dtype.categories.dtype.kind
+    else:
+        current_dtype_kind = current_dtype.kind
+
+    int_kinds = {"i", "u"}
+    if current_dtype_kind != expected_dtype.kind and not (
+        current_dtype_kind in int_kinds and expected_dtype.kind in int_kinds
+    ):
+        raise TypeError(
+            f"Column {col.name} is expected to be categorical with underlying data type "
+            f"{expected_dtype}, but has incompatible type {current_dtype}"
+        )
+
+    expected_values = dtype.categories
+    actual_values = pd.Series(col.dropna().unique())
+    bad_values = actual_values[~actual_values.isin(expected_values)]
+    if len(bad_values):
+        display_max_values = 20
+        addendum = (
+            f"(truncated to {display_max_values} unique values)"
+            if len(bad_values) > display_max_values
+            else ""
+        )
+        raise ValueError(
+            f"Column {col.name} is expected to have values in the set {expected_values.tolist()}, "
+            f"but also contains values {bad_values[:display_max_values].tolist()}{addendum}"
+        )