mostlyai-engine 1.0.0__py3-none-any.whl

mostlyai/engine/_encoding_types/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2025 MOSTLY AI
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

mostlyai/engine/_encoding_types/language/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2025 MOSTLY AI
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

mostlyai/engine/_encoding_types/language/text.py

@@ -0,0 +1,41 @@
+# Copyright 2025 MOSTLY AI
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pandas as pd
+
+from mostlyai.engine._common import safe_convert_string
+
+
+def analyze_text(values: pd.Series, root_keys: pd.Series, _: pd.Series | None = None) -> dict:
+    # ideally, we should ensure that values are converted to string in a consistent way across analyze/encode/qa steps
+    values = safe_convert_string(values)
+    nchars = values.map(str).str.len()
+    stats = {"nchar_max": int(nchars.max()), "nchar_sum": int(nchars.sum()), "count": len(values)}
+    return stats
+
+
+def analyze_reduce_text(stats_list: list[dict], _: bool = True) -> dict:
+    nchar_max = 0
+    nchar_sum = 0
+    count = 0
+    for stats in stats_list:
+        nchar_max = max(stats["nchar_max"], nchar_max)
+        nchar_sum += stats["nchar_sum"]
+        count += stats["count"]
+
+    stats = {
+        "nchar_avg": round(nchar_sum / count, 1),
+        "nchar_max": nchar_max,
+    }
+    return stats

mostlyai/engine/_encoding_types/tabular/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2025 MOSTLY AI
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

mostlyai/engine/_encoding_types/tabular/categorical.py

@@ -0,0 +1,115 @@
+# Copyright 2025 MOSTLY AI
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Categorical encoding maps each categorical value to its own integer code.
+"""
+
+import numpy as np
+import pandas as pd
+
+from mostlyai.engine._common import safe_convert_string
+
+CATEGORICAL_UNKNOWN_TOKEN = "_RARE_"
+CATEGORICAL_NULL_TOKEN = "<<NULL>>"
+CATEGORICAL_SUB_COL_SUFFIX = "cat"
+CATEGORICAL_ESCAPE_CHAR = "\x01"
+
+
+def safe_categorical_escape(values: pd.Series) -> pd.Series:
+    """Inplace escaping of categorical values"""
+    reserved_tokens = (CATEGORICAL_UNKNOWN_TOKEN, CATEGORICAL_NULL_TOKEN)
+    reserved_tokens_replacement_map = {t: CATEGORICAL_ESCAPE_CHAR + t for t in reserved_tokens}
+    # first, prefix values starting with escape char with another escape char
+    mask = values.str.startswith(CATEGORICAL_ESCAPE_CHAR, na=False)
+    values.loc[mask] = values.loc[mask].str.slice_replace(stop=1, repl=CATEGORICAL_ESCAPE_CHAR * 2)
+    # second, add escape char to all reserved tokens
+    values = values.replace(reserved_tokens_replacement_map)
+    return values
+
+
+def safe_categorical_unescape(values: pd.Series) -> pd.Series:
+    """Inplace un-escaping of categorical values"""
+    # de-prefix all values starting with escape char by removing just the first one
+    mask = values.str.startswith(CATEGORICAL_ESCAPE_CHAR, na=False)
+    values.loc[mask] = values.loc[mask].str[1:]
+    return values
+
+
+def analyze_categorical(values: pd.Series, root_keys: pd.Series, _: pd.Series | None = None) -> dict:
+    # ensure a safe representation of values: 1. string dtype; 2. escape reserved tokens
+    values = safe_categorical_escape(safe_convert_string(values))
+    # count distinct root_keys per categorical value for rare-category protection
+    df = pd.concat([root_keys, values], axis=1)
+    cnt_values = df.groupby(values.name)[root_keys.name].nunique().to_dict()
+    stats = {"has_nan": sum(values.isna()) > 0, "cnt_values": cnt_values}
+    return stats
+
+
+def analyze_reduce_categorical(stats_list: list[dict], value_protection: bool = True) -> dict:
+    # sum up all counts for each categorical value
+    cnt_values: dict[str, int] = {}
+    for item in stats_list:
+        for value, count in item["cnt_values"].items():
+            cnt_values[value] = cnt_values.get(value, 0) + count
+    # create alphabetically sorted list of non-rare categories
+    known_categories = [k for k in sorted(cnt_values.keys())]
+    if value_protection:
+        # stochastic threshold for rare categories
+        rare_min = 5 + int(3 * np.random.uniform())
+    else:
+        rare_min = 0
+    categories = [k for k in known_categories if cnt_values[k] >= rare_min]
+    no_of_rare_categories = len(known_categories) - len(categories)
+    # add special token for MISSING categories, if any are present
+    if any([j["has_nan"] for j in stats_list]):
+        categories = [CATEGORICAL_NULL_TOKEN] + categories
+    # add special token for UNKNOWN categories at first position
+    categories = [CATEGORICAL_UNKNOWN_TOKEN] + categories
+    stats = {
+        "no_of_rare_categories": no_of_rare_categories,
+        "codes": {categories[i]: i for i in range(len(categories))},
+        "cardinalities": {CATEGORICAL_SUB_COL_SUFFIX: len(categories)},
+    }
+    return stats
+
+
+def encode_categorical(values: pd.Series, stats: dict, _: pd.Series | None = None) -> pd.DataFrame:
+    # ensure a safe representation of values: 1. string dtype; 2. escape reserved tokens
+    values = safe_categorical_escape(safe_convert_string(values))
+    known_categories = [str(k) for k in stats["codes"].keys()]
+    values = values.copy()
+    if CATEGORICAL_NULL_TOKEN in known_categories:
+        values[values.isna()] = CATEGORICAL_NULL_TOKEN
+    values[~values.isin(known_categories)] = CATEGORICAL_UNKNOWN_TOKEN
+
+    # map categories to their corresponding codes
+    codes = pd.Series(
+        pd.Categorical(values, categories=known_categories).codes,
+        name=CATEGORICAL_SUB_COL_SUFFIX,
+        index=values.index,
+    )
+    return codes.to_frame()
+
+
+def decode_categorical(df_encoded: pd.DataFrame, stats: dict) -> pd.Series:
+    categories = stats["codes"].keys()
+    values = pd.Series(
+        pd.Categorical.from_codes(df_encoded[CATEGORICAL_SUB_COL_SUFFIX], categories=categories),
+        dtype="string",
+    )
+    values[values == CATEGORICAL_NULL_TOKEN] = pd.NA
+    # convert escaped values to their original representation
+    values = safe_categorical_unescape(values)
+    return values

mostlyai/engine/_encoding_types/tabular/character.py

@@ -0,0 +1,155 @@
+# Copyright 2025 MOSTLY AI
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Character encoding splits any value into its characters, and encodes each position then separately as a categorical.
+"""
+
+import numpy as np
+import pandas as pd
+
+from mostlyai.engine._common import safe_convert_string
+
+UNKNOWN_TOKEN = "\0"
+MAX_LENGTH_CHARS = 50
+
+
+def analyze_character(values: pd.Series, root_keys: pd.Series, _: pd.Series | None = None) -> dict:
+    values = safe_convert_string(values)
+    df_split = split_sub_columns_character(values)
+    has_nan = sum(df_split["nan"]) > 0
+    # count distinct root_keys per token for each character position
+    df = pd.concat([root_keys, df_split], axis=1)
+    characters = {
+        sub_col: df.groupby(sub_col)[root_keys.name].nunique().to_dict()
+        for sub_col in df_split.columns
+        if sub_col.startswith("P")
+    }
+    stats = {
+        "max_string_length": len(characters),
+        "has_nan": has_nan,
+        "characters": characters,
+    }
+    return stats
+
+
+def analyze_reduce_character(stats_list: list[dict], value_protection: bool = True) -> dict:
+    # gather maximum string length across partitions
+    max_string_length = max(stats["max_string_length"] for stats in stats_list)
+    positions = [f"P{idx}" for idx in range(max_string_length)]
+    # gather codes for each position
+    codes: dict[str, dict[str, int]] = {pos: {} for pos in positions}
+    for pos in positions:
+        cnt_values: dict[str, int] = {}
+        # sum up all counts for each token
+        for item in stats_list:
+            for value, count in item["characters"].get(pos, {}).items():
+                cnt_values[value] = cnt_values.get(value, 0) + count
+        # create alphabetically sorted list of non-rare tokens
+        known_categories = [k for k in sorted(cnt_values.keys())]
+        if value_protection:
+            # stochastic threshold for rare tokens
+            rare_min = 5 + int(3 * np.random.uniform())
+        else:
+            rare_min = 0
+        categories = [k for k in known_categories if cnt_values[k] >= rare_min]
+        # add special token for UNKNOWN at first position
+        categories = [UNKNOWN_TOKEN] + [c for c in categories if c != UNKNOWN_TOKEN]
+        # assign codes for each token
+        codes[pos] = {categories[i]: i for i in range(len(categories))}
+    # determine cardinalities
+    cardinalities = {}
+    has_nan = any([s["has_nan"] for s in stats_list])
+    if has_nan:
+        cardinalities["nan"] = 2  # binary
+    for sub_col, sub_col_codes in codes.items():
+        cardinalities[sub_col] = len(sub_col_codes)
+    stats = {
+        "has_nan": has_nan,
+        "max_string_length": max_string_length,
+        "codes": codes,
+        "cardinalities": cardinalities,
+    }
+    return stats
+
+
+def encode_character(values: pd.Series, stats: dict, _: pd.Series | None = None) -> pd.DataFrame:
+    values = safe_convert_string(values)
+    max_string_length = stats["max_string_length"]
+    df_split = split_sub_columns_character(values, max_string_length)
+    if not stats["has_nan"]:
+        df_split.drop(["nan"], axis=1, inplace=True)
+    for idx in range(max_string_length):
+        sub_col = f"P{idx}"
+        np_codes = np.array(pd.Categorical(df_split[sub_col], categories=stats["codes"][sub_col]).codes)
+        np.place(np_codes, np_codes == -1, 0)
+        df_split[sub_col] = np_codes
+    return df_split
+
+
+def split_sub_columns_character(
+    values: pd.Series,
+    max_string_length: int | None = None,
+) -> pd.DataFrame:
+    if not pd.api.types.is_string_dtype(values):
+        raise ValueError("expected to be string")
+    is_na = pd.Series(values.isna().astype("int"), name="nan").to_frame()
+    values = values.fillna("")
+    # trim strings to a maximum length
+    values = values.str.slice(stop=MAX_LENGTH_CHARS)
+    # pad strings to string_length
+    if max_string_length is None:
+        max_string_length = values.str.len().max()
+        max_string_length = (
+            int(max_string_length)  # type: ignore
+            if np.isscalar(max_string_length) and not np.isnan(max_string_length)
+            else 0
+        )
+    else:
+        values = values.str.slice(stop=max_string_length)
+    # explode to wide dataframe
+    padded_values = values.str.ljust(max_string_length, UNKNOWN_TOKEN)
+    chars_df = padded_values.str.split("", expand=True)
+    if not chars_df.empty:
+        chars_df = chars_df.drop([0, max_string_length + 1], axis=1)
+        chars_df.columns = [f"P{idx}" for idx in range(max_string_length)]
+    else:  # chars_df.empty is True
+        # even though the input is empty, we still need to return a dataframe with the correct columns
+        chars_df = pd.DataFrame(columns=[f"P{idx}" for idx in range(max_string_length)])
+    df = pd.concat([is_na, chars_df], axis=1)
+    return df
+
+
+def decode_character(df_encoded: pd.DataFrame, stats: dict) -> pd.Series:
+    if len(stats["codes"].keys()) > 0:
+        df_decoded = pd.DataFrame(
+            {
+                sub_col: pd.Series(
+                    pd.Categorical.from_codes(df_encoded[sub_col], categories=stats["codes"][sub_col]),
+                    dtype="string",
+                )
+                for sub_col in stats["codes"].keys()
+            },
+        )
+        values = df_decoded.apply(lambda item: "".join(item), axis=1, result_type="reduce").astype(
+            str
+        )  # necessary to keep string dtype for empty df_decoded
+        # remove unknown tokens and strip trailing whitespaces
+        values = values.apply(lambda item: item.replace(UNKNOWN_TOKEN, "")).str.rstrip()
+    else:
+        # handle de-generate case, where no tokens were stored
+        values = pd.Series(pd.NA).repeat(df_encoded.shape[0])
+    if stats["has_nan"]:
+        values[df_encoded["nan"] == 1] = pd.NA
+    return values

mostlyai/engine/_encoding_types/tabular/datetime.py

@@ -0,0 +1,254 @@
+# Copyright 2025 MOSTLY AI
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Datetime encoding splits a datetime into its parts, and encodes each part separately. For each part its
+minimum and maximum values are determined. Any part is then encoded as `x - min_value`, resulting in integers
+ranging from 0 to `max_value - min_value`. Thus, the corresponding cardinality is `max_value + 1 - min_value`.
+"""
+
+import calendar
+
+import numpy as np
+import pandas as pd
+from dateutil import parser  # type: ignore
+
+from mostlyai.engine._common import safe_convert_datetime
+from mostlyai.engine._dtypes import is_date_dtype, is_timestamp_dtype
+
+DATETIME_PARTS = [
+    "year",
+    "month",
+    "day",
+    "hour",
+    "minute",
+    "second",
+    "ms_E2",
+    "ms_E1",
+    "ms_E0",
+]
+
+
+def analyze_datetime(values: pd.Series, root_keys: pd.Series, _: pd.Series | None = None) -> dict:
+    values = safe_convert_datetime(values)
+    df = pd.concat([root_keys, values], axis=1)
+    # determine lowest/highest values by root ID, and return Top 10
+    min_dates = df.groupby(root_keys.name)[values.name].min().dropna()
+    min11 = min_dates.sort_values(ascending=True).head(11).astype(str).tolist()
+    max_dates = df.groupby(root_keys.name)[values.name].max().dropna()
+    max11 = max_dates.sort_values(ascending=False).head(11).astype(str).tolist()
+    # split into datetime parts
+    df_split = split_sub_columns_datetime(values)
+    is_not_nan = df_split["nan"] == 0
+    has_nan = any(df_split["nan"] == 1)
+    # extract min/max value for each part to determine valid value range
+    if any(is_not_nan):
+        min_values = {k: int(df_split[k][is_not_nan].min()) for k in DATETIME_PARTS}
+        max_values = {k: int(df_split[k][is_not_nan].max()) for k in DATETIME_PARTS}
+    else:
+        def_values = {"year": 2022, "month": 1, "day": 1}
+        min_values = {k: 0 for k in DATETIME_PARTS} | def_values
+        max_values = {k: 0 for k in DATETIME_PARTS} | def_values
+    # return stats
+    stats = {
+        "has_nan": has_nan,
+        "min_values": min_values,
+        "max_values": max_values,
+        "min11": min11,
+        "max11": max11,
+    }
+    return stats
+
+
+def analyze_reduce_datetime(stats_list: list[dict], value_protection: bool = True) -> dict:
+    # check if there are missing values
+    has_nan = any([j["has_nan"] for j in stats_list])
+    # determine min/max values for each part
+    keys = stats_list[0]["min_values"].keys()
+    min_values = {k: min([j["min_values"][k] for j in stats_list]) for k in keys}
+    max_values = {k: max([j["max_values"][k] for j in stats_list]) for k in keys}
+    # check if any record has non-zero timestamp information
+    has_time = max_values["hour"] > 0 or max_values["minute"] > 0 or max_values["second"] > 0
+    has_ms = has_time and (max_values["ms_E2"] > 0 or max_values["ms_E1"] > 0 or max_values["ms_E0"] > 0)
+    # determine min / max 5 values to map too low / too high values to
+    min11 = sorted([v for min11 in [j["min11"] for j in stats_list] for v in min11], reverse=False)[:11]
+    max11 = sorted([v for max11 in [j["max11"] for j in stats_list] for v in max11], reverse=True)[:11]
+    if value_protection:
+        # extreme value protection - discard lowest/highest 5 values
+        if len(min11) < 11 or len(max11) < 11:
+            # less than 11 subjects with non-NULL values; we need to protect all
+            min5 = []
+            max5 = []
+            has_time = False
+            has_ms = False
+        else:
+            min5 = [str(v) for v in min11[5:10]]  # drop 1 to 5th lowest; keep 6th to 10th lowest
+            max5 = [str(v) for v in max11[5:10]]  # drop 1 to 5th highest; keep 6th to 10th highest
+            # update min/max year based on first four letters of protected min/max dates
+            max_values["year"] = int(max5[0][0:4])
+            min_values["year"] = int(min5[0][0:4])
+    else:
+        min5 = min11[0:4]
+        max5 = max11[0:4]
+    # determine cardinalities
+    cardinalities = {}
+    if has_nan:
+        cardinalities["nan"] = 2  # binary
+    cardinalities["year"] = max_values["year"] + 1 - min_values["year"]
+    cardinalities["month"] = max_values["month"] + 1 - min_values["month"]
+    cardinalities["day"] = max_values["day"] + 1 - min_values["day"]
+    if has_time:
+        cardinalities["hour"] = max_values["hour"] + 1 - min_values["hour"]
+        cardinalities["minute"] = max_values["minute"] + 1 - min_values["minute"]
+        cardinalities["second"] = max_values["second"] + 1 - min_values["second"]
+    if has_ms:
+        cardinalities["ms_E2"] = max_values["ms_E2"] + 1 - min_values["ms_E2"]
+        cardinalities["ms_E1"] = max_values["ms_E1"] + 1 - min_values["ms_E1"]
+        cardinalities["ms_E0"] = max_values["ms_E0"] + 1 - min_values["ms_E0"]
+    stats = {
+        "cardinalities": cardinalities,
+        "has_nan": has_nan,
+        "has_time": has_time,
+        "has_ms": has_ms,
+        "min_values": min_values,
+        "max_values": max_values,
+        "min5": min5,
+        "max5": max5,
+    }
+    return stats
+
+
+def encode_datetime(values: pd.Series, stats: dict, _: pd.Series | None = None) -> pd.DataFrame:
+    # convert
+    values = safe_convert_datetime(values)
+    values = values.copy()
+    # reset index, as `values.mask` can throw errors for misaligned indices
+    values.reset_index(drop=True, inplace=True)
+    # replace extreme values with randomly sampled 5-th to 10-th largest/smallest values
+    min5 = stats["min5"] if len(stats["min5"]) > 0 else [0]
+    max5 = stats["max5"] if len(stats["max5"]) > 0 else [0]
+    min5 = pd.Series(min5, dtype=values.dtype)
+    max5 = pd.Series(max5, dtype=values.dtype)
+    values.mask(
+        values < min5[0],
+        min5.sample(n=len(values), replace=True, ignore_index=True),
+        inplace=True,
+    )
+    values.mask(
+        values > max5[0],
+        max5.sample(n=len(values), replace=True, ignore_index=True),
+        inplace=True,
+    )
+    # split to sub_columns
+    df = split_sub_columns_datetime(values)
+    is_not_nan = df["nan"] == 0
+    # encode values so that each datetime part ranges from 0 to `max_value-min_value`
+    for key in DATETIME_PARTS:
+        # subtract minimum value
+        df[key] = df[key].where(~is_not_nan, df[key] - stats["min_values"][key])
+        # ensure that any value is mapped onto valid value range
+        df[key] = np.minimum(df[key], stats["max_values"][key] - stats["min_values"][key])
+        df[key] = np.maximum(df[key], 0)
+    if not stats["has_nan"]:
+        df.drop(["nan"], inplace=True, axis=1)
+    if not stats["has_time"]:
+        df.drop(["hour", "minute", "second"], inplace=True, axis=1)
+    if not stats["has_ms"]:
+        df.drop(["ms_E2", "ms_E1", "ms_E0"], inplace=True, axis=1)
+    return df
+
+
+def split_sub_columns_datetime(values: pd.Series) -> pd.DataFrame:
+    if not is_date_dtype(values) and not is_timestamp_dtype(values):
+        raise ValueError("expected to be datetime")
+    values = values.astype("datetime64[us]")
+
+    # fast datetime part extraction taken from https://stackoverflow.com/a/56260054
+    dt = values.to_numpy()
+    parts = np.empty(dt.shape + (7,), dtype="u4")
+    year, month, day, hour, minute, second = (dt.astype(f"M8[{x}]") for x in "YMDhms")
+    parts[:, 0] = year + 1970  # Gregorian Year
+    parts[:, 1] = (month - year) + 1  # month
+    parts[:, 2] = (day - month) + 1  # dat
+    parts[:, 3] = (dt - day).astype("m8[h]")  # hour
+    parts[:, 4] = (dt - hour).astype("m8[m]")  # minute
+    parts[:, 5] = (dt - minute).astype("m8[s]")  # second
+    parts[:, 6] = (dt - second).astype("m8[us]")  # microsecond
+    # create pd.DataFrame with datetime parts
+    sub_columns = {
+        "nan": values.reset_index(drop=True).isna(),
+        "year": pd.Series(parts[:, 0]),
+        "month": pd.Series(parts[:, 1]),
+        "day": pd.Series(parts[:, 2]),
+        "hour": pd.Series(parts[:, 3]),
+        "minute": pd.Series(parts[:, 4]),
+        "second": pd.Series(parts[:, 5]),
+        "ms_E2": pd.Series(np.floor(parts[:, 6] / 100_000) % 10),
+        "ms_E1": pd.Series(np.floor((parts[:, 6] / 10_000) % 10)),
+        "ms_E0": pd.Series(np.floor((parts[:, 6] / 1_000) % 10)),
+    }
+    df = pd.DataFrame(sub_columns)
+    df = df.fillna(0)
+    df = df.astype("int")
+    return df
+
+
+def decode_datetime(df_encoded: pd.DataFrame, stats: dict):
+    # decode y/m/d components
+    y = df_encoded["year"] + stats["min_values"]["year"]
+    m = df_encoded["month"] + stats["min_values"]["month"]
+    d = df_encoded["day"] + stats["min_values"]["day"]
+    # fix invalid dates by setting these to last day of month
+    is_leap = y.apply(lambda x: calendar.isleap(x))
+    d[is_leap & (m == 2) & (d > 29)] = 29
+    d[~is_leap & (m == 2) & (d > 28)] = 28
+    d[((m == 4) | (m == 6) | (m == 9) | (m == 11)) & (d > 30)] = 30
+    # concatenate to datetime string
+    y = y.astype(str)
+    m = m.astype(str).str.zfill(2)
+    d = d.astype(str).str.zfill(2)
+    dt_format = "%Y-%m-%d"
+    values = y + "-" + m + "-" + d
+    if stats["has_time"]:
+        hh = (df_encoded["hour"] + stats["min_values"]["hour"]).astype(str).str.zfill(2)
+        mm = (df_encoded["minute"] + stats["min_values"]["minute"]).astype(str).str.zfill(2)
+        ss = (df_encoded["second"] + stats["min_values"]["second"]).astype(str).str.zfill(2)
+        dt_format += " %H:%M:%S"
+        values = values + " " + hh + ":" + mm + ":" + ss
+    if stats["has_ms"]:
+        ms2 = df_encoded["ms_E2"] + stats["min_values"]["ms_E2"]
+        ms1 = df_encoded["ms_E1"] + stats["min_values"]["ms_E1"]
+        ms0 = df_encoded["ms_E0"] + stats["min_values"]["ms_E0"]
+        ms = (100 * ms2 + 10 * ms1 + ms0).astype(str).str.zfill(3)
+        dt_format += ".%f"
+        values = values + "." + ms
+    if "nan" in df_encoded.columns:
+        values[df_encoded["nan"] == 1] = pd.NA
+    # replace extreme values with randomly sampled 5-th to 10-th largest/smallest values
+    if len(stats["min5"]) > 0 and len(stats["max5"]) > 0:
+        # format datetime with accordance to the expected unified format when reading from stats
+        min5 = [parser.parse(i).strftime(dt_format) for i in stats["min5"]]
+        max5 = [parser.parse(i).strftime(dt_format) for i in stats["max5"]]
+        is_too_low = values.notna() & (values < min5[0])
+        is_too_high = values.notna() & (values > max5[0])
+        values.loc[is_too_low] = np.random.choice(min5, size=sum(is_too_low))
+        values.loc[is_too_high] = np.random.choice(max5, size=sum(is_too_high))
+    elif "nan" in df_encoded.columns:
+        # set all values to NaN if no valid values were present
+        values[df_encoded["nan"] == 0] = pd.NA
+    # convert from string to datetime
+    values = pd.to_datetime(values)
+    if not stats["has_time"]:
+        values = pd.to_datetime(values.dt.date)
+    return values