pyrregular 0.1.0__tar.gz

pyrregular-0.1.0/PKG-INFO

@@ -0,0 +1,8 @@
+Metadata-Version: 2.1
+Name: pyrregular
+Version: 0.1.0
+Author: Francesco Spinnato, Cristiano Landi
+Author-email: francesco.spinnato@di.unipi.it
+Requires-Python: >=3.9
+Provides-Extra: test
+Provides-Extra: notebooks

pyrregular-0.1.0/README.md

@@ -0,0 +1 @@
+# pyrregular

pyrregular-0.1.0/pyrregular/accessor.py

@@ -0,0 +1,184 @@
+import numpy as np
+import xarray as xr
+from pyrregular.conversion_utils import reset_time_index, ak_dropnan, fill_time_index
+import sparse
+import awkward as ak
+
+
+
+@xr.register_dataarray_accessor("irr")
+class IrregularAccessor:
+    def __init__(self, da):
+        self._da = da
+        self.dims = {dim: i for i, dim in enumerate(da.dims)}
+
+    def __getitem__(self, key):
+        import numpy as np
+
+        out = self._da.__getitem__(key)
+        if out["time_id"].size == 1:
+            return out
+        dims = {dim: i for i, dim in enumerate(out.dims)}
+        return out[..., np.sort(out.data.coords[dims["time_id"]])]
+        # return out[..., np.sort(out.data.coords[-1])]
+
+    def get_task(self, task="default"):
+        return self._da.attrs["configs"][task]
+
+    def get_task_target_and_split(self, task="default"):
+        task = self.get_task(task)
+        return self._da[task["target"]].data, self._da[task["split"]].data
+
+    def reset_time_index(
+        self,
+        ts_level=True,
+        index_scale=1e-9,
+        absolute_time=True,
+        concatenate_time=False,
+        normalize_time=False,
+    ):
+        return reset_time_index(
+            arr=self._da.data,
+            time_id=self._da["time_id"].data,
+            ts_level=ts_level,
+            ts_idx=self.dims["ts_id"],
+            signal_idx=self.dims["signal_id"],
+            time_idx=self.dims["time_id"],
+            index_scale=index_scale,
+            absolute_time=absolute_time,
+            concatenate_time=concatenate_time,
+            normalize_time=normalize_time
+        )
+
+    def to_dense(
+        self,
+        reset_time_index=True,
+        ts_level=True,
+        index_scale=1e-9,
+        absolute_time=True,
+        concatenate_time=False,
+        normalize_time=False,
+    ):
+        if reset_time_index:
+            X, T = self.reset_time_index(
+                ts_level=ts_level,
+                index_scale=index_scale,
+                absolute_time=absolute_time,
+                concatenate_time=concatenate_time,
+                normalize_time=normalize_time
+            )
+        else:
+            X = self._da.data
+            T = self._da["time_id"].data.reshape(1, 1, -1)
+            if concatenate_time:
+                X = sparse.concatenate([X, T], axis=1)
+        return X.todense(), T.todense()
+
+    def to_tslearn(
+        self,
+        reset_time_index=True,
+        ts_level=True,
+        index_scale=1e-9,
+        absolute_time=True,
+        concatenate_time=False,
+    ):
+        X, T = self.to_dense(
+            reset_time_index=reset_time_index,
+            ts_level=ts_level,
+            index_scale=index_scale,
+            absolute_time=absolute_time,
+            concatenate_time=concatenate_time,
+        )
+        return np.swapaxes(X, 1, 2), np.swapaxes(T, 1, 2)
+
+    def to_aeon(
+        self,
+        reset_time_index=True,
+        ts_level=True,
+        index_scale=1e-9,
+        absolute_time=True,
+        concatenate_time=False,
+    ):
+        X, T = self.to_dense(
+            reset_time_index=reset_time_index,
+            ts_level=ts_level,
+            index_scale=index_scale,
+            absolute_time=absolute_time,
+            concatenate_time=concatenate_time,
+        )
+        return X, T
+
+    def to_sktime(
+        self,
+        reset_time_index=True,
+        ts_level=True,
+        index_scale=1e-9,
+        absolute_time=True,
+        concatenate_time=False,
+    ):
+        X, T = self.to_dense(
+            reset_time_index=reset_time_index,
+            ts_level=ts_level,
+            index_scale=index_scale,
+            absolute_time=absolute_time,
+            concatenate_time=concatenate_time,
+        )
+        return X, T
+
+    def to_awkward(
+        self,
+        reset_time_index=True,
+        ts_level=True,
+        index_scale=1e-9,
+        absolute_time=True,
+        concatenate_time=False,
+        dropna=True,
+    ):
+        X, T = self.to_dense(
+            reset_time_index=reset_time_index,
+            ts_level=ts_level,
+            index_scale=index_scale,
+            absolute_time=absolute_time,
+            concatenate_time=concatenate_time,
+        )
+        X = ak.Array(X)
+        T = ak.Array(T)
+        if dropna:
+            X = ak_dropnan(X)
+            T = ak_dropnan(T)
+        return X, T
+
+    def to_list(
+        self,
+        reset_time_index=True,
+        ts_level=True,
+        index_scale=1e-9,
+        absolute_time=True,
+        concatenate_time=False,
+        dropna=True,
+    ):
+        X, T = self.to_awkward(
+            reset_time_index=reset_time_index,
+            ts_level=ts_level,
+            index_scale=index_scale,
+            absolute_time=absolute_time,
+            concatenate_time=concatenate_time,
+            dropna=dropna,
+        )
+        return X.to_list(), T.to_list()
+
+    def to_long(self, reset_time_index=True, ts_level=True, index_scale=1e-9, absolute_time=True):
+        if reset_time_index:
+            X, _ = self.reset_time_index(
+                ts_level=ts_level,
+                index_scale=index_scale,
+                absolute_time=absolute_time,
+                concatenate_time=False,
+            )
+            return np.concatenate([X.coords, X.data[np.newaxis, :]], axis=0).T
+        else:
+            T = self._da["time_id"].data.astype(np.float_)[self._da.data.coords[self.dims["time_id"]]] * index_scale
+            if not absolute_time:
+                T = T - T[0]
+            return np.concatenate([self._da.data.coords, T[np.newaxis, :]], axis=0).T
+

pyrregular-0.1.0/pyrregular/backend.py

@@ -0,0 +1,18 @@
+from xarray.backends import BackendEntrypoint
+from pyrregular.io_utils import load_from_file
+from xarray.core.indexing import NdArrayLikeIndexingAdapter
+from pyrregular.accessor import IrregularAccessor  # this is needed to register the accessor
+
+
+class IrregularEntrypoint(BackendEntrypoint):
+    def open_dataset(
+        self,
+        filename_or_obj,
+        *,
+        drop_variables=None,
+    ):
+        da = load_from_file(filename_or_obj)
+        da.data = NdArrayLikeIndexingAdapter(da.data)
+        da.name = "data"
+        return da.to_dataset()
+

pyrregular-0.1.0/pyrregular/conversion_utils.py

@@ -0,0 +1,124 @@
+import numba as nb
+import numpy as np
+import sparse
+import awkward as ak
+from copy import deepcopy
+
+
+
+@nb.njit
+def dense_rank(a):
+    unique_values = np.unique(a)
+    ranks = np.searchsorted(unique_values, a)
+    return ranks
+
+
+@nb.njit
+def find_breakpoints(a):
+    return np.hstack(
+        (np.array([0]), np.argwhere(np.diff(a)).ravel() + 1, np.array([len(a)]))
+    )
+
+
+@nb.njit
+def find_2d_breakpoints(a, b):
+    return np.hstack(
+        (
+            np.array([0]),
+            np.argwhere(np.diff(a) | np.diff(b)).ravel() + 1,
+            np.array([len(a)]),
+        )
+    )
+
+
+@nb.njit(parallel=True)
+def remove_fill_values_from_time_idx(
+    coords, ts_level=True, ts_idx=0, signal_idx=1, time_idx=-1
+):
+    if ts_level:
+        breakpoints = find_breakpoints(coords[ts_idx, :])
+    else:
+        breakpoints = find_2d_breakpoints(coords[ts_idx, :], coords[signal_idx, :])
+    out = coords.copy()
+    for i in nb.prange(len(breakpoints) - 1):
+        start = breakpoints[i]
+        end = breakpoints[i + 1]
+        dense_time = dense_rank(coords[time_idx, start:end])
+        out[time_idx, start:end] = dense_time
+    return out
+
+
+def reset_time_index(arr: sparse.COO, time_id: np.ndarray, ts_level=True, ts_idx=0, signal_idx=1, time_idx=-1,
+                     index_scale=1e-9, absolute_time=True, concatenate_time=False, normalize_time=False):
+    new_coords = remove_fill_values_from_time_idx(
+        arr.coords,
+        ts_level=ts_level,
+        ts_idx=ts_idx,
+        signal_idx=signal_idx,
+        time_idx=time_idx,
+    )
+    new_time_idx = sparse.COO(
+        coords=new_coords,
+        data=time_id.astype(np.float_)[arr.coords[time_idx]] * index_scale,
+        fill_value=arr.fill_value,
+    )
+    if ts_level:
+        new_time_idx = sparse.nanmax(new_time_idx, axis=signal_idx, keepdims=True)
+    if not absolute_time:
+        new_time_idx = new_time_idx - new_time_idx[:, :, 0:1]
+    if normalize_time:
+        abs_mean = sparse.nanmean(new_time_idx)
+        new_time_idx = (new_time_idx - sparse.nanmin(new_time_idx, axis=2, keepdims=True))
+        new_time_idx = new_time_idx / (sparse.nanmax(new_time_idx, axis=2, keepdims=True) + (abs_mean * 1e-8))  # avoids divisions by 0 when there is only 1 timestamp
+    if concatenate_time:
+        return sparse.concatenate(
+            [sparse.COO(coords=new_coords, data=arr.data, fill_value=arr.fill_value), new_time_idx], axis=1
+        ), new_time_idx
+    else:
+        return (
+            sparse.COO(coords=new_coords, data=arr.data, fill_value=arr.fill_value),
+            new_time_idx,
+        )
+
+
+def ak_dropnan(arr, axis=None):
+    return ak.drop_none(ak.nan_to_none(arr), axis=axis)
+
+
+def to_pypots(X, y=None):
+    if y is None:
+        return dict(
+            X=np.swapaxes(X, 1, 2),
+        )
+    else:
+        return dict(
+            X=np.swapaxes(X, 1, 2),
+            y=y,
+        )
+
+
+def to_tslearn(X):
+    return X.swapaxes(1, 2)
+
+
+def fill_time_index(arr):
+    T = deepcopy(arr)
+
+    # time delta
+    a_diff = T[:, :, 1:] - T[:, :, :-1]
+
+    # mean time delta
+    delta_mean = np.nanmean(a_diff, axis=2, keepdims=True)
+
+    # last timestep
+    last_valid_t = np.nanmax(T, axis=2, keepdims=True)
+
+    # find where the nans are
+    nan_mask = np.isnan(T)
+
+    # where nans are there is an increasing value from 1 to the last nan
+    nan_indices = np.cumsum(nan_mask, axis=2)
+
+    replacement = (last_valid_t + delta_mean * nan_indices)[nan_mask]
+    T[nan_mask] = replacement
+    return T

pyrregular-0.1.0/pyrregular/data_utils.py

@@ -0,0 +1,51 @@
+import pandas as pd
+import numpy as np
+import pathlib
+
+HUGGINGFACE_REPO_NAME = "pyrregular"
+DATA_ORIGINAL_FOLDER = "data" + "/" + HUGGINGFACE_REPO_NAME + "/" + "data_raw"
+DATA_INTERMEDIATE_FOLDER = "data" + "/" + HUGGINGFACE_REPO_NAME + "/" + "data_mid"
+DATA_FINAL_FOLDER = "data" + "/" + HUGGINGFACE_REPO_NAME + "/" + "data_final"
+METADATA_FOLDER = "metadata"
+
+
+def get_project_root() -> pathlib.Path:
+    return pathlib.Path(__file__).parent
+
+
+def data_original_folder():
+    return get_project_root() / ".." / DATA_ORIGINAL_FOLDER
+
+
+def data_intermediate_folder():
+    return get_project_root() / ".." / DATA_INTERMEDIATE_FOLDER
+
+
+def data_final_folder():
+    return get_project_root() / ".." / DATA_FINAL_FOLDER
+
+
+def metadata_folder():
+    return get_project_root() / ".." / METADATA_FOLDER
+
+
+def list_final_datasets():
+    return sorted(list(data_final_folder().glob("*.h5")))
+
+
+def list_metadata_files():
+    return sorted(list(metadata_folder().glob("*.yml")))
+
+
+def infer_static_columns(df, id_column, dropna=False):
+    df_grouped = df.groupby(id_column).nunique(dropna=dropna)
+    return [
+        c
+        for c in df_grouped.columns
+        if np.array_equal(df_grouped[c].to_numpy(), np.ones(len(df_grouped[c])))
+        and c != id_column
+    ]
+
+
+if __name__ == "__main__":
+    out = list_final_datasets()

pyrregular-0.1.0/pyrregular/datasets/abf.py

@@ -0,0 +1,180 @@
+from scipy.stats import skewnorm
+import numpy as np
+import pandas as pd
+
+from pyrregular.data_utils import data_original_folder, data_final_folder
+from tqdm.auto import tqdm
+from pathlib import Path
+from pyrregular.io_utils import load_yaml, read_csv
+from scipy.stats import skew
+from pyrregular.reader_interface import ReaderInterface
+from xarray import DataArray
+from sklearn.preprocessing import LabelEncoder
+
+
+LABEL_MAP = {0: "bowl", 1: "alembic", 2: "flask"}
+
+RANDOM_STATE = 42
+
+
+class Abf(ReaderInterface):
+    fast_to_test = True
+
+    @staticmethod
+    def read_original_version(verbose=False):
+        return read_abf(verbose=verbose)
+
+    @staticmethod
+    def _fix_intermediate_version(data: DataArray, verbose=True) -> DataArray:
+        split_default = ["train" if i == 1 else "test" for i in data["split"]]
+        data = data.assign_coords(split_default=("ts_id", split_default))
+        # the following is necessary to make the data compatible with some model benchmarks
+        data = data.assign_coords(time_id=("time_id", (data["time_id"].to_numpy() * 100_000_000).astype(int)))
+        data = data.rename({"class_default": "class_labels"})
+        data = data.rename({"y": "class_default"})
+        return data
+
+
+def _sample_skew_data(a, size):
+    return skewnorm.rvs(a, size=size)
+
+
+def _shape_function(n):
+    x_values = np.linspace(0, 1, n)
+    theta_values = x_values * np.pi
+    y_values = -np.sin(theta_values)
+    y_values = (y_values - y_values.mean()) / y_values.std()
+    return y_values
+
+
+def _generate_time_instances(instances_per_class, size, skewness):
+    values = np.sort(
+        [_sample_skew_data(skewness, size) for _ in range(instances_per_class)]
+    )
+    values = (values - values.min(axis=1, keepdims=True)) / (
+        values.max(axis=1, keepdims=True) - values.min(axis=1, keepdims=True)
+    )
+    return values
+
+
+def _generate_abf_data(
+    instances_per_class=10,
+    size=128,
+    skewness=10,
+    random_state=None,
+    noise=True,
+    noise_level=0.15,
+    n_train_instances_per_class=10,
+):
+    np.random.seed(random_state)
+    y = _shape_function(n=size)
+    y = y.reshape(1, -1)
+    Y = np.repeat(y, instances_per_class * 3, axis=0)
+    if noise:
+        Y += np.random.normal(0, noise_level, size=(instances_per_class * 3, size))
+    cylinders_t = _generate_time_instances(instances_per_class, size, 0)
+    bells_t = _generate_time_instances(instances_per_class, size, skewness)
+    funnels_t = _generate_time_instances(instances_per_class, size, -skewness)
+    split_vector = _get_train_test_split(
+        instances_per_class * 3, n_train_instances_per_class
+    )
+    times = np.concatenate((cylinders_t, bells_t, funnels_t))
+    skewness_real = skew(times, axis=1)
+    return (
+        times,
+        np.concatenate(
+            (
+                np.zeros(instances_per_class),
+                np.ones(instances_per_class),
+                np.full(instances_per_class, 2),
+            )
+        ),
+        Y,
+        skewness_real,
+        split_vector,
+    )
+
+
+def _generate_long_abf(
+    instances_per_class=10,
+    size=128,
+    skewness=10,
+    random_state=None,
+    noise=True,
+    noise_level=0.15,
+):
+    return _abf_to_long(
+        *_generate_abf_data(
+            instances_per_class, size, skewness, random_state, noise, noise_level
+        )
+    )
+
+
+def _save_long_icbf(
+    instances_per_class=10,
+    size=128,
+    skewness=10,
+    random_state=RANDOM_STATE,
+    noise=True,
+    noise_level=0.25,
+):
+    df = _generate_long_abf(
+        instances_per_class, size, skewness, random_state, noise, noise_level
+    )
+    df.to_csv(data_original_folder() / "alembics_bowls_flasks" / "abf_long.csv", index=False)
+    return
+
+
+def _abf_to_long(t, y, X, skewness, split_vector):
+    df = pd.DataFrame()
+    for i in tqdm(range(X.shape[0])):
+        for j in range(X.shape[1]):
+            df = pd.concat(
+                [
+                    df,
+                    pd.DataFrame(
+                        {
+                            "time_id": t[i, j],
+                            "y": int(y[i]),
+                            "value": X[i, j],
+                            "class_default": LABEL_MAP[y[i]],
+                            "skewness": skewness[i],
+                            "ts_id": str(i),
+                            "signal_id": str(0),
+                            "split": split_vector[i],
+                        },
+                        index=[0],
+                    ),
+                ]
+            )
+    return df
+
+
+def _get_train_test_split(n_instances, n_instances_train_per_class=10):
+    starting_idxs = np.arange(0, n_instances, n_instances // 3)
+    split_vector = np.zeros(n_instances, dtype=int)
+    for idx in starting_idxs:
+        split_vector[idx : idx + n_instances_train_per_class] = 1
+    return split_vector
+
+
+def read_abf(verbose=False):
+    return read_csv(
+        filenames=[data_original_folder() / "alembics_bowls_flasks/abf_long.csv"],
+        ts_id="ts_id",
+        time_id="time_id",
+        signal_id="signal_id",
+        value_id="value",
+        dims={
+            "ts_id": ["y", "class_default", "split", "skewness"],
+            "signal_id": [],
+            "time_id": [],
+        },
+        verbose=verbose,
+        time_index_as_datetime=False,
+    )
+
+
+if __name__ == "__main__":
+    # Abf.save_fixed()
+    df = Abf.load_final_version()

pyrregular-0.1.0/pyrregular/datasets/ais.py

@@ -0,0 +1,92 @@
+import pandas as pd
+from pyrregular.io_utils import (
+    read_csv,
+    load_yaml,
+    save_to_file,
+    load_from_file,
+)
+from pyrregular.data_utils import data_original_folder, data_final_folder
+from pathlib import Path
+from xarray import DataArray
+
+from pyrregular.data_utils import data_original_folder
+from pyrregular.io_utils import (
+    read_csv,
+    load_yaml,
+)
+from pyrregular.reader_interface import ReaderInterface
+import numpy as np
+from sklearn.model_selection import train_test_split
+
+
+class Ais(ReaderInterface):
+    @staticmethod
+    def read_original_version(verbose=False):
+        return read_ais(verbose=verbose)
+
+    @staticmethod
+    def _fix_intermediate_version(data: DataArray, verbose=True) -> DataArray:
+        # we use the length of the vessel as the class label (3 sizes)
+        labels = np.digitize(data["length"].to_numpy(), bins=[80, 100])
+        train_indices, test_indices = train_test_split(
+            np.arange(len(data["length"])),
+            test_size=0.3,
+            stratify=labels,
+            random_state=42,
+        )
+        split = np.array(["train"] * len(data["length"]))
+        split[test_indices] = "test"
+
+        data = data.assign_coords(
+            split_default=("ts_id", split), class_default=("ts_id", labels)
+        )
+        return data
+
+
+def _dataset_ais(filenames: list):
+    # Read the CSV file into a Pandas DataFrame
+    for filename in filenames:
+        df = pd.read_parquet(filename)
+        df["date_time_utc"] = pd.to_datetime(
+            df["date_time_utc"], format="%Y-%m-%d %H:%M:%S"
+        ).apply(
+            lambda x: int(x.timestamp())
+        )  # FIXME
+        for i in range(len(df)):
+            row = (
+                df.iloc[i : i + 1]
+                .melt(id_vars=["mmsi", "date_time_utc", "imo_nr", "length"])
+                .to_dict(orient="records")
+            )
+            for new_row in row:
+                yield new_row
+
+
+def read_ais(verbose=False):
+    attrs = load_yaml(
+        str(
+            data_original_folder()
+            / "terrestrial_vessel_automatic_identification_system/attrs.yml"
+        )
+    )
+    return read_csv(
+        filenames=[
+            data_original_folder()
+            / "terrestrial_vessel_automatic_identification_system"
+            / f"2020-2_group_{i}"
+            for i in range(40)
+        ],
+        ts_id="mmsi",
+        time_id="date_time_utc",
+        signal_id="variable",
+        value_id="value",
+        dims={"ts_id": ["imo_nr", "length"], "signal_id": [], "time_id": []},
+        reader_fun=_dataset_ais,
+        attrs=attrs,
+        verbose=verbose,
+    )
+
+
+if __name__ == "__main__":
+    # Ais.save_fixed()
+    df = Ais.load_final_version()