labda 0.0.2__py3-none-any.whl

labda/structure/collection.py

@@ -0,0 +1,103 @@
+import multiprocessing as mp
+import secrets
+from functools import partial
+from pathlib import Path
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+from .subject import Subject
+
+
+def _process_subject(subject, func, **kwargs):
+    func(subject, **kwargs)
+
+    return subject
+
+
+class Collection(BaseModel):
+    id: Optional[str] = Field(default_factory=lambda: secrets.token_hex(4))
+    subjects: list[Subject] = Field(default_factory=list)
+
+    def __repr__(self):
+        subjects_repr = ", ".join([p.metadata.id for p in self.subjects])  # type: ignore
+        return f"Collection(id={self.id}, subjects=[{subjects_repr}])"
+
+    def add_subject(self, subject: Subject):
+        subjects_ids = [s.metadata.id for s in self.subjects]
+
+        if subject.metadata.id in subjects_ids:
+            raise ValueError(
+                f"Subject with id '{subject.metadata.id}' already exists in collection."
+            )
+
+        subject.collection = self.id  # type: ignore
+        self.subjects.append(subject)  # type: ignore
+
+    def get_subject(self, id: str) -> Subject:
+        for subject in self.subjects:
+            if subject.metadata.id == id:
+                return subject
+        raise ValueError(f"Subject with id '{id}' not found.")
+
+    @classmethod
+    def from_folder(cls, path: str | Path, id: str | None = None) -> "Collection":
+        if isinstance(path, str):
+            path = Path(path)
+
+        subjects = [Subject.from_parquet(file) for file in path.glob("*.parquet")]
+        return cls(id=id, subjects=subjects)
+
+    def to_folder(self, path: str | Path, overwrite: bool = False):
+        if isinstance(path, str):
+            path = Path(path)
+
+        path.mkdir(parents=True, exist_ok=True)
+        for subject in self.subjects:
+            subject.to_parquet(
+                path / f"{subject.metadata.id}.parquet", overwrite=overwrite
+            )
+
+    def detect_trips(self, **kwargs):
+        with mp.Pool(mp.cpu_count()) as pool:
+            self.subjects = pool.map(
+                partial(_process_subject, func=Subject.detect_trips, **kwargs),
+                self.subjects,
+            )
+
+    def detect_activity_intensity(self, **kwargs) -> None:
+        with mp.Pool(mp.cpu_count()) as pool:
+            self.subjects = pool.map(
+                partial(
+                    _process_subject, func=Subject.detect_activity_intensity, **kwargs
+                ),
+                self.subjects,
+            )
+
+    # TODO: Rework whole consistency check
+    # TODO: Add also check columns consistency
+    def _check_consistency(self, attribute, error_message):
+        # TODO: Add docstring, better name for method
+        values = [getattr(subject.metadata, attribute) for subject in self.subjects]
+        unique = set(values)
+
+        if len(unique) != 1:
+            print(f"{error_message}: {unique}")
+
+    def _check_consistent_sampling_frequencies(self):
+        self._check_consistency(
+            "sampling_frequency", "Sampling frequencies are not consistent"
+        )
+
+    def _check_consistent_crs(self):
+        self._check_consistency("crs", "CRS are not consistent")
+
+    def _check_consistent_timezones(self):
+        self._check_consistency("timezone", "Timezones are not consistent")
+
+    def check_consistency(self):
+        self._check_consistent_sampling_frequencies()
+        self._check_consistent_crs()
+        self._check_consistent_timezones()
+
+        print("Consistency check finished.")

labda/structure/domains.py

@@ -0,0 +1,29 @@
+from pathlib import Path
+
+import pandas as pd
+from shapely import wkt
+
+from .validation.domains import SCHEMA
+
+
+def from_csv(path: str | Path) -> pd.DataFrame:
+    # TODO: Check datetime overlap.
+    # TODO: Add column for "importance" in case of overlapping domains.
+    if isinstance(path, str):
+        path = Path(path)
+
+    domains = pd.read_csv(
+        path,
+        dtype={
+            "subject_id": "string",
+            "domain": "string",
+            "geometry": "string",
+        },
+        delimiter=";",
+        parse_dates=["start", "end"],
+        date_format="%d-%m-%Y %H:%M:%S",
+    )
+
+    domains["geometry"] = domains["geometry"].apply(wkt.loads)
+
+    return SCHEMA.validate(domains)

labda/structure/linkage.py

@@ -0,0 +1,28 @@
+from pathlib import Path
+
+import pandas as pd
+from shapely import wkt
+
+from .validation.linkage import SCHEMA
+
+
+def from_csv(path: str | Path) -> pd.DataFrame:
+    if isinstance(path, str):
+        path = Path(path)
+
+    df = pd.read_csv(
+        path,
+        dtype={
+            "subject_id": "string",
+            "sensor_id": "string",
+        },
+        delimiter=";",
+    )
+
+    if "start" in df.columns:
+        df["start"] = pd.to_datetime(df["start"])
+
+    if "end" in df.columns:
+        df["end"] = pd.to_datetime(df["end"])
+
+    return SCHEMA.validate(df)

labda/structure/merging.py

@@ -0,0 +1,133 @@
+import pandas as pd
+
+from .collection import Collection
+from .subject import Metadata, Subject
+
+
+def _check_duplicate_cols(left: pd.DataFrame, right: pd.DataFrame) -> None:
+    duplicate_cols = set(left.columns) & set(right.columns)
+    if duplicate_cols:
+        raise ValueError(f"Before merging, remove duplicate columns: {duplicate_cols}")
+
+
+def _check_ids(left: Subject, right: Subject) -> str:
+    if left.metadata.id != right.metadata.id:
+        raise ValueError(f"IDs do not match: {left.metadata.id} != {right.metadata.id}")
+
+    return left.metadata.id
+
+
+def _check_sampling_frequency(left: Subject, right: Subject) -> float:
+    if left.metadata.sampling_frequency != right.metadata.sampling_frequency:
+        raise ValueError(
+            f"Sampling frequency mismatch: {left.metadata.sampling_frequency} != {right.metadata.sampling_frequency}"
+        )
+
+    return left.metadata.sampling_frequency
+
+
+def _check_timezones(left: Subject, right: Subject) -> str | None:
+    if (
+        (left.metadata.timezone)
+        and (right.metadata.timezone)
+        and left.metadata.timezone != right.metadata.timezone
+    ):
+        raise ValueError(
+            f"Timezone mismatch: {left.metadata.timezone} != {right.metadata.timezone}"
+        )
+
+    return left.metadata.timezone or right.metadata.timezone
+
+
+def _check_crs(left: Subject, right: Subject) -> str | None:
+    if (
+        (left.metadata.crs)
+        and (right.metadata.crs)
+        and left.metadata.crs != right.metadata.crs
+    ):
+        raise ValueError(f"CRS mismatch: {left.metadata.crs} != {right.metadata.crs}")
+
+    return left.metadata.crs or right.metadata.crs
+
+
+def _get_subjects(collection: Collection) -> list[str]:
+    return [subject.metadata.id for subject in collection.subjects]
+
+
+def merge_subjects(left: Subject, right: Subject, **kwargs) -> Subject:
+    if not isinstance(left, Subject) and not isinstance(right, Subject):
+        raise ValueError(f"Unsupported types: {type(left)} and {type(right)}")
+
+    id = _check_ids(left, right)
+    sf = _check_sampling_frequency(left, right)
+    tz = _check_timezones(left, right)
+    crs = _check_crs(left, right)
+
+    metadata = Metadata(
+        id=id,
+        sensor=left.metadata.sensor + right.metadata.sensor,
+        sampling_frequency=sf,
+        timezone=tz,
+        crs=crs,
+    )
+
+    # FIXME: Add suffixes to columns, even if not duplicated
+    if not kwargs.get("suffixes"):
+        _check_duplicate_cols(left.df, right.df)
+    merged = pd.merge(left.df, right.df, left_index=True, right_index=True, **kwargs)
+
+    return Subject(metadata=metadata, df=merged)
+
+
+def merge_collections(
+    left: Collection,
+    right: Collection,
+    *,
+    keep: bool = False,
+    **kwargs,
+) -> Collection:
+    if not isinstance(left, Collection) and not isinstance(right, Collection):
+        raise ValueError(f"Unsupported types: {type(left)} and {type(right)}")
+
+    left_ids = _get_subjects(left)
+    right_ids = _get_subjects(right)
+    subjects = set(left_ids + right_ids)
+
+    collection = Collection()
+
+    for subject in subjects:
+        try:
+            left_subject = left.get_subject(subject)
+        except Exception:
+            left_subject = None
+
+        try:
+            right_subject = right.get_subject(subject)
+        except Exception:
+            right_subject = None
+
+        if left_subject and right_subject:
+            merged_subject = merge_subjects(left_subject, right_subject, **kwargs)
+            collection.add_subject(merged_subject)
+        else:
+            print(
+                f"Error while merging | {subject} | Subject not found in both collections."
+            )
+
+            # FIXME: Fix, it is not working
+            # FIXME: Suffix to check if exists
+            # FIXME: Native pd.merge function adds suffixes to columns with the same name otherwise not (no suffixe and we don't know the origin of the column)
+
+            if keep:
+                if left_subject:
+                    # suffix = kwargs.get("suffixes")
+                    # if suffix:
+                    #     left_subject.df = left_subject.df.add_suffix(suffix[0])
+                    collection.add_subject(left_subject)
+                elif right_subject:
+                    # suffix = kwargs.get("suffixes")
+                    # if suffix:
+                    #     right_subject.df = right_subject.df.add_suffix(suffix[1])
+                    collection.add_subject(right_subject)
+
+    return collection

labda/structure/resampling.py

@@ -0,0 +1,82 @@
+from typing import Any
+
+import pandas as pd
+
+from ..structure.validation.subject import Column
+
+# TODO: Create one big configuration with SCHEMA (validation) and RESAMPLING
+
+
+def mode(x):
+    vals = x.to_list()
+    return max(vals, key=vals.count)
+
+
+UPSAMPLING = [
+    {"column": Column.SUBJECT_ID, "method": "first"},
+    {"column": Column.WEAR, "method": mode},
+    {"column": Column.TIMEDELTA, "method": "sum"},
+    # ---
+    {"column": Column.VERTICAL_COUNTS, "method": "sum"},
+    {"column": Column.HORIZONTAL_COUNTS, "method": "sum"},
+    {"column": Column.PERPENDICULAR_COUNTS, "method": "sum"},
+    {"column": Column.VM_COUNTS, "method": "mean"},
+    {"column": Column.POSITION, "method": mode},
+    {"column": Column.STEPS, "method": "sum"},
+    # ---
+    {"column": Column.LATITUDE, "method": "first"},
+    {"column": Column.LONGITUDE, "method": "first"},
+    {"column": Column.NSAT_VIEWED, "method": "first"},
+    {"column": Column.NSAT_USED, "method": "first"},
+    {"column": Column.NSAT_RATIO, "method": "first"},
+    {"column": Column.SNR_VIEWED, "method": "first"},
+    {"column": Column.SNR_USED, "method": "first"},
+    {"column": Column.PDOP, "method": "first"},
+    {"column": Column.HDOP, "method": "first"},
+    {"column": Column.VDOP, "method": "first"},
+    # ---
+    {"column": Column.DISTANCE, "method": "sum"},
+    {"column": Column.ELEVATION, "method": "first"},
+    {"column": Column.SPEED, "method": "mean"},
+    {"column": Column.ACCELERATION, "method": "mean"},
+    {"column": Column.DIRECTION, "method": "first"},
+    {"column": Column.ENVIRONMENT, "method": mode},
+    # ---
+    {"column": Column.HEART_RATE, "method": "mean"},
+    {"column": Column.LUX, "method": "mean"},
+    {"column": Column.ACTIVITY_INTENSITY, "method": mode},
+]
+
+
+def upsample(
+    df: pd.DataFrame,
+    from_sf: float,
+    to_sf: float,
+    mapper: list[dict[str, Any]] | None = None,
+) -> pd.DataFrame:
+    if from_sf > to_sf:
+        raise ValueError(f"Downsampling is not supported | {from_sf} > {to_sf}")
+    elif from_sf == to_sf:
+        return df
+
+    df = df.copy()
+
+    if not mapper:
+        mapper = UPSAMPLING
+
+    resampling = {v["column"]: v["method"] for v in mapper if v["column"] in df.columns}
+
+    dropped = [
+        colname for colname in df.columns if colname not in resampling.keys()
+    ]  # TODO: Check that it works properly.
+    if dropped:
+        print(
+            f"Columns that are not in the resampling configuration will be dropped | Dropping: {dropped}"
+        )  # TODO: Write warning what columns are dropped.
+
+    df = df.resample(f"{to_sf}s").agg(resampling)  # type: ignore
+    df.dropna(
+        axis=0, how="all", inplace=True
+    )  # Dropping rows with all NaN values - they were created by resampling.
+
+    return df

labda/structure/subject.py

@@ -0,0 +1,294 @@
+from datetime import timedelta
+from enum import StrEnum
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Optional
+
+if TYPE_CHECKING:
+    # This is a workaround to avoid circular imports.
+    # TODO: Is there a better way to do this?
+    from .collection import Collection
+
+import pandas as pd
+import pyarrow as pa
+import pyarrow.parquet as pq
+from pydantic import BaseModel, Field
+
+from ..expanders import (
+    add_acceleration,
+    add_direction,
+    add_distance,
+    add_speed,
+    add_timedelta,
+)
+from ..processing import detect_domains
+from ..processing.accelerometer import detect_activity_intensity, detect_wear
+from ..processing.spatial import detect_transportation, detect_trips, get_timeline
+from ..structure.resampling import upsample
+from ..utils import columns_exists, columns_not_exists
+from .validation.domains import SCHEMA as DOMAINS_SCHEMA
+from .validation.subject import SCHEMA, Column
+
+
+class Vendor(StrEnum):
+    ACTIGRAPH = "ActiGraph"
+    XIAOMI = "Xiaomi"
+    SENS = "SENS"
+    GARMIN = "Garmin"
+    QSTARZ = "Qstarz"
+    GGIR = "GGIR"
+    SENSECAP = "SenseCap"
+    ANDROID = "Android"
+    IOS = "iOS"
+
+
+class Sensor(BaseModel):
+    id: str
+    serial_number: str | None = None
+    model: str | None = None
+    vendor: Vendor | None = None
+    firmware_version: str | None = None
+    extra: dict[str, Any] | None = None
+
+    class Config:
+        coerce_numbers_to_str = True
+
+
+class Metadata(BaseModel):
+    id: str
+    sensor: list[Sensor] = Field(default_factory=list)
+    sampling_frequency: float = Field(ge=0, description="Sampling frequency in seconds")
+    crs: str | None = None
+    timezone: str | None = None
+
+    class Config:
+        coerce_numbers_to_str = True
+
+
+class Subject(BaseModel):
+    metadata: Metadata
+    collection: Optional["Collection"] = None
+    df: pd.DataFrame
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    def __str__(self):
+        return str(f"{self.metadata}\n{self.df}")
+
+    @property
+    def timeline(self):
+        if not self.metadata.crs:
+            raise ValueError("Records object does not have a crs defined.")
+
+        columns_exists(
+            self.df,
+            [Column.TRIP_ID, Column.TRIP_STATUS, Column.STATIONARY_ID],
+        )
+
+        return get_timeline(self.df, crs=self.metadata.crs)
+
+    @property
+    def domains(self):
+        # TODO: Implement this property.
+        raise NotImplementedError("This method is not implemented yet.")
+
+    def to_parquet(
+        self,
+        path: str | Path,
+        *,
+        overwrite: bool = False,
+    ) -> None:
+        if isinstance(path, str):
+            path = Path(path)
+
+        if path.exists() and not overwrite:
+            raise FileExistsError(
+                f"The file '{path}' already exists. If you want to overwrite it, set the 'overwrite' argument to 'True'."
+            )
+        else:
+            path.parent.mkdir(parents=True, exist_ok=True)
+
+        custom_metadata = {"labda".encode(): self.metadata.model_dump_json().encode()}
+        self.validate()
+        table = pa.Table.from_pandas(self.df)
+
+        existing_metadata = table.schema.metadata
+        combined_meta = {**custom_metadata, **existing_metadata}
+
+        table = table.replace_schema_metadata(combined_meta)
+        pq.write_table(table, path)
+
+    @classmethod
+    def from_parquet(cls, path: str | Path) -> "Subject":
+        if isinstance(path, str):
+            path = Path(path)
+
+        table = pq.read_table(path)
+        df = table.to_pandas()
+        custom_metadata = Metadata.model_validate_json(
+            table.schema.metadata["labda".encode()]
+        )
+        cls = cls(metadata=custom_metadata, df=df)
+        cls.validate()
+
+        return cls
+
+    def validate(
+        self,
+        *,
+        extra_columns: bool = False,
+    ):
+        self.df = SCHEMA.validate(self.df)
+
+        # Order columns as defined in Column
+        records_columns = [col.value for col in Column]
+        ordered_columns = [col for col in records_columns if col in self.df.columns]
+
+        # Append extra columns that are not in Column at the end, alphabetically
+        if extra_columns:
+            extra = sorted(set(self.df.columns) - set(records_columns))
+            ordered_columns.extend(extra)
+
+        self.df = self.df[ordered_columns]
+
+    def add_timedelta(
+        self,
+        *,
+        name: str = Column.TIMEDELTA,
+        overwrite: bool = False,
+    ):
+        self.df = add_timedelta(self.df, name=name, overwrite=overwrite)
+
+    def add_distance(
+        self,
+        *,
+        name: str = Column.DISTANCE,
+        overwrite: bool = False,
+    ):
+        self.df = add_distance(
+            self.df, crs=self.metadata.crs, name=name, overwrite=overwrite
+        )
+
+    def add_speed(
+        self,
+        *,
+        name: str = Column.SPEED,
+        overwrite: bool = False,
+    ):
+        self.df = add_speed(
+            self.df, crs=self.metadata.crs, name=name, overwrite=overwrite
+        )
+
+    def add_acceleration(
+        self,
+        *,
+        name: str = Column.ACCELERATION,
+        overwrite: bool = False,
+    ):
+        self.df = add_acceleration(
+            self.df, crs=self.metadata.crs, name=name, overwrite=overwrite
+        )
+
+    def add_direction(
+        self,
+        *,
+        name: str = Column.DIRECTION,
+        overwrite: bool = False,
+    ):
+        self.df = add_direction(self.df, name=name, overwrite=overwrite)
+
+    def detect_trips(
+        self,
+        cut_points: dict[str, Any] | None = None,
+        *,
+        gap_duration: timedelta,
+        stop_radius: int | float,
+        stop_duration: timedelta,
+        window: int | None = 3,
+        pause_radius: int | float | None = None,
+        pause_duration: timedelta | None = None,
+        min_duration: timedelta | None = None,
+        min_length: int | float | None = None,
+        min_distance: int | float | None = None,
+        max_speed: int | float | None = None,
+        overwrite: bool = False,
+    ) -> None:
+        if not self.metadata.crs:
+            raise ValueError("Records object does not have a CRS defined.")
+
+        self.df = detect_trips(
+            self.df,
+            crs=self.metadata.crs,
+            sampling_frequency=self.metadata.sampling_frequency,
+            overwrite=overwrite,
+            gap_duration=gap_duration,
+            stop_radius=stop_radius,
+            stop_duration=stop_duration,
+            pause_radius=pause_radius,
+            pause_duration=pause_duration,
+            min_duration=min_duration,
+            min_length=min_length,
+            min_distance=min_distance,
+            max_speed=max_speed,
+        )
+
+        if cut_points:
+            self.df = detect_transportation(
+                self.df,
+                self.metadata.crs,
+                cut_points,
+                window=window,
+                overwrite=overwrite,
+            )
+
+    def detect_domains(
+        self,
+        domains: pd.DataFrame,
+        *,
+        overwrite: bool = False,
+    ) -> None:
+        domains = DOMAINS_SCHEMA.validate(domains)
+        subject_domains = domains[
+            (domains[Column.SUBJECT_ID] == self.metadata.id)
+            | (domains[Column.SUBJECT_ID].isna())
+        ]
+        self.df = detect_domains(self.df, subject_domains, overwrite=overwrite)
+
+    def detect_activity_intensity(
+        self,
+        cut_points: dict[str, Any],
+        *,
+        overwrite: bool = False,
+    ) -> None:
+        self.df = detect_activity_intensity(
+            self.df,
+            cut_points,
+            self.metadata.sampling_frequency,
+            overwrite=overwrite,
+        )
+
+    def detect_wear(
+        self,
+        min_duration: timedelta,
+        interruption_duration: timedelta,
+        *,
+        overwrite: bool = False,
+    ) -> None:
+        self.df = detect_wear(
+            self.df,
+            self.metadata.sampling_frequency,
+            min_duration,
+            interruption_duration=interruption_duration,
+            overwrite=overwrite,
+        )
+
+    def upsample(
+        self,
+        sampling_frequency: float,
+        *,
+        mapper: list[dict[str, Any]] | None = None,
+    ) -> None:
+        self.df = upsample(
+            self.df, self.metadata.sampling_frequency, sampling_frequency, mapper
+        )
+        self.metadata.sampling_frequency = sampling_frequency