labda 0.0.2__py3-none-any.whl

CHANGELOG.md

@@ -0,0 +1,16 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+
+- Parser: Qstarz
+- Basic subject and collection manipulaton.
+- Expanding data with timedelta, speed, acceleration, distance, direction.
+- Getting sampling frequency, CRS and timezone automatically from dataset.
+- Initial documentation with mkdocs and material theme.

labda/__init__.py

@@ -0,0 +1 @@
+from .structure import Collection, Column, Subject, merge_collections, merge_subjects

labda/assets/__init__.py

@@ -0,0 +1,2 @@
+from .counts_cut_points import CUT_POINTS as COUNTS_CUT_POINTS
+from .transportation_cut_points import CUT_POINTS as TRANSPORTATION_CUT_POINTS

labda/assets/counts_cut_points.py

@@ -0,0 +1,35 @@
+from ..structure.validation.subject import Column
+
+CUT_POINTS = [
+    {
+        "id": 1,
+        "name": "Freedson Adult (1998)",
+        "reference": "https://journals.lww.com/acsm-msse/fulltext/1998/05000/calibration_of_the_computer_science_and.21.aspx",
+        "sampling_frequency": 60,
+        "category": "adult",
+        "placement": "hip",
+        "required_data": Column.VERTICAL_COUNTS,
+        "cut_points": [
+            {"name": "sedentary", "max": 99},
+            {"name": "light", "max": 1951},
+            {"name": "moderate", "max": 5724},
+            {"name": "vigorous", "max": 9498},
+            {"name": "very_vigorous", "max": float("inf")},
+        ],
+    },
+    {
+        "id": 2,
+        "name": "Freedson Adult VM3 (2011)",
+        "reference": "https://doi.org/10.1016/j.jsams.2011.04.003",
+        "sampling_frequency": 60,
+        "category": "adult",
+        "placement": "hip",
+        "required_data": Column.VM_COUNTS,
+        "cut_points": [
+            {"name": "light", "max": 2689},
+            {"name": "moderate", "max": 6166},
+            {"name": "vigorous", "max": 9642},
+            {"name": "very_vigorous", "max": float("inf")},
+        ],
+    },
+]

labda/assets/transportation_cut_points.py

@@ -0,0 +1,16 @@
+from ..structure.validation.subject import Column
+
+CUT_POINTS = [
+    {
+        "id": 1,
+        "name": "Carlson PALMS Validation (2015)",
+        "reference": "https://doi.org/10.1249/MSS.0000000000000446",
+        "required_data": Column.SPEED,
+        "units": "km/h",
+        "cut_points": [
+            {"name": "walking/running", "max": 9},
+            {"name": "bicycle", "max": 35},
+            {"name": "vehicle", "max": float("inf")},
+        ],
+    },
+]

labda/expanders/__init__.py

@@ -0,0 +1,8 @@
+from .extras import add_timedelta
+from .spatial import (
+    add_acceleration,
+    add_direction,
+    add_distance,
+    add_elevation,
+    add_speed,
+)

labda/expanders/accelerometer.py

@@ -0,0 +1,29 @@
+import numpy as np
+import pandas as pd
+
+from ..structure.validation.subject import Column
+from ..utils import columns_exists, columns_not_exists
+
+
+def add_counts_vector_magnitude(
+    df: pd.DataFrame,
+    *,
+    name: str = Column.VM_COUNTS,
+    overwrite: bool = False,
+) -> pd.DataFrame:
+    counts_cols = [
+        Column.VERTICAL_COUNTS,
+        Column.HORIZONTAL_COUNTS,
+        Column.PERPENDICULAR_COUNTS,
+    ]
+
+    columns_exists(df, counts_cols)  # type: ignore
+    columns_not_exists(df, [name], overwrite=overwrite)
+
+    df = df.copy()
+    df[name] = np.linalg.norm(
+        df[counts_cols].astype(float),
+        axis=1,
+    )
+
+    return df

labda/expanders/extras.py

@@ -0,0 +1,33 @@
+import pandas as pd
+
+from ..structure.validation.subject import Column
+from ..utils import columns_not_exists
+
+
+def add_timedelta(
+    df: pd.DataFrame,
+    *,
+    name: str = Column.TIMEDELTA,
+    overwrite: bool = False,
+) -> pd.DataFrame:
+    """
+    Adds a timedelta column to the DataFrame.
+
+    This function calculates the time difference between each row and the previous row and adds it as a new column to the DataFrame.
+
+    Args:
+        df (pd.DataFrame): The DataFrame to which to add the timedelta column.
+        name (str, optional): The name of the new timedelta column. Defaults to "timedelta".
+        overwrite (bool, optional): Whether to overwrite the existing timedelta column if it exists. Defaults to False.
+
+    Returns:
+        pd.DataFrame: The DataFrame with the added timedelta column.
+    """
+
+    columns_not_exists(df, [name], overwrite=overwrite)
+
+    df = df.copy()
+
+    df[name] = df.index.diff()  # type: ignore
+
+    return df

labda/expanders/spatial.py

@@ -0,0 +1,242 @@
+import numpy as np
+import pandas as pd
+
+from ..structure.validation.subject import Column
+from ..utils import (
+    columns_exists,
+    columns_not_exists,
+    convert_to_geodataframe,
+    get_crs,
+    get_crs_units,
+    get_unique_column_name,
+)
+from .extras import add_timedelta
+
+__all__ = [
+    "add_speed",
+    "add_distance",
+    "add_acceleration",
+    "add_direction",
+    "add_elevation",
+    # "add_weather",
+    # "add_address",
+]
+
+
+def add_speed(
+    df: pd.DataFrame,
+    *,
+    crs: str | None = None,
+    name: str = Column.SPEED,
+    overwrite: bool = False,
+) -> pd.DataFrame:
+    """
+    Adds a speed column to the DataFrame by calculating the speed based on the distance and time.
+    Units are based on the CRS. If CRS is not specified, the CRS will be guessed based on the coordinates (WGS84 by default).
+
+    Args:
+        df (pd.DataFrame): The input DataFrame.
+        crs (str | None, optional): The coordinate reference system. Defaults to None.
+        name (str, optional): The name of the speed dataframe.Column. Defaults to "speed".
+        overwrite (bool, optional): Whether to overwrite the existing speed column if it already exists.
+            Defaults to False.
+
+    Returns:
+        pd.DataFrame: The DataFrame with the speed column added.
+
+    Raises:
+        ValueError: If the speed column already exists in the DataFrame and overwrite is set to False.
+    """
+
+    columns_not_exists(df, [name], overwrite=overwrite)
+
+    df = df.copy()
+
+    timedelta_colname = get_unique_column_name(df)
+    df = add_timedelta(df, name=timedelta_colname)
+
+    distance_colname = get_unique_column_name(df)
+    df = add_distance(df, crs=crs, name=distance_colname)
+
+    df[name] = df[distance_colname] / df[timedelta_colname].dt.total_seconds()
+
+    # TODO: This is temporary fix for changing m/s to km/h for now.
+    if crs:
+        units = get_crs_units(crs)
+
+        if units == "metre":
+            df[name] = df[name] * 3.6
+
+    return df.drop(columns=[timedelta_colname, distance_colname])
+
+
+def add_distance(
+    df: pd.DataFrame,
+    *,
+    crs: str | None = None,
+    name: str = Column.DISTANCE,
+    overwrite: bool = False,
+) -> pd.DataFrame:
+    """
+    Adds a column to the DataFrame with the distance between consecutive points.
+    Units are based on the CRS. If CRS is not specified, the CRS will be guessed based on the coordinates (WGS84 by default).
+
+    Args:
+        df (pd.DataFrame): The input DataFrame.
+        crs (str | None, optional): The coordinate reference system of the DataFrame. Defaults to None.
+        name (str, optional): The name of the new dataframe.Column. Defaults to "distance".
+        overwrite (bool, optional): Whether to overwrite the existing column with the same name. Defaults to False.
+
+    Returns:
+        pd.DataFrame: The DataFrame with the added distance dataframe.Column.
+
+    Raises:
+        ValueError: If the column with the specified name already exists in the DataFrame and overwrite is False.
+    """
+
+    columns_exists(df, [Column.LATITUDE, Column.LONGITUDE])
+    columns_not_exists(df, [name], overwrite=overwrite)
+
+    df = df.copy()
+    gdf = convert_to_geodataframe(df, crs=crs)
+    if gdf.crs is None:
+        crs = get_crs(gdf)
+        gdf = gdf.set_crs(crs)
+
+    df[name] = gdf.distance(gdf.shift(1))  # type: ignore
+
+    return df
+
+
+def add_acceleration(
+    df: pd.DataFrame,
+    *,
+    crs: str | None = None,
+    name: str = Column.ACCELERATION,
+    overwrite: bool = False,
+) -> pd.DataFrame:
+    """
+    Adds a column to the DataFrame with the acceleration between consecutive points.
+    Acceleration is calculated based on the change in speed over time.
+    Units are based on the CRS. If CRS is not specified, the CRS will be guessed based on the coordinates (WGS84 by default).
+
+    Args:
+        df (pd.DataFrame): The input DataFrame.
+        crs (str | None, optional): The coordinate reference system of the DataFrame. Defaults to None.
+        name (str, optional): The name of the new dataframe.Column. Defaults to "acceleration".
+        overwrite (bool, optional): Whether to overwrite the existing column with the same name. Defaults to False.
+
+    Returns:
+        pd.DataFrame: The DataFrame with the added acceleration dataframe.Column.
+
+    Raises:
+        ValueError: If the column with the specified name already exists in the DataFrame and overwrite is False.
+    """
+
+    # TODO: Needs to be tested.
+    columns_not_exists(df, [name], overwrite=overwrite)
+
+    df = df.copy()
+
+    speed_colname = get_unique_column_name(df)
+    df = add_speed(df, crs=crs, name=speed_colname)
+
+    timedelta_colname = get_unique_column_name(df)
+    df = add_timedelta(df, name=timedelta_colname)
+
+    df[name] = (df[speed_colname] - df[speed_colname].shift()) / df[
+        timedelta_colname
+    ].dt.total_seconds()
+
+    return df.drop(columns=[speed_colname, timedelta_colname])
+
+
+def add_direction(
+    df: pd.DataFrame,
+    *,
+    name: str = Column.DIRECTION,
+    overwrite: bool = False,
+) -> pd.DataFrame:
+    """
+    Adds a column to the DataFrame with the direction between consecutive points.
+    Direction is calculated based on the change in longitude and latitude.
+
+    Args:
+        df (pd.DataFrame): The input DataFrame.
+        name (str, optional): The name of the new dataframe.Column. Defaults to "direction".
+        overwrite (bool, optional): Whether to overwrite the existing column with the same name. Defaults to False.
+
+    Returns:
+        pd.DataFrame: The DataFrame with the added direction dataframe.Column.
+
+    Raises:
+        ValueError: If the column with the specified name already exists in the DataFrame and overwrite is False.
+    """
+
+    # TODO: Needs to be tested.
+    columns_exists(df, [Column.LATITUDE, Column.LONGITUDE])
+    columns_not_exists(df, [name], overwrite=overwrite)
+
+    df = df.copy()
+
+    # Convert to radians
+    lon1, lat1 = np.radians(df[Column.LONGITUDE]), np.radians(df[Column.LATITUDE])
+    lon2, lat2 = (
+        np.radians(df[Column.LONGITUDE].shift()),
+        np.radians(df[Column.LATITUDE].shift()),
+    )
+
+    dlon = lon2 - lon1
+
+    x = np.cos(lat2) * np.sin(dlon)
+    y = np.cos(lat1) * np.sin(lat2) - np.sin(lat1) * np.cos(lat2) * np.cos(dlon)
+
+    direction = np.arctan2(x, y)
+    direction = np.degrees(direction)
+    direction = (direction + 360) % 360
+
+    df[name] = direction
+
+    return df
+
+
+def add_elevation(
+    df: pd.DataFrame,
+    *,
+    name: str = Column.ELEVATION,
+    overwrite: bool = False,
+) -> pd.DataFrame:
+    # TODO: Add elevation.
+
+    columns_not_exists(df, [name], overwrite=overwrite)
+
+    raise NotImplementedError("This method needs to be implemented.")
+
+
+# def add_weather(
+#     df: pd.DataFrame,
+#     *,
+#     name: str = "weather",
+#     overwrite: bool = False,
+# ) -> pd.DataFrame:
+#     # TODO: https://open-meteo.com/en/docs/historical-weather-api/
+#     # TODO: Add weather.
+
+#     if not overwrite:
+#         check_columns_existence(df, [name], exists=False)
+
+#     raise NotImplementedError("This method needs to be implemented.")
+
+
+# def add_address(
+#     df: pd.DataFrame,
+#     *,
+#     name: str = "address",
+#     overwrite: bool = False,
+# ) -> pd.DataFrame:
+#     # TODO: Add address.
+
+#     if not overwrite:
+#         check_columns_existence(df, [name], exists=False)
+
+#     raise NotImplementedError("This method needs to be implemented.")

labda/parsers/__init__.py

@@ -0,0 +1,4 @@
+from . import gadgetbridge as Gadgetbridge
+from . import qstarz as Qstarz
+from . import actigraph as Actigraph
+from .bulk import bulk_parser

labda/parsers/actigraph.py

@@ -0,0 +1,232 @@
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+from dateutil.parser import parse as parse_dt
+
+from ..structure.subject import Metadata, Sensor, Subject, Vendor
+from ..structure.validation.subject import SCHEMA, Column
+from ..utils import get_sampling_frequency
+
+# TODO: Add param for column mapping if there is no header.
+# TODO: It is not perfect, but it is a start and it should be able to parse automatically almost all ActiGraph files from SDU.
+
+
+def filter_datetime(
+    df: pd.DataFrame,
+    start: datetime | None = None,
+    end: datetime | None = None,
+) -> pd.DataFrame:
+    if start and end:
+        if end < start:
+            raise ValueError("The end time must be after the start time.")
+
+    if start:
+        df = df[df[Column.DATETIME] >= start]
+
+    if end:
+        df = df[df[Column.DATETIME] <= end]
+
+    return df
+
+
+def get_metadata(metadata: list[str]) -> dict[str, Any]:
+    model = metadata[0].split("ActiGraph")[-1].split()[0].strip()
+    firmware = metadata[0].split("Firmware")[-1].split()[0].strip()
+    serial_number = metadata[1].split()[-1].strip()
+    start_time = metadata[2].split()[-1].strip()
+    start_date = metadata[3].split()[-1].strip()
+    start_datetime = parse_dt(start_date + " " + start_time)
+    sampling_frequency = pd.to_timedelta(
+        metadata[4].split()[-1].strip()
+    ).total_seconds()
+
+    return {
+        "model": model,
+        "firmware": firmware,
+        "serial_number": serial_number,
+        "start_datetime": start_datetime,
+        "sampling_frequency": sampling_frequency,
+    }
+
+
+def handle_dataframe_header(data: list[str]) -> pd.DataFrame:
+    data_header = data[0].split(",")
+    data = data[1:]
+    df = pd.DataFrame(data)
+    df = df[0].str.split(",", expand=True)
+    df.columns = data_header
+    df.columns = df.columns.str.strip().str.lower()
+
+    column_mapping = {
+        "epoch": "time",
+        "axis1": Column.VERTICAL_COUNTS,
+        "axis2": Column.HORIZONTAL_COUNTS,
+        "axis3": Column.PERPENDICULAR_COUNTS,
+        "activity": Column.VERTICAL_COUNTS,
+        "activity (horizontal)": Column.HORIZONTAL_COUNTS,
+        "3rd axis": Column.PERPENDICULAR_COUNTS,
+        "vector magnitude": Column.VM_COUNTS,
+        "vm": Column.VM_COUNTS,
+        "steps": Column.STEPS,
+        "lux": Column.LUX,
+        "inclinometer off": "non-wear",
+        "inclinometer standing": "standing",
+        "inclinometer sitting": "sitting",
+        "inclinometer lying": "lying",
+    }
+
+    for old_column, new_column in column_mapping.items():
+        if old_column in df.columns:
+            df.rename(columns={old_column: new_column}, inplace=True)
+
+    if Column.VM_COUNTS in df.columns and df[Column.VM_COUNTS].dtype == object:
+        df[Column.VM_COUNTS] = df[Column.VM_COUNTS].str.replace('"', "")
+
+    return df
+
+
+def handle_inclinometer(df: pd.DataFrame) -> pd.DataFrame:
+    df = df.copy()
+    columns = ["non-wear", "standing", "sitting", "lying"]
+    exists_columns = []
+
+    for column in columns:
+        if column in df.columns:
+            exists_columns.append(column)
+
+    if exists_columns:
+        df[Column.POSITION] = df[exists_columns].idxmax(axis=1)
+
+    if "non-wear" in exists_columns:
+        df[Column.WEAR] = True
+        df.loc[df[Column.POSITION] == "non-wear", Column.WEAR] = False
+        df.loc[df[Column.WEAR] == False, Column.POSITION] = pd.NA
+
+    df.drop(columns=exists_columns, inplace=True)
+
+    return df
+
+
+def handle_datetime(
+    df: pd.DataFrame,
+    metadata: dict[str, Any],
+    dt_format: str | None = None,
+) -> pd.DataFrame:
+    df = df.copy()
+
+    if "date" in df.columns and "time" in df.columns:
+        df[Column.DATETIME] = pd.to_datetime(
+            df["date"] + " " + df["time"], format=dt_format
+        )
+        df.drop(columns=["date", "time"], inplace=True)
+    elif metadata.get("start_datetime") and metadata.get("sampling_frequency"):
+        start_datetime = metadata["start_datetime"]
+        sampling_frequency = metadata["sampling_frequency"]
+        df[Column.DATETIME] = start_datetime + pd.to_timedelta(
+            df.index * sampling_frequency, unit="s"
+        )
+
+    return df
+
+
+def from_csv(
+    path: str | Path,
+    line: int = 10,
+    metadata: bool = True,
+    *,
+    subject_id: str | None = None,
+    sensor_id: str | None = None,
+    serial_number: str | None = None,
+    model: str | None = None,
+    vendor: Vendor = Vendor.ACTIGRAPH,
+    firmware_version: str | None = None,
+    start: datetime | None = None,
+    end: datetime | None = None,
+    timezone: str | None = None,
+    dt_format: str | None = None,
+) -> Subject:
+    if isinstance(path, str):
+        path = Path(path)
+
+    if not path.is_file():
+        raise ValueError(f"Invalid file path: {path}")
+
+    if path.suffix != ".csv":
+        raise ValueError(f"Invalid file extension: {path.suffix}")
+
+    with path.open("r") as f:
+        file = f.read()
+
+    lines = file.splitlines()
+
+    if metadata:
+        parsed_metadata = get_metadata(lines[0:line])
+    else:
+        parsed_metadata = {}
+
+    data = lines[line:]
+
+    if any(char.isalpha() for char in data[0]):
+        df = handle_dataframe_header(data)
+        df = handle_inclinometer(df)
+    else:
+        df = pd.DataFrame(
+            data,
+        )
+        df = df[0].str.split(",", expand=True)
+        df = df.iloc[:, :3]
+        df.columns = [
+            Column.VERTICAL_COUNTS,
+            Column.HORIZONTAL_COUNTS,
+            Column.PERPENDICULAR_COUNTS,
+        ]
+
+    df = handle_datetime(df, parsed_metadata, dt_format)
+    df = filter_datetime(df, start, end)
+
+    df.set_index(Column.DATETIME, inplace=True)
+    sampling_frequency = parsed_metadata.get("sampling_frequency")
+
+    df.dropna(how="all", inplace=True)
+
+    if not sampling_frequency:
+        sampling_frequency = get_sampling_frequency(df)
+    df = df.resample(f"{sampling_frequency}s").nearest()
+
+    columns = [col.value for col in Column]
+    ordered_columns = [col for col in columns if col in df.columns]
+    df = df[ordered_columns]
+
+    df = SCHEMA.validate(df)
+
+    sensor_id = (
+        sensor_id if sensor_id else parsed_metadata.get("serial_number") or path.stem
+    )  # type: str
+    serial_number = (
+        serial_number if serial_number else parsed_metadata.get("serial_number")
+    )
+    model = model if model else parsed_metadata.get("model")
+    firmware_version = (
+        firmware_version if firmware_version else parsed_metadata.get("firmware")
+    )
+
+    subject_id = subject_id or path.stem
+
+    sensor = Sensor(
+        id=sensor_id,
+        serial_number=serial_number,
+        model=model,
+        vendor=vendor,
+        firmware_version=firmware_version,
+    )  # type: ignore
+
+    metadata = Metadata(
+        id=subject_id,
+        sensor=[sensor],
+        sampling_frequency=sampling_frequency,
+        timezone=timezone,
+    )  # type: ignore
+
+    return Subject(metadata=metadata, df=df)  # type: ignore

labda/parsers/bulk.py

@@ -0,0 +1,31 @@
+from pathlib import Path
+from typing import Callable
+
+from labda.structure import Collection
+
+# TODO: Add bulk parsing with linkage file
+# TODO: Add parallel processing of files.
+
+
+def bulk_parser(folder: str | Path, parser: Callable, id: str | None = None, **kwargs):
+    if isinstance(folder, str):
+        folder = Path(folder)
+
+    if not folder.is_dir():
+        raise ValueError(f"'{folder}' is not a valid folder.")
+
+    files = list(Path(folder).glob("*"))
+
+    if id is None:
+        id = folder.name
+
+    collection = Collection(id=id)
+
+    for file in files:
+        try:
+            subject = parser(file, **kwargs)
+            collection.add_subject(subject)
+        except Exception as e:
+            print(f"Error while parsing | {file} | {e}")  # TODO: Log error
+
+    return collection