labda 0.0.2__tar.gz → 0.0.4__tar.gz

{labda-0.0.2 → labda-0.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: labda
-Version: 0.0.2
+Version: 0.0.4
 Summary: Library for Advanced Behavioural Data Analysis
 Home-page: https://labda.josefheidler.cz
 License: 	CC-BY-SA-4.0
@@ -20,6 +20,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: Scientific/Engineering
 Requires-Dist: geopandas (>=0.14.2,<0.15.0)
+Requires-Dist: loguru (>=0.7.2,<0.8.0)
 Requires-Dist: openpyxl (>=3.1.2,<4.0.0)
 Requires-Dist: pandas (>=2.2.0,<3.0.0)
 Requires-Dist: pandera (>=0.18.0,<0.19.0)
@@ -44,7 +45,8 @@ TO-DO...
 TO-DO...
 
 ## License
-<p xmlns:cc="http://creativecommons.org/ns#" xmlns:dct="http://purl.org/dc/terms/"><a property="dct:title" rel="cc:attributionURL" href="https://labda.josefheidler.cz/">Library for Advanced Behavioural Data Analysis</a> by <a rel="cc:attributionURL dct:creator" property="cc:attributionName" href="https://josefheidler.cz/">Josef Heidler</a> is licensed under <a href="http://creativecommons.org/licenses/by/4.0/?ref=chooser-v1" target="_blank" rel="license noopener noreferrer" style="display:inline-block;">CC BY 4.0<img style="height:22px!important;margin-left:3px;vertical-align:text-bottom;" src="https://mirrors.creativecommons.org/presskit/icons/cc.svg?ref=chooser-v1"><img style="height:22px!important;margin-left:3px;vertical-align:text-bottom;" src="https://mirrors.creativecommons.org/presskit/icons/by.svg?ref=chooser-v1"></a></p>
+
+[Library for Advanced Behavioural Data Analysis](https://labda.josefheidler.cz/) by [Josef Heidler](https://labda.josefheidler.cz/) is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).
 
 ## Documentation
 

labda-0.0.4/README.md

@@ -0,0 +1,27 @@
+# LABDA: Library for Advanced Behavioural Data Analysis
+
+**LABDA is currently under development and may undergo significant API changes that could result in breaking changes.**
+
+## What is it?
+
+TO-DO...
+
+## Main Features
+
+TO-DO...
+
+## License
+
+[Library for Advanced Behavioural Data Analysis](https://labda.josefheidler.cz/) by [Josef Heidler](https://labda.josefheidler.cz/) is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).
+
+## Documentation
+
+TO-DO...
+
+## Getting Help, Discussion and Development
+
+TO-DO...
+
+## Contributing to labda
+
+TO-DO..

{labda-0.0.2 → labda-0.0.4}/labda/__init__.py

@@ -1 +1,4 @@
+from .logging import configure_logger
 from .structure import Collection, Column, Subject, merge_collections, merge_subjects
+
+configure_logger("ERROR")

{labda-0.0.2 → labda-0.0.4}/labda/assets/transportation_cut_points.py

@@ -8,7 +8,7 @@ CUT_POINTS = [
         "required_data": Column.SPEED,
         "units": "km/h",
         "cut_points": [
-            {"name": "walking/running", "max": 9},
+            {"name": "walk/run", "max": 9},
             {"name": "bicycle", "max": 35},
             {"name": "vehicle", "max": float("inf")},
         ],

labda-0.0.4/labda/logging.py

@@ -0,0 +1,54 @@
+import sys
+from pathlib import Path
+
+from loguru import logger
+
+from .structure.subject import Subject
+
+
+def color_level(record):
+    match record["level"].name:
+        case "ERROR":
+            color = "red"
+        case "SUCCESS":
+            color = "green"
+        case _:
+            color = "white"
+
+    return color
+
+
+def formatter(record):
+    color = color_level(record)
+    return (
+        f"<{color}>"
+        + "{time:YYYY-MM-DD HH:mm:ss} | {level} | {message}\n"
+        + f"</{color}>"
+    )
+
+
+def configure_logger(level: str):
+    logger.remove()
+    logger.add(
+        sys.stderr,
+        format=formatter,
+        colorize=True,
+        level=level,
+    )
+
+
+def success_parsing_log(
+    subject: Subject,
+    path: Path,
+    func: str,
+):
+    meta = subject.metadata
+    sampling_frequency = (
+        f"SF: {meta.sampling_frequency}s" if meta.sampling_frequency else ""
+    )
+    timezone = f", TZ: {meta.timezone}" if meta.timezone else ""
+    crs = f", CRS: {meta.crs}" if meta.crs else ""
+
+    logger.success(
+        f"{func} | {path} | Parsed {len(subject.df)} records ({sampling_frequency}{timezone}{crs})"
+    )

labda-0.0.4/labda/parallel.py

@@ -0,0 +1,41 @@
+import multiprocessing as mp
+from functools import partial
+from typing import Any, Callable
+
+from .logging import logger
+from .structure.subject import Subject
+
+
+def process_obj(obj: Any, func: Callable, **kwargs) -> Any | None:
+    try:
+        if isinstance(obj, Subject):
+            func(obj, **kwargs)
+            return obj
+
+        return func(obj, **kwargs)
+    except Exception as e:
+        logger.error(f"{func.__module__}:{func.__name__} | {obj} | {e}")
+
+
+def parallel_processing(
+    func: Callable,
+    objs: list[Any],
+    n_cores: int | str = "max",
+    **kwargs,
+) -> list[Any]:
+    max_cores = mp.cpu_count()
+
+    if n_cores == "max":
+        n_cores = max_cores
+    elif isinstance(n_cores, int) and n_cores > max_cores:
+        raise ValueError(f"Maximum number of cores is {max_cores}. Received {n_cores}.")
+    else:
+        raise ValueError(f"Invalid value for n_cores: {n_cores}.")
+
+    with mp.Pool(n_cores) as pool:
+        results = pool.map(
+            partial(process_obj, func=func, **kwargs),
+            objs,
+        )
+
+    return results

{labda-0.0.2 → labda-0.0.4}/labda/parsers/actigraph.py

@@ -1,36 +1,24 @@
 from datetime import datetime
 from pathlib import Path
-from typing import Any
+from typing import Any, Tuple
 
 import pandas as pd
 from dateutil.parser import parse as parse_dt
 
+from ..logging import success_parsing_log
 from ..structure.subject import Metadata, Sensor, Subject, Vendor
 from ..structure.validation.subject import SCHEMA, Column
-from ..utils import get_sampling_frequency
+from ..utils import (
+    align_datetimes,
+    filter_datetime,
+    get_sampling_frequency,
+)
+from .utils import set_timezone
 
 # TODO: Add param for column mapping if there is no header.
 # TODO: It is not perfect, but it is a start and it should be able to parse automatically almost all ActiGraph files from SDU.
 
 
-def filter_datetime(
-    df: pd.DataFrame,
-    start: datetime | None = None,
-    end: datetime | None = None,
-) -> pd.DataFrame:
-    if start and end:
-        if end < start:
-            raise ValueError("The end time must be after the start time.")
-
-    if start:
-        df = df[df[Column.DATETIME] >= start]
-
-    if end:
-        df = df[df[Column.DATETIME] <= end]
-
-    return df
-
-
 def get_metadata(metadata: list[str]) -> dict[str, Any]:
     model = metadata[0].split("ActiGraph")[-1].split()[0].strip()
     firmware = metadata[0].split("Firmware")[-1].split()[0].strip()
@@ -137,15 +125,15 @@ def from_csv(
     metadata: bool = True,
     *,
     subject_id: str | None = None,
-    sensor_id: str | None = None,
+    sensor_id: str | None = None,  # type: ignore
     serial_number: str | None = None,
     model: str | None = None,
     vendor: Vendor = Vendor.ACTIGRAPH,
     firmware_version: str | None = None,
     start: datetime | None = None,
     end: datetime | None = None,
-    timezone: str | None = None,
-    dt_format: str | None = None,
+    timezone: str | None | Tuple[str, str] = None,
+    datetime_format: str | None = None,
 ) -> Subject:
     if isinstance(path, str):
         path = Path(path)
@@ -183,17 +171,18 @@ def from_csv(
             Column.PERPENDICULAR_COUNTS,
         ]
 
-    df = handle_datetime(df, parsed_metadata, dt_format)
-    df = filter_datetime(df, start, end)
+    df = handle_datetime(df, parsed_metadata, datetime_format)
 
     df.set_index(Column.DATETIME, inplace=True)
-    sampling_frequency = parsed_metadata.get("sampling_frequency")
+    df, timezone = set_timezone(df, timezone)
 
-    df.dropna(how="all", inplace=True)
+    df = filter_datetime(df, start, end)
 
+    sampling_frequency = parsed_metadata.get("sampling_frequency")
     if not sampling_frequency:
         sampling_frequency = get_sampling_frequency(df)
-    df = df.resample(f"{sampling_frequency}s").nearest()
+
+    df = align_datetimes(df, sampling_frequency)
 
     columns = [col.value for col in Column]
     ordered_columns = [col for col in columns if col in df.columns]
@@ -229,4 +218,9 @@ def from_csv(
         timezone=timezone,
     )  # type: ignore
 
-    return Subject(metadata=metadata, df=df)  # type: ignore
+    subject = Subject(metadata=metadata, df=df)  # type: ignore
+
+    func = f"{__name__}:from_csv"
+    success_parsing_log(subject, path, func)
+
+    return subject

labda-0.0.4/labda/parsers/bulk.py

@@ -0,0 +1,39 @@
+from pathlib import Path
+from typing import Callable
+
+from ..parallel import parallel_processing, process_obj
+from ..structure import Collection
+
+# TODO: Add bulk parsing with linkage file
+
+
+def bulk_parser(
+    folder: str | Path,
+    parser: Callable,
+    id: str | None = None,
+    parallel: bool = True,
+    n_cores: int | str = "max",
+    **kwargs,
+):
+    if isinstance(folder, str):
+        folder = Path(folder)
+
+    if not folder.is_dir():
+        raise ValueError(f"'{folder}' is not a valid folder.")
+
+    files = list(Path(folder).glob("*"))
+
+    if id is None:
+        id = folder.name
+
+    if parallel:
+        results = parallel_processing(parser, files, n_cores, **kwargs)
+    else:
+        results = [process_obj(file, parser, **kwargs) for file in files]
+
+    # Remove None values from results
+    results = list(filter(None, results))
+
+    collection = Collection(id=id, subjects=results)
+
+    return collection

{labda-0.0.2 → labda-0.0.4}/labda/parsers/gadgetbridge.py

@@ -1,4 +1,3 @@
-import secrets
 from datetime import datetime
 from pathlib import Path
 
@@ -112,20 +111,15 @@ def _get_xiaomi_devices(
     return devices
 
 
-def _get_wear_by_hr(df: pd.DataFrame) -> pd.DataFrame:
-    # FIXME: Rework this function.
-    df = df.copy().reset_index(drop=True)
+def _get_wear_by_hr(df: pd.DataFrame, limit: int | None = None) -> pd.DataFrame:
+    df = df.copy()
     df[Column.WEAR] = pd.NA
-    indices = df[Column.HEART_RATE].dropna().index
-    intervals = indices.to_series().diff()
+    df.loc[df[Column.HEART_RATE].notna(), Column.WEAR] = True
 
-    if not intervals.isna().all():
-        interval = int(intervals.mode().iloc[0]) - 1
-        df.loc[df[Column.HEART_RATE].notna(), Column.WEAR] = True
+    if limit:
+        df[Column.WEAR] = df[Column.WEAR].bfill(limit=limit)
 
-        if not interval == 0:
-            df[Column.WEAR] = df[Column.WEAR].bfill(limit=interval)
-        df.loc[df[Column.WEAR].isna(), Column.WEAR] = False
+    df.loc[df[Column.WEAR].isna(), Column.WEAR] = False
 
     return df
 
@@ -168,8 +162,8 @@ def _generate_records(
     for record in records:
         id = record[0]  # type: ignore
         df = record[1]  # type: ignore
-        df = _get_wear_by_hr(df)
         df.set_index(Column.DATETIME, inplace=True)
+        df = _get_wear_by_hr(df, 4)
         df = SCHEMA.validate(df)
 
         sampling_frequency = get_sampling_frequency(df)

{labda-0.0.2 → labda-0.0.4}/labda/parsers/qstarz.py

@@ -1,22 +1,49 @@
 from datetime import datetime
 from pathlib import Path
+from typing import Tuple
 
+import numpy as np
 import pandas as pd
 
+from ..logging import success_parsing_log
 from ..structure.subject import Metadata, Sensor, Subject, Vendor
 from ..structure.validation.subject import SCHEMA, Column
 from ..utils import (
     align_datetimes,
     columns_exists,
-    convert_to_dataframe,
-    convert_to_geodataframe,
-    get_crs,
+    filter_datetime,
     get_sampling_frequency,
-    get_timezone,
 )
+from .utils import set_crs, set_timezone
 
 # TODO: Can be refactored.,
-# TODO: Remove all spaces from the column names, so there is no need to check if there is space or not.
+
+
+def detect_environment_from_gps_signal(
+    df: pd.DataFrame,
+    method: str,
+    limit: int | float,
+) -> pd.DataFrame:
+    df = df.copy()
+
+    methods = [
+        Column.SNR_USED,  # 225 (F1: 46,80)
+        Column.SNR_VIEWED,  # 260 (F1: 71,18)
+        Column.NSAT_USED,  # 7 (F1: 48,86)
+        Column.NSAT_VIEWED,  # 9 (F1: 42,93)
+        Column.NSAT_RATIO,  # 70 (F1: 46,40)
+    ]
+    if method not in methods:
+        raise ValueError(f"Method muset be one of {methods}")
+
+    if method not in df.columns:
+        raise ValueError(f"Column '{method}' does not exist in dataframe")
+
+    mask = df[method].notna()
+    df.loc[mask, Column.ENVIRONMENT] = "outdoor"
+    df.loc[mask & (df[method] <= limit), Column.ENVIRONMENT] = "indoor"
+
+    return df
 
 
 def parse_snr(df: pd.DataFrame) -> pd.DataFrame:
@@ -162,37 +189,10 @@ def parse_elevation(df: pd.DataFrame) -> pd.DataFrame:
     return df
 
 
-def filter_datetime(
-    df: pd.DataFrame,
-    start: datetime | None = None,
-    end: datetime | None = None,
-) -> pd.DataFrame:
-    if start and end:
-        if end < start:
-            raise ValueError("The end time must be after the start time.")
-
-    if start:
-        df = df[df[Column.DATETIME] >= start]
-
-    if end:
-        df = df[df[Column.DATETIME] <= end]
-
-    return df
-
-
-def handle_crs(df: pd.DataFrame, crs: str) -> pd.DataFrame:
-    gdf = convert_to_geodataframe(df, crs="WGS84")
-    gdf.to_crs(crs, inplace=True)
-    df = convert_to_dataframe(gdf)
-
-    return df
-
-
 def parse_datetime(df: pd.DataFrame, dt_format: str | None = None) -> pd.DataFrame:
     df = df.copy()
     column = None
 
-    # FIXME: Fix this...
     if "DATE" in df.columns and "TIME" in df.columns:
         date = "DATE"
         time = "TIME"
@@ -228,8 +228,15 @@ def parse_coordinates(df: pd.DataFrame) -> pd.DataFrame:
         [Column.LATITUDE, Column.LONGITUDE]
     ].apply(pd.to_numeric)
 
-    df.loc[df["N/S"] == "S", Column.LATITUDE] = -df[Column.LATITUDE]
-    df.loc[df["E/W"] == "W", Column.LONGITUDE] = -df[Column.LONGITUDE]
+    df["N/S"] = df["N/S"].str.strip()
+    df[Column.LATITUDE] = np.where(
+        df["N/S"] == "S", -df[Column.LATITUDE], df[Column.LATITUDE]
+    )
+
+    df["E/W"] = df["E/W"].str.strip()
+    df[Column.LONGITUDE] = np.where(
+        df["E/W"] == "W", -df[Column.LONGITUDE], df[Column.LONGITUDE]
+    )
 
     return df
 
@@ -279,9 +286,11 @@ def from_csv(
     start: datetime | None = None,
     end: datetime | None = None,
     sampling_frequency: float | None = None,
-    crs: str | None = None,
-    timezone: str | None = None,
-    dt_format: str | None = None,
+    crs: str | None | Tuple[str, str] = None,
+    timezone: str | None | Tuple[str, str] = None,
+    datetime_format: str | None = None,
+    nsat_ratio: float = 70,
+    snr_viewed: int = 260,
 ) -> Subject:
     if isinstance(path, str):
         path = Path(path)
@@ -299,15 +308,7 @@ def from_csv(
     remove_repeated_headers(df)
 
     df = parse_coordinates(df)
-    df = parse_datetime(df, dt_format)
-
-    if not crs:
-        crs = get_crs(df)
-
-    df = handle_crs(df, crs)
-
-    if not timezone:
-        timezone = get_timezone(df)
+    df = parse_datetime(df, datetime_format)
 
     df.drop_duplicates(
         subset=[Column.DATETIME], inplace=True
@@ -317,7 +318,6 @@ def from_csv(
     if "VALID" in df.columns:
         df = df[~(df["VALID"] == "Estimated (dead reckoning)")]
 
-    df = filter_datetime(df, start, end)
     df = parse_distance(df)
     df = parse_speed(df)
     df = parse_elevation(df)
@@ -330,7 +330,16 @@ def from_csv(
             (df[Column.NSAT_USED] * 100) / df[Column.NSAT_VIEWED]
         ).round(2)
 
+    if Column.SNR_VIEWED in df.columns and snr_viewed:
+        df = detect_environment_from_gps_signal(df, Column.SNR_VIEWED, snr_viewed)
+    elif Column.NSAT_RATIO in df.columns and nsat_ratio:
+        df = detect_environment_from_gps_signal(df, Column.NSAT_RATIO, nsat_ratio)
+
     df.set_index(Column.DATETIME, inplace=True)
+    df, timezone = set_timezone(df, timezone)
+    df, crs = set_crs(df, crs)
+
+    df = filter_datetime(df, start, end)
 
     if not sampling_frequency:
         sampling_frequency = get_sampling_frequency(df)
@@ -363,4 +372,9 @@ def from_csv(
         timezone=timezone,
     )
 
-    return Subject(metadata=metadata, df=df)
+    subject = Subject(metadata=metadata, df=df)
+
+    func = f"{__name__}:from_csv"
+    success_parsing_log(subject, path, func)
+
+    return subject

labda-0.0.4/labda/parsers/utils.py

@@ -0,0 +1,60 @@
+from typing import Tuple
+
+import pandas as pd
+
+from ..logging import logger
+from ..utils import change_crs, change_timezone, get_crs, get_timezone
+
+DEFAULT_CRS_SENSOR = "EPSG:4326"
+
+# TODO: Maybe those functions should be refactored, i.e. if (None, timezone) -> timezone would be guessed from the data and then changed to the target timezone. Same for CRS.
+
+
+def set_timezone(
+    df: pd.DataFrame,
+    timezone: str | None | Tuple[str, str] = None,
+) -> Tuple[pd.DataFrame, str | None]:
+    if (
+        isinstance(timezone, tuple)
+        and len(timezone) == 2
+        and all(isinstance(tz, str) for tz in timezone)
+    ):
+        source_tz = timezone[0]
+        target_tz = timezone[1]
+        df = change_timezone(df, source_tz, target_tz)
+        timezone = target_tz
+    elif isinstance(timezone, str):
+        pass
+    elif not timezone:
+        if "latitude" in df.columns and "longitude" in df.columns:
+            timezone = get_timezone(df)
+        else:
+            timezone = None
+    else:
+        raise ValueError(f"Invalid value for timezone: {timezone}")
+
+    return df, timezone
+
+
+def set_crs(
+    df: pd.DataFrame,
+    crs: str | None | Tuple[str, str] = None,
+) -> Tuple[pd.DataFrame, str]:
+    if (
+        isinstance(crs, tuple)
+        and len(crs) == 2
+        and all(isinstance(tz, str) for tz in crs)
+    ):
+        source_crs = crs[0]
+        target_crs = crs[1]
+        df = change_crs(df, source_crs, target_crs)
+        crs = target_crs
+    elif isinstance(crs, str):
+        df = change_crs(df, DEFAULT_CRS_SENSOR, crs)
+    elif not crs:
+        crs = get_crs(df)
+        df = change_crs(df, DEFAULT_CRS_SENSOR, crs)
+    else:
+        raise ValueError(f"Invalid value for crs: {crs}")
+
+    return df, crs

{labda-0.0.2 → labda-0.0.4}/labda/processing/spatial/manipulations.py

@@ -41,6 +41,7 @@ def stop_splitter(
 
     columns_not_exists(df, [name], overwrite=overwrite)
 
+    # TODO: How to handle NA values, maybe just filter to have only valid points (with coordinates).
     df = df.copy()
     df[name] = False
 

{labda-0.0.2 → labda-0.0.4}/labda/processing/spatial/timeline.py

@@ -11,6 +11,7 @@ def get_information(df: pd.DataFrame, crs: str) -> pd.Series:
     start = df.index.min()
     end = df.index.max()
     duration = end - start
+    segment_id = df["segment_id"].iloc[0]
 
     distance_col = get_unique_column_name(df)
     df = add_distance(df, crs=crs, name=distance_col)
@@ -28,7 +29,11 @@ def get_information(df: pd.DataFrame, crs: str) -> pd.Series:
             location_id = df["stationary_id"].iloc[0]
         else:
             status = "transport"
-            mode = df["trip_mode"].iloc[0]
+
+            if Column.TRIP_MODE in df.columns:
+                mode = df[Column.TRIP_MODE].iloc[0]
+            else:
+                mode = pd.NA
             location_id = pd.NA
     else:
         status = "stationary"
@@ -44,6 +49,7 @@ def get_information(df: pd.DataFrame, crs: str) -> pd.Series:
 
     return pd.Series(
         {
+            "segment_id": segment_id,
             "trip_id": trip_id,
             "stationary_id": location_id,
             "start": start,
@@ -74,7 +80,8 @@ def get_transport_information(df: pd.DataFrame, crs: str):
 
 
 def get_timeline(df: pd.DataFrame, crs: str) -> pd.DataFrame:
-    # TODO: Add a check for specific trip columns.
+    # TODO: Add a check for specific trip columns (that they are exists, so timeline can be calculated)
+    # TODO: Add indoor/outdoor trip detection, same one as in removal of indoor trips inside trips.py (it should mark whole trip if it was inside or not - maybe?)
     df = df.copy()
 
     trips = (
@@ -87,7 +94,11 @@ def get_timeline(df: pd.DataFrame, crs: str) -> pd.DataFrame:
         lambda x: get_information(x, crs=crs)
     )
 
-    summary = pd.concat([trips, stops], ignore_index=True).sort_values(["start", "end"])
+    summary = (
+        pd.concat([trips, stops], ignore_index=True)
+        .sort_values(["start", "end"])
+        .reset_index(drop=True)
+    )
 
     return summary.astype(
         {

{labda-0.0.2 → labda-0.0.4}/labda/processing/spatial/transportation.py

@@ -7,6 +7,9 @@ from ...expanders.spatial import add_speed
 from ...structure.validation.subject import Column
 from ...utils import columns_exists, columns_not_exists, get_unique_column_name
 
+# TODO: Add something, so even pause has a mode.
+# TODO: Option: pause_mode = True/False, before, after, mixed?
+
 
 def _set_trip_mode(df: pd.DataFrame, name: str):
     modes = df[name].value_counts()