gpxtractor 0.1.0__tar.gz

gpxtractor-0.1.0/LICENSE

@@ -0,0 +1,7 @@
+Copyright (c) 2026 Charles Stapylton-Smith
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

gpxtractor-0.1.0/PKG-INFO

@@ -0,0 +1,79 @@
+Metadata-Version: 2.4
+Name: gpxtractor
+Version: 0.1.0
+Summary: GPX, TCX and FIT data extraction for Python
+Author-email: Charlie Stapylton <278091496+c-stap@users.noreply.github.com>
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: lxml>=6.0.2
+Requires-Dist: numpy>=2.3.1
+Requires-Dist: pyarrow>=20.0.0
+Requires-Dist: pandas>=2.3.1
+Requires-Dist: duckdb>=1.4.4
+Provides-Extra: cli
+Requires-Dist: visidata; extra == "cli"
+Dynamic: license-file
+
+# gpxtractor
+
+**GPX, TCX and FIT data extraction for Python**
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+## Description
+
+
+## Features
+
+
+## Installation
+
+```bash
+git clone 
+cd gpxtractor
+pip install .
+```
+
+## Usage Example
+Use the `gpxtractor.extract_data` function that returns a gpxtractor.Activity instance.
+
+```python
+import gpxtractor
+
+activity = gpxtractor.extract_data("your-gpx-tcx-or-fit_file.gpx")
+
+print(activity.sport) # Output: name of the sport in the file as a string
+
+```
+
+The records attribute is a `pandas.DataFrame` holding the records extracted from the file\n
+with the `gpxtractor.extract_data` function. So the usual `pandas.DataFrame` methods can be applied
+
+```python
+print(activity.records.head())
+```
+
+Once an instance of an Activity as been created with the `extract_data` function, the method\n
+`transform_records` can be used to calculate distance and speed if missing from the file as well as\n
+elevation incremental difference, gradient and in the case of running activities, pace.
+
+```python
+activity.transform_records()
+print(activity.records.head())
+```
+
+And once the records have been transformed with `transform_records`, it is possible to use the 2\n
+following methods to calculate aggregated data for kilometre and lap splits.
+
+```python
+activity.compute_km_splits()
+print(activity.km_splits)
+
+activity.compute_lap_splits()
+print(activity.lap_splits)
+```
+Note: the `compute_lap_splits` will only compute lap splits if the file contains lap data which is not\n
+the case for GPX files. It does not update the `lap_splits` attribute otherwise.
+
+

gpxtractor-0.1.0/README.md

@@ -0,0 +1,62 @@
+# gpxtractor
+
+**GPX, TCX and FIT data extraction for Python**
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+## Description
+
+
+## Features
+
+
+## Installation
+
+```bash
+git clone 
+cd gpxtractor
+pip install .
+```
+
+## Usage Example
+Use the `gpxtractor.extract_data` function that returns a gpxtractor.Activity instance.
+
+```python
+import gpxtractor
+
+activity = gpxtractor.extract_data("your-gpx-tcx-or-fit_file.gpx")
+
+print(activity.sport) # Output: name of the sport in the file as a string
+
+```
+
+The records attribute is a `pandas.DataFrame` holding the records extracted from the file\n
+with the `gpxtractor.extract_data` function. So the usual `pandas.DataFrame` methods can be applied
+
+```python
+print(activity.records.head())
+```
+
+Once an instance of an Activity as been created with the `extract_data` function, the method\n
+`transform_records` can be used to calculate distance and speed if missing from the file as well as\n
+elevation incremental difference, gradient and in the case of running activities, pace.
+
+```python
+activity.transform_records()
+print(activity.records.head())
+```
+
+And once the records have been transformed with `transform_records`, it is possible to use the 2\n
+following methods to calculate aggregated data for kilometre and lap splits.
+
+```python
+activity.compute_km_splits()
+print(activity.km_splits)
+
+activity.compute_lap_splits()
+print(activity.lap_splits)
+```
+Note: the `compute_lap_splits` will only compute lap splits if the file contains lap data which is not\n
+the case for GPX files. It does not update the `lap_splits` attribute otherwise.
+
+

gpxtractor-0.1.0/gpxtractor/__init__.py

@@ -0,0 +1,12 @@
+"""
+GPX, TCX and FIT data extraction for Python
+======================================
+
+gpxtractor is a python package to extract data from
+gpx, tcx and fit files and present it in a dataframe.
+"""
+
+__version__ = "0.1.0"
+
+
+from gpxtractor._core import Activity, extract_data

gpxtractor-0.1.0/gpxtractor/_core.py

@@ -0,0 +1,265 @@
+import gzip
+import pathlib
+from typing import Optional
+from dataclasses import dataclass, field
+import pyarrow as pa
+import pandas as pd
+
+import gpxtractor._xml_extraction as xml_ext
+import gpxtractor._fit_extraction as fit_ext
+import gpxtractor._transformation as tr
+import gpxtractor._utils as ut
+
+
+@dataclass
+class Activity:
+    """Stores and manages records and metadata parsed from a gpx, tcx or
+    fit file.
+
+    This class is designed to hold structured data and associated metadata
+    extracted from a gpx, tcx or fit file, providing methods for accessing
+    and transforming the records.
+
+    Parameters
+    ----------
+    file_type : str
+        Can be any of the following: 'GPX', 'TCX' or 'FIT'.
+        Corresponds to the type of the file for which the instance of the
+        class holds data.
+
+    sport : str
+        Is the type of sport as extracted from the file in lower case.
+
+    records : pandas.DataFrame
+        DataFrame holding the records extracted from the gpx, tcx or fit file.
+
+    Attributes
+    ----------
+    is_transformed : bool
+        initially False, becomes True once either the method
+        `transform_records` or `full_transform` is used.
+
+    file_type : str
+        Can be any of the following: 'GPX', 'TCX' or 'FIT'.
+        Corresponds to the type of the file for which the instance of the
+        class holds data.
+
+    sport : None or str
+        Is the type of sport as extracted from the file in lower case.
+
+    start_time : None or pandas.Timestamp
+        Is None before a transformation method has been called.
+        A pandas Timestamp with timezone information indicating the start
+        time of the activity.
+
+    elapsed_time : None or int
+        Is None before a transformation method has been called.
+        An integer indicating the total elapsed time of the activity in
+        seconds.
+
+    distance : None or float
+        Is None before a transformation method has been called.
+        A float indicating the total distance covered during the activity
+        in kilometres.
+
+    avg_speed : None or float
+        Is None before a transformation method has been called.
+        A float indicating the average speed over the activity in kph.
+
+    avg_pace : None or str
+        Is None before a transformation method has been called.
+        A string indicating the average pace over the activity in min per km.
+
+    elevation_gain : None or int
+        Is None before a transformation method has been called.
+        An integer indicating the total elevation gained during the activity
+        in meters.
+
+    elevation_loss : None or int
+        Is None before a transformation method has been called.
+        An integer indicating the total elevation lossed during the activity
+        in meters.
+
+    avg_heart_rate : None or int
+        Is None before a transformation method has been called.
+        An integer indicating the average heart rate of the activity in bpm.
+
+    max_heart_rate : None or int
+        Is None before a transformation method has been called.
+        An integer indicating the maximum heart rate of the activity in bpm.
+
+    avg_cadence : None or int
+        Is None before a transformation method has been called.
+        An integer indicating the average cadence of the activity in either
+        rpm or, in the case of a running activity spm.
+
+    max_cadence : None or int
+        Is None before a transformation method has been called.
+        An integer indicating the maximum cadence of the activity in either
+        rpm or, in the case of a running activity spm.
+
+    records : pandas.DataFrame
+        DataFrame holding the records extracted from the gpx, tcx or fit file.
+        Records can be transformed with the methods `transform_records` or
+        `full_transform`.
+
+    km_splits : None or pandas.DataFrame
+        Initially None. DataFrame holding the transformed and aggregated data
+        grouped by kilometre splits once the `compute_km_splits` or
+        `full_transform` method has been used.
+
+    lap_splits : None or pandas.DataFrame
+        Initially None. DataFrame holding the transformed and aggregated data
+        grouped by lap splits once the `compute_lap_splits` or
+        `full_transform` method has been used. Can only hold data if the file
+        has lap data which is not the case for gpx files.
+    """
+
+    file_type: str
+    sport: str
+    records: pd.DataFrame
+    is_transformed: bool = field(default=False, init=False)
+    start_time: Optional[pd.Timestamp] = field(default=None, init=False)
+    elapsed_time: Optional[int] = field(default=None, init=False)
+    distance: Optional[float] = field(default=None, init=False)
+    avg_speed: Optional[float] = field(default=None, init=False)
+    max_speed: Optional[float] = field(default=None, init=False)
+    avg_pace: Optional[str] = field(default=None, init=False)
+    elevation_gain: Optional[int] = field(default=None, init=False)
+    elevation_loss: Optional[int] = field(default=None, init=False)
+    avg_heart_rate: Optional[int] = field(default=None, init=False)
+    max_heart_rate: Optional[int] = field(default=None, init=False)
+    avg_cadence: Optional[int] = field(default=None, init=False)
+    max_cadence: Optional[int] = field(default=None, init=False)
+    km_splits: Optional[pd.DataFrame] = field(default=None, init=False)
+    lap_splits: Optional[pd.DataFrame] = field(default=None, init=False)
+
+    def __str__(self):
+        records_str = str(self.records.head())
+        km_splits_str = (
+            str(self.km_splits.head()) if self.km_splits is not None else None
+        )
+        lap_splits_str = (
+            str(self.lap_splits.head()) if self.lap_splits is not None else None
+        )
+        return (
+            "Activity(\n"
+            f"  is_transformed: {self.is_transformed}\n"
+            f"  file_type: {self.file_type}\n"
+            f"  sport: {self.sport}\n"
+            f"  start_time: {self.start_time}\n"
+            f"  elapsed_time: {self.elapsed_time}\n"
+            f"  distance: {self.distance}\n"
+            f"  avg_speed: {self.avg_speed}\n"
+            f"  max_speed: {self.max_speed}\n"
+            f"  avg_pace: {self.avg_pace}\n"
+            f"  elevation_gain: {self.elevation_gain}\n"
+            f"  elevation_loss: {self.elevation_loss}\n"
+            f"  avg_heart_rate: {self.avg_heart_rate}\n"
+            f"  max_heart_rate: {self.max_heart_rate}\n"
+            f"  avg_cadence: {self.avg_cadence}\n"
+            f"  max_cadence: {self.max_cadence}\n"
+            f"  records:\n{records_str}\n"
+            f"  km_splits:\n{km_splits_str}\n"
+            f"  lap_splits:\n{lap_splits_str}\n"
+            ")"
+        )
+
+    def _transform_records_to_pyarrow(self):
+        if not self.is_transformed:
+            self.records = pa.Table.from_pandas(self.records)
+            self.records = tr.transform_data(self.records, self.sport)
+            stats = tr.compute_overall_stats(self.records)
+            self.start_time = stats["start_time"].at[0]
+            self.elapsed_time = int(stats["elapsed_time"].at[0])
+            self.distance = float(stats["distance"].at[0])
+            self.avg_speed = float(stats["avg_speed"].at[0])
+            self.max_speed = float(stats["max_speed"].at[0])
+            self.avg_pace = stats["avg_pace"].at[0]
+            self.elevation_gain = int(stats["elevation_gain"].at[0])
+            self.elevation_loss = int(stats["elevation_loss"].at[0])
+            self.avg_heart_rate = int(stats["avg_heart_rate"].at[0])
+            self.max_heart_rate = int(stats["max_heart_rate"].at[0])
+            self.avg_cadence = int(stats["avg_cadence"].at[0])
+            self.max_cadence = int(stats["max_cadence"].at[0])
+
+    def transform_records(self):
+        """Transforms the data in the records attributes to calculate distance,
+        speed if absent and elevation difference, gradient and, in the case of
+        running activities, pace.
+        """
+        if not self.is_transformed:
+            self._transform_records_to_pyarrow()
+            self.records = self.records.to_pandas(types_mapper=pd.ArrowDtype)
+            self.is_transformed = True
+
+    def compute_lap_splits(self):
+        """If there is lap data in the records, updates the lap_splits to a
+        DataFrame holding the transformed and aggregated data grouped by lap
+        splits. Note: there is no lap data in gpx files.
+        """
+        if self.file_type != "GPX" and self.is_transformed:
+            self.records = pa.Table.from_pandas(self.records)
+            self.lap_splits = tr.compute_lap_data(self.records)
+            self.records = self.records.to_pandas(types_mapper=pd.ArrowDtype)
+
+    def compute_km_splits(self):
+        """Updates km_splits attribute to a DataFrame holding the transformed
+        and aggregated data grouped by kilometre splits.
+        """
+        if self.is_transformed:
+            self.records = pa.Table.from_pandas(self.records)
+            self.km_splits = tr.compute_km_data(self.records)
+            self.records = self.records.to_pandas(types_mapper=pd.ArrowDtype)
+
+    def full_transform(self):
+        """Transforms data in records, computes km and lap splits"""
+        if not self.is_transformed:
+            self._transform_records_to_pyarrow()
+            self.km_splits = tr.compute_km_data(self.records)
+            if self.file_type != "GPX":
+                self.lap_splits = tr.compute_lap_data(self.records)
+            self.records = self.records.to_pandas(types_mapper=pd.ArrowDtype)
+            self.is_transformed = True
+
+
+def extract_data(file_path: pathlib.Path) -> Activity:
+    """Extract records from a gpx, tcx or fit file.
+    Create and return a new Activity instance where records are
+    stored as a pandas.DataFrame in the records attribute and the
+    sport is stored as a string in the sport attribute.
+
+    Parameters
+    ----------
+    file_path : pathlib.Path
+        Path to a file of type .gpx, .tcx or .fit. Can be gzipped.
+
+    Returns
+    -------
+    gpxtractor.Activity
+
+    Raises
+    ------
+    ValueError
+        if the file type is not gpx, tcx or fit or their gzipped
+        equivalent.
+    """
+    extensions = ut._get_file_extensions(file_path)
+    match extensions:
+        case ".gpx" | ".gpx.gz":
+            sport, records = ut._handle_gzipped_xml_files(
+                file_path, extensions, xml_ext.get_sport_from_gpx, xml_ext.extract_gpx
+            )
+        case ".tcx" | ".tcx.gz":
+            sport, records = ut._handle_gzipped_xml_files(
+                file_path, extensions, xml_ext.get_sport_from_tcx, xml_ext.extract_tcx
+            )
+        case ".fit":
+            sport, records = fit_ext.extract_fit(file_path)
+        case ".fit.gz":
+            with gzip.open(file_path, "rb") as gz:
+                sport, records = fit_ext.extract_fit(gz)
+        case _:
+            raise ValueError("Not a valid file type: Try a GPX, TCX or FIT file")
+    file_type = ut._get_file_type_from_extensions(extensions)
+    return Activity(file_type=file_type, sport=sport, records=records)

gpxtractor-0.1.0/gpxtractor/_fit_extraction.py

@@ -0,0 +1,90 @@
+import pathlib
+import numpy as np
+import pandas as pd
+import fitdecode
+
+
+def _convert_fit_coords_to_deg(coord):
+    """Convert semicircle 32-bit integer coordinate to degrees"""
+    return coord * (180 / 2**31)
+
+
+def _generate_frame_from_fit(fit_file: pathlib.Path, selected_frames: list):
+    with fitdecode.FitReader(fit_file, check_crc=False) as fit:
+        for frame in fit:
+            if (
+                frame.frame_type == fitdecode.FIT_FRAME_DATA
+                and frame.name in selected_frames
+            ):
+                yield frame
+
+
+def _extract_str(frame, field_name: str):
+    if frame.has_field(field_name) and frame.get_value(field_name) is not None:
+        return frame.get_value(field_name)
+    return None
+
+
+def _extract_value(frame, field_name: str, datatype):
+    if frame.has_field(field_name) and frame.get_value(field_name) is not None:
+
+        return datatype(frame.get_value(field_name))
+    return 0 if datatype is int else np.nan
+
+
+def get_sport_from_fit(fit_content) -> str:
+    for frame in _generate_frame_from_fit(fit_content, ["session"]):
+        return _extract_str(frame, "sport")
+    return None
+
+
+def extract_fit(file_path: pathlib.Path) -> pd.DataFrame:
+    lap_number = 1
+    laps = []
+    times = []
+    lats = []
+    lons = []
+    eles = []
+    dists = []
+    speeds = []
+    hrs = []
+    cads = []
+
+    for frame in _generate_frame_from_fit(file_path, ["lap", "record", "session"]):
+        if frame.name == "record":
+            laps.append(lap_number)
+            times.append(_extract_str(frame, "timestamp"))
+            lats.append(_extract_value(frame, "position_lat", float))
+            lons.append(_extract_value(frame, "position_long", float))
+            eles.append(_extract_value(frame, "altitude", float))
+            dists.append(_extract_value(frame, "distance", float))
+            speeds.append(_extract_value(frame, "speed", float))
+            hrs.append(_extract_value(frame, "heart_rate", int))
+            cads.append(_extract_value(frame, "cadence", int))
+        elif frame.name == "lap":
+            lap_number += 1
+        elif frame.name == "session":
+            sport = _extract_str(frame, "sport")
+
+    laps = np.array(laps, dtype=np.uint16)
+    lats = _convert_fit_coords_to_deg(np.array(lats, dtype=np.float32))
+    lons = _convert_fit_coords_to_deg(np.array(lons, dtype=np.float32))
+    eles = np.array(eles, dtype=np.float32)
+    dists = np.array(dists, dtype=np.float32)
+    speeds = np.array(speeds, dtype=np.float32)
+    hrs = np.array(hrs, dtype=np.uint8)
+    cads = np.array(cads, dtype=np.uint8)
+
+    return sport, pd.DataFrame(
+        {
+            "lap": laps,
+            "timestamp": pd.to_datetime(times),
+            "latitude": lats,
+            "longitude": lons,
+            "distance": dists,
+            "speed": speeds,
+            "altitude": eles,
+            "heart_rate": hrs,
+            "cadence": cads,
+        }
+    )

gpxtractor-0.1.0/gpxtractor/_transformation.py

@@ -0,0 +1,99 @@
+import inspect
+from importlib_resources import files
+import pandas as pd
+import pyarrow as pa
+import duckdb
+
+
+def get_var_name(var):
+    callers_local_vars = inspect.currentframe().f_back.f_locals.items()
+    return [name for name, val in callers_local_vars if val is var][0]
+
+
+def is_col_all_null(table: pa.Table, col: str) -> bool:
+    null_mask = pa.compute.is_null(table.column(col))
+    return pa.compute.all(null_mask).as_py()
+
+
+def add_empty_col_if_absent(arrow_table: pa.Table, col: str, datatype) -> pa.Table:
+    if col not in arrow_table.schema.names:
+        empty_values = pa.nulls(len(arrow_table), type=datatype)
+        return arrow_table.append_column(col, empty_values)
+    else:
+        return arrow_table
+
+
+def query_table(arrow_table: pa.Table, sql_file: str) -> pa.Table:
+    sql_path = files("gpxtractor.sql").joinpath(sql_file)
+    safe_table_name = get_var_name(arrow_table)
+    sql_query = sql_path.read_text().format(table_name=safe_table_name)
+    return duckdb.sql(sql_query).arrow().read_all()
+
+
+def compute_distance_and_speed(arrow_table: pa.Table) -> pa.Table:
+    sql_haversine_file = files("gpxtractor.sql").joinpath("haversine_formula.sql")
+    haversine_formula = sql_haversine_file.read_text()
+    duckdb.sql(haversine_formula)
+    sql_file = "compute_distance_and_speed.sql"
+    return query_table(arrow_table, sql_file)
+
+
+def compute_speed(arrow_table: pa.Table) -> pa.Table:
+    sql_file = "compute_speed.sql"
+    return query_table(arrow_table, sql_file)
+
+
+def preprocess_data(arrow_table: pa.Table) -> pa.Table:
+    sql_file = "preprocess_data.sql"
+    return query_table(arrow_table, sql_file)
+
+
+def preprocess_running_data(arrow_table: pa.Table) -> pa.Table:
+    sql_file = "preprocess_running_data.sql"
+    return query_table(arrow_table, sql_file)
+
+
+def transform_data(arrow_table: pa.Table, sport: str) -> pa.Table:
+    REQUIRED_COLUMNS = {
+        "timestamp": pa.timestamp("us"),
+        "latitude": pa.float32(),
+        "longitude": pa.float32(),
+        "altitude": pa.float32(),
+        "heart_rate": pa.uint8(),
+        "cadence": pa.uint8(),
+        "lap": pa.uint16(),
+    }
+    for col, datatype in REQUIRED_COLUMNS.items():
+        arrow_table = add_empty_col_if_absent(arrow_table, col, datatype)
+    if "distance" not in arrow_table.schema.names or is_col_all_null(
+        arrow_table, "distance"
+    ):
+        arrow_table = compute_distance_and_speed(arrow_table)
+    elif "speed" not in arrow_table.schema.names or is_col_all_null(
+        arrow_table, "speed"
+    ):
+        arrow_table = compute_speed(arrow_table)
+    if sport == "running":
+        arrow_table = preprocess_running_data(arrow_table)
+    else:
+        arrow_table = preprocess_data(arrow_table)
+
+    return arrow_table
+
+
+def compute_km_data(arrow_table: pa.Table) -> pd.DataFrame:
+    sql_file = "km_data_query.sql"
+    arrow_table = query_table(arrow_table, sql_file)
+    return arrow_table.to_pandas(types_mapper=pd.ArrowDtype)
+
+
+def compute_lap_data(arrow_table: pa.Table) -> pd.DataFrame:
+    sql_file = "lap_data_query.sql"
+    arrow_table = query_table(arrow_table, sql_file)
+    return arrow_table.to_pandas(types_mapper=pd.ArrowDtype)
+
+
+def compute_overall_stats(arrow_table: pa.Table):
+    sql_file = "overall_stats.sql"
+    arrow_table = query_table(arrow_table, sql_file)
+    return arrow_table.to_pandas(types_mapper=pd.ArrowDtype)

gpxtractor-0.1.0/gpxtractor/_utils.py

@@ -0,0 +1,32 @@
+import pathlib
+import gzip
+
+
+def _get_file_extensions(file_path: pathlib.Path) -> str:
+    path = pathlib.Path(file_path)
+    return "".join(path.suffixes)
+
+
+def _get_file_type_from_extensions(extensions) -> str:
+    match extensions:
+        case ".gpx" | ".gpx.gz":
+            return "GPX"
+        case ".tcx" | ".tcx.gz":
+            return "TCX"
+        case ".fit" | ".fit.gz":
+            return "FIT"
+
+
+def _handle_gzipped_xml_files(
+    file_path: pathlib.Path, extensions, sport_func, extraction_func
+):
+    is_gzipped = ".gz" in extensions
+    if is_gzipped:
+        with gzip.open(file_path, "rt") as gz:
+            sport = sport_func(gz)
+        with gzip.GzipFile(file_path, "r") as gz:
+            return sport, extraction_func(gz)
+    else:
+        with open(file_path, "r") as file:
+            sport = sport_func(file)
+        return sport, extraction_func(file_path)