trackclean 0.1.0__py3-none-any.whl

trackclean/__init__.py

@@ -0,0 +1,128 @@
+"""trackclean - clean, smooth, segment and measure raw GPS trajectories.
+
+Quick start::
+
+    import trackclean as tc
+
+    track = tc.read_track("ride.gpx")
+    result = tc.clean(track, mode="bike")
+    print(result.report)
+    print(tc.summarize(result.track, mode="bike").format())
+    tc.write(result.track, "ride.clean.gpx")
+"""
+
+from __future__ import annotations
+
+from .clean import (
+    CleanResult,
+    clean,
+    collapse_stationary,
+    detect_jumps,
+    remove_invalid,
+    remove_outliers,
+    shift_naive_time,
+    sort_and_dedup,
+    split_segments_on_gaps,
+)
+from .filters import hampel, hampel_filter, kalman_rts, kalman_smooth, smooth_elevation
+from .geodesy import (
+    LocalProjection,
+    bearing,
+    cumulative_distance,
+    haversine,
+    step_distances,
+    vincenty,
+)
+from .io import read, read_track, write
+from .metrics import (
+    TrackStats,
+    elapsed_time,
+    elevation_gain_loss,
+    moving_time,
+    pace_min_per_km,
+    radius_of_gyration,
+    speed_profile,
+    stops_summary,
+    summarize,
+    track_distance,
+)
+from .modes import MODES, ModeProfile, get_mode
+from .privacy import trim_ends
+from .resample import resample_distance, resample_time
+from .segment import (
+    ModeGuess,
+    Segmentation,
+    StayPoint,
+    Trip,
+    detect_stay_points,
+    infer_mode,
+    segment_trips,
+    split_on_gaps,
+)
+from .simplify import douglas_peucker, sed_error, simplify, td_tr, visvalingam
+from .timeutil import format_time, parse_time, parse_times
+from .track import Track, TrackSet, Waypoint
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "MODES",
+    "CleanResult",
+    "LocalProjection",
+    "ModeGuess",
+    "ModeProfile",
+    "Segmentation",
+    "StayPoint",
+    "Track",
+    "TrackSet",
+    "TrackStats",
+    "Trip",
+    "Waypoint",
+    "__version__",
+    "bearing",
+    "clean",
+    "collapse_stationary",
+    "cumulative_distance",
+    "detect_jumps",
+    "detect_stay_points",
+    "douglas_peucker",
+    "elapsed_time",
+    "elevation_gain_loss",
+    "format_time",
+    "get_mode",
+    "hampel",
+    "hampel_filter",
+    "haversine",
+    "infer_mode",
+    "kalman_rts",
+    "kalman_smooth",
+    "moving_time",
+    "pace_min_per_km",
+    "parse_time",
+    "parse_times",
+    "radius_of_gyration",
+    "read",
+    "read_track",
+    "remove_invalid",
+    "remove_outliers",
+    "resample_distance",
+    "resample_time",
+    "sed_error",
+    "segment_trips",
+    "shift_naive_time",
+    "simplify",
+    "smooth_elevation",
+    "sort_and_dedup",
+    "speed_profile",
+    "split_on_gaps",
+    "split_segments_on_gaps",
+    "step_distances",
+    "stops_summary",
+    "summarize",
+    "td_tr",
+    "track_distance",
+    "trim_ends",
+    "vincenty",
+    "visvalingam",
+    "write",
+]

trackclean/__main__.py

@@ -0,0 +1,7 @@
+"""Allow ``python -m trackclean``."""
+
+import sys
+
+from .cli import main
+
+sys.exit(main())

trackclean/clean.py

@@ -0,0 +1,399 @@
+"""Trajectory cleaning: ordering, outliers, teleports, standstill drift and the full pipeline."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
+
+import numpy as np
+
+from .filters import hampel_filter, kalman_smooth, measurement_sigma, smooth_elevation
+from .geodesy import LocalProjection
+from .modes import ModeProfile, get_mode
+from .segment import detect_stay_points
+from .timeutil import TzLike, resolve_tz
+from .track import Track
+
+__all__ = [
+    "CleanResult",
+    "clean",
+    "collapse_stationary",
+    "detect_jumps",
+    "remove_invalid",
+    "remove_outliers",
+    "shift_naive_time",
+    "sort_and_dedup",
+    "split_segments_on_gaps",
+]
+
+
+def remove_invalid(track: Track, *, drop_null_island: bool = True) -> Tuple[Track, int]:
+    """Drop points with missing/out-of-range coordinates (and exact ``0, 0`` fixes)."""
+    ok = (
+        np.isfinite(track.lat)
+        & np.isfinite(track.lon)
+        & (np.abs(track.lat) <= 90.0)
+        & (np.abs(track.lon) <= 180.0)
+    )
+    if drop_null_island:
+        ok &= ~((track.lat == 0.0) & (track.lon == 0.0))
+    return track.take(ok), int((~ok).sum())
+
+
+def sort_and_dedup(track: Track, *, keep: str = "first") -> Tuple[Track, Dict[str, int]]:
+    """Sort by time (stable), drop points without a time and points sharing a timestamp.
+
+    Segment order is preserved: points are sorted within the track as a whole, then
+    segment ids are re-numbered in order of appearance. ``keep`` is ``"first"`` or
+    ``"best"`` (the fix with the smallest ``accuracy``/``hdop`` among duplicates).
+    """
+    stats = {"no_time": 0, "duplicates": 0, "reordered": 0}
+    if not len(track):
+        return track.copy(), stats
+    has_t = np.isfinite(track.time)
+    stats["no_time"] = int((~has_t).sum())
+    tr = track.take(has_t)
+    order = np.argsort(tr.time, kind="stable")
+    stats["reordered"] = int(np.count_nonzero(order != np.arange(order.size)))
+    tr = tr.take(order)
+    if len(tr) < 2:
+        return tr, stats
+    quality = None
+    if keep == "best":
+        q = tr.fields.get("accuracy", tr.fields.get("hdop"))
+        if q is not None:
+            quality = np.where(np.isfinite(q), q, np.inf)
+    same = np.concatenate([[False], np.diff(tr.time) == 0])
+    if quality is None:
+        keep_mask = ~same
+    else:
+        keep_mask = np.ones(len(tr), dtype=bool)
+        group = np.cumsum(~same) - 1
+        for g in np.unique(group[same]):
+            idx = np.flatnonzero(group == g)
+            best = idx[int(np.argmin(quality[idx]))]
+            keep_mask[idx] = False
+            keep_mask[best] = True
+    stats["duplicates"] = int((~keep_mask).sum())
+    return tr.take(keep_mask), stats
+
+
+def shift_naive_time(track: Track, tz: TzLike) -> Track:
+    """Re-interpret times that were read as UTC but were really local time in ``tz``.
+
+    Useful for loggers that write local clock time without an offset. Handles daylight
+    saving transitions for IANA zones.
+    """
+    from datetime import datetime, timezone
+
+    tzi = resolve_tz(tz)
+    out = track.copy()
+    for i, t in enumerate(track.time):
+        if np.isfinite(t):
+            naive = datetime.fromtimestamp(float(t), tz=timezone.utc).replace(tzinfo=None)
+            out.time[i] = naive.replace(tzinfo=tzi).timestamp() + (t - np.floor(t))
+    return out
+
+
+def _step_xy(track: Track) -> Tuple[np.ndarray, np.ndarray]:
+    proj = LocalProjection.for_points(track.lat, track.lon)
+    return proj.forward(track.lat, track.lon)
+
+
+def _gate(
+    d: np.ndarray,
+    dt: np.ndarray,
+    v_ref: np.ndarray,
+    profile: ModeProfile,
+    noise: np.ndarray,
+    k_sigma: float,
+) -> np.ndarray:
+    """True where a step of length d over dt seconds is kinematically implausible."""
+    v_allowed = np.minimum(profile.max_speed, v_ref + profile.max_accel * dt)
+    return np.asarray(d > v_allowed * dt + k_sigma * noise)
+
+
+def _robust_speed(d: np.ndarray, dt: np.ndarray, half: int = 4) -> np.ndarray:
+    v = d / np.maximum(dt, 1e-3)
+    if v.size == 0:
+        return v
+    pad = np.concatenate([np.full(half, np.nan), v, np.full(half, np.nan)])
+    win = np.lib.stride_tricks.sliding_window_view(pad, 2 * half + 1)
+    import warnings
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", RuntimeWarning)
+        return np.asarray(np.nanmedian(win, axis=1))
+
+
+def detect_jumps(
+    track: Track,
+    mode: Union[str, ModeProfile] = "walk",
+    *,
+    k_sigma: float = 4.0,
+) -> np.ndarray:
+    """Indices ``i`` of steps ``i -> i+1`` that are kinematically impossible.
+
+    A step is a jump when its length exceeds ``min(max_speed, v_ref + max_accel*dt) * dt``
+    plus ``k_sigma`` times the combined position noise of the two fixes, where ``v_ref`` is
+    a robust (median) local speed. The noise allowance keeps ordinary GPS scatter on slow
+    modes from being flagged.
+    """
+    prof = get_mode(mode)
+    if len(track) < 2 or not track.has_time:
+        return np.zeros(0, dtype=np.int64)
+    x, y = _step_xy(track)
+    d = np.hypot(np.diff(x), np.diff(y))
+    dt = np.diff(track.time)
+    sig = measurement_sigma(track, prof.gps_sigma)
+    noise = np.hypot(sig[:-1], sig[1:])
+    v_ref = _robust_speed(d, dt)
+    bad = _gate(d, dt, v_ref, prof, noise, k_sigma)
+    bad &= np.diff(track.segment) == 0
+    return np.flatnonzero(bad)
+
+
+def remove_outliers(
+    track: Track,
+    mode: Union[str, ModeProfile] = "walk",
+    *,
+    k_sigma: float = 4.0,
+    max_block: int = 30,
+    max_block_time: float = 300.0,
+    min_segment_points: int = 3,
+    max_iter: int = 10,
+) -> Tuple[Track, Dict[str, int]]:
+    """Remove speed/acceleration outliers and resolve teleports.
+
+    1. Steps that violate the kinematic gate (see :func:`detect_jumps`) are found.
+    2. A jump away followed, within ``max_block`` points and ``max_block_time`` seconds, by
+       a jump back to where the track left off (the bridging step passes the gate) marks
+       the points in between as outliers: single spikes and short displaced bursts
+       (multipath, cell-tower fixes) both fall into this case.
+    3. Jumps that never come back are genuine discontinuities (receiver re-acquired after a
+       tunnel, logger restarted elsewhere): the track is split into a new segment there.
+       Resulting fragments shorter than ``min_segment_points`` are dropped.
+
+    Returns the cleaned track and counters ``{"spikes", "teleports", "fragments"}``.
+    """
+    prof = get_mode(mode)
+    stats = {"spikes": 0, "teleports": 0, "fragments": 0}
+    tr = track.copy()
+    if len(tr) < 3 or not tr.has_time:
+        return tr, stats
+    sig_all = measurement_sigma(tr, prof.gps_sigma)
+    tr.fields["__sig"] = sig_all
+    for _ in range(max_iter):
+        jumps = detect_jumps(tr, prof, k_sigma=k_sigma)
+        if jumps.size == 0:
+            break
+        x, y = _step_xy(tr)
+        t = tr.time
+        sig = tr.fields["__sig"]
+        d_all = np.hypot(np.diff(x), np.diff(y))
+        v_ref = _robust_speed(d_all, np.diff(t))
+        drop = np.zeros(len(tr), dtype=bool)
+        used: Set[int] = set()
+        for a_pos, a in enumerate(jumps):
+            if a in used:
+                continue
+            for b in jumps[a_pos + 1 :]:
+                if b - a > max_block or t[b + 1] - t[a] > max_block_time:
+                    break
+                if tr.segment[a] != tr.segment[b + 1]:
+                    break
+                dd = float(np.hypot(x[b + 1] - x[a], y[b + 1] - y[a]))
+                dtt = float(t[b + 1] - t[a])
+                vr = float(np.nanmin([v_ref[a], v_ref[min(b + 1, v_ref.size - 1)]]))
+                noise = float(np.hypot(sig[a], sig[b + 1]))
+                vr = vr if np.isfinite(vr) else 0.0
+                implausible = _gate(
+                    np.array([dd]),
+                    np.array([dtt]),
+                    np.array([vr]),
+                    prof,
+                    np.array([noise]),
+                    k_sigma,
+                )[0]
+                if not implausible:
+                    drop[a + 1 : b + 1] = True
+                    used.update(int(j) for j in jumps if a <= j <= b)
+                    break
+        if drop.any():
+            stats["spikes"] += int(drop.sum())
+            tr = tr.take(~drop)
+            continue
+        # No bridging found: unresolved jumps are discontinuities -> new segments.
+        cut = np.diff(tr.segment) != 0
+        cut[jumps] = True
+        tr.segment = np.concatenate([[0], np.cumsum(cut)]).astype(np.int64)
+        stats["teleports"] += int(jumps.size)
+        break
+    # Drop fragments that are too short to be trusted.
+    keep = np.ones(len(tr), dtype=bool)
+    for s in tr.segment_slices():
+        if s.stop - s.start < min_segment_points:
+            keep[s] = False
+            stats["fragments"] += s.stop - s.start
+    tr = tr.take(keep)
+    tr.fields.pop("__sig", None)
+    return tr.renumber_segments(), stats
+
+
+def split_segments_on_gaps(track: Track, max_gap: float) -> Tuple[Track, int]:
+    """Start a new segment wherever the time step exceeds ``max_gap`` seconds."""
+    if len(track) < 2 or not track.has_time:
+        return track.copy(), 0
+    cut = np.diff(track.time) > max_gap
+    cut |= np.diff(track.segment) != 0
+    out = track.copy()
+    out.segment = np.concatenate([[0], np.cumsum(cut)]).astype(np.int64)
+    n_gaps = int((np.diff(track.time) > max_gap).sum())
+    return out, n_gaps
+
+
+def collapse_stationary(
+    track: Track,
+    radius: float = 25.0,
+    min_duration: float = 120.0,
+) -> Tuple[Track, int]:
+    """Collapse GPS drift while standing still.
+
+    Stays of at least ``min_duration`` seconds within ``radius`` metres (stay-point
+    detection, :func:`trackclean.detect_stay_points`) have every point moved to the median
+    centre of the stay. Timestamps are kept, so dwell time survives while the spurious
+    "jitter distance" (often hundreds of metres per hour) disappears.
+    Returns the new track and the number of stays collapsed.
+    """
+    out = track.copy()
+    count = 0
+    for s in track.segment_slices():
+        part = track.take(s)
+        if len(part) < 3 or not part.has_time:
+            continue
+        stays = detect_stay_points(part, radius, min_duration, centroid="median")
+        if not stays:
+            continue
+        proj = part.projection()
+        x, y = proj.forward(part.lat, part.lon)
+        # Wandering indoor drift often splits one stop into several adjacent stays:
+        # merge neighbours separated by a few fixes whose centres are close.
+        groups = [[stays[0].start_index, stays[0].end_index]]
+        for sp in stays[1:]:
+            ga, gb = groups[-1]
+            cx0, cy0 = np.median(x[ga : gb + 1]), np.median(y[ga : gb + 1])
+            cx1, cy1 = (
+                np.median(x[sp.start_index : sp.end_index + 1]),
+                np.median(y[sp.start_index : sp.end_index + 1]),
+            )
+            short_gap = part.time[sp.start_index] - part.time[gb] <= min_duration / 2
+            if short_gap and np.hypot(cx1 - cx0, cy1 - cy0) <= radius:
+                groups[-1][1] = sp.end_index
+            else:
+                groups.append([sp.start_index, sp.end_index])
+        for ga, gb in groups:
+            cx = float(np.median(x[ga : gb + 1]))
+            cy = float(np.median(y[ga : gb + 1]))
+            lat, lon = proj.inverse(cx, cy)
+            out.lat[s.start + ga : s.start + gb + 1] = float(lat)
+            out.lon[s.start + ga : s.start + gb + 1] = float(lon)
+            count += 1
+    return out, count
+
+
+@dataclass
+class CleanResult:
+    """Output of :func:`clean`: the cleaned track and what each step did."""
+
+    track: Track
+    report: Dict[str, Any] = field(default_factory=dict)
+    steps: List[str] = field(default_factory=list)
+
+    def summary(self) -> str:
+        lines = [f"{k}: {v}" for k, v in self.report.items()]
+        return "\n".join(lines)
+
+
+def clean(
+    track: Track,
+    mode: Union[str, ModeProfile] = "walk",
+    *,
+    outliers: bool = True,
+    hampel: bool = True,
+    kalman: bool = True,
+    collapse_stops: bool = True,
+    elevation: bool = True,
+    max_gap: Optional[float] = None,
+    process_noise: Optional[float] = None,
+    gps_sigma: Optional[float] = None,
+    k_sigma: float = 4.0,
+    store_speed: bool = False,
+) -> CleanResult:
+    """Run the full cleaning pipeline.
+
+    Steps, in order (each can be switched off):
+
+    1. drop invalid coordinates, points without time, duplicates; sort by time;
+    2. split segments at signal gaps longer than ``max_gap`` (default: mode profile);
+    3. remove speed/acceleration outliers and split at teleports (:func:`remove_outliers`);
+    4. Hampel filter on east/north coordinates (moderate outliers the gate cannot see);
+    5. Kalman constant-velocity filter + RTS smoother, per segment, using per-point
+       ``accuracy``/``hdop`` when present;
+    6. collapse standstill drift to a single position per stay (:func:`collapse_stationary`);
+    7. elevation spike removal and smoothing.
+
+    ``mode`` is ``walk``, ``run``, ``bike``, ``car``, ``transit`` or a custom
+    :class:`~trackclean.ModeProfile`.
+    """
+    prof = get_mode(mode)
+    if gps_sigma is not None:
+        prof = prof.with_(gps_sigma=gps_sigma)
+    if process_noise is not None:
+        prof = prof.with_(process_noise=process_noise)
+    report: Dict[str, Any] = {"mode": prof.name, "input_points": len(track)}
+    steps: List[str] = []
+
+    tr, n_invalid = remove_invalid(track)
+    report["invalid_removed"] = n_invalid
+    steps.append("remove_invalid")
+    if not len(tr):
+        report["output_points"] = 0
+        return CleanResult(tr, report, steps)
+    has_time = bool(np.isfinite(tr.time).any())
+    if has_time:
+        tr, st = sort_and_dedup(tr)
+        report["no_time_removed"] = st["no_time"]
+        report["duplicates_removed"] = st["duplicates"]
+        report["reordered_points"] = st["reordered"]
+        steps.append("sort_and_dedup")
+        tr, n_gaps = split_segments_on_gaps(tr, max_gap if max_gap is not None else prof.max_gap)
+        report["gaps_split"] = n_gaps
+        steps.append("split_gaps")
+    if outliers and has_time:
+        tr, st = remove_outliers(tr, prof, k_sigma=k_sigma)
+        report["outliers_removed"] = st["spikes"]
+        report["teleports_split"] = st["teleports"]
+        report["fragments_dropped"] = st["fragments"]
+        steps.append("remove_outliers")
+    if hampel:
+        tr, n_h = hampel_filter(tr)
+        report["hampel_replaced"] = n_h
+        steps.append("hampel")
+    if kalman and has_time:
+        tr = kalman_smooth(
+            tr,
+            process_noise=prof.process_noise,
+            sigma=prof.gps_sigma,
+            store_speed=store_speed,
+        )
+        steps.append("kalman_rts")
+    if collapse_stops and has_time:
+        tr, n_c = collapse_stationary(tr, prof.jitter_radius, prof.jitter_time)
+        report["stationary_collapsed"] = n_c
+        steps.append("collapse_stationary")
+    if elevation and tr.has_elevation:
+        tr = smooth_elevation(tr)
+        steps.append("smooth_elevation")
+    report["output_points"] = len(tr)
+    report["segments"] = tr.n_segments
+    return CleanResult(tr, report, steps)