bottomtime 0.1.0__py3-none-any.whl

bottomtime/ingest/garmin_fit.py

@@ -0,0 +1,303 @@
+"""Ingest Garmin Descent FIT activity files.
+
+Decodes with the official garmin-fit-sdk. Known dive channels land in typed
+columns of garmin_samples; every other field (named or unknown-numeric) is
+preserved per sample in extra_json. Whole message types the FIT profile does
+not know (233, 325, ...) are preserved per occurrence in undecoded_payloads.
+
+An optional Garmin Connect index JSON (a list of activity dicts with
+activityId, startTimeGMT, startTimeLocal, ...) supplies true UTC offsets and
+Connect metadata; without it, times fall back to FIT timestamps.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+from garmin_fit_sdk import Decoder, Stream
+
+from ..store.db import source_dive_exists, utcnow
+from .archive import archive_file
+
+DECODER_VERSION = 1
+
+FIT_EPOCH_OFFSET = 631065600  # FIT timestamps count from 1989-12-31 00:00 UTC
+SEMICIRCLE = 180.0 / 2**31
+
+# record message field -> garmin_samples column
+RECORD_COLUMNS = {
+    "depth": "depth_m",
+    "temperature": "temp_c",
+    "heart_rate": "heart_rate",
+    "absolute_pressure": "absolute_pressure",
+    "next_stop_depth": "next_stop_depth_m",
+    "next_stop_time": "next_stop_time_s",
+    "time_to_surface": "tts_s",
+    "ndl_time": "ndl_s",
+    "cns_load": "cns_pct",
+    "n2_load": "n2_load",
+    "po2": "po2",
+    "ascent_rate": "ascent_rate",
+    "air_time_remaining": "air_time_remaining_s",
+    "pressure_sac": "pressure_sac",
+    "volume_sac": "volume_sac",
+    "rmv": "rmv",
+}
+
+# message types routed to dedicated tables rather than header_json
+NON_HEADER_KEYS = {"record_mesgs", "event_mesgs", "dive_alarm_mesgs", "dive_gas_mesgs"}
+
+
+def _jsonable(value):
+    if isinstance(value, (bytes, bytearray)):
+        return {"__hex__": bytes(value).hex()}
+    if isinstance(value, (list, tuple)):
+        return [_jsonable(v) for v in value]
+    if isinstance(value, dict):
+        return {str(k): _jsonable(v) for k, v in value.items()}
+    if isinstance(value, float) and value != value:  # NaN
+        return None
+    return value
+
+
+def load_index(index_path: Path | None) -> dict[str, dict]:
+    if not index_path or not index_path.exists():
+        return {}
+    entries = json.loads(index_path.read_text())
+    return {str(e["activityId"]): e for e in entries}
+
+
+def _parse_connect_time(value: str) -> datetime:
+    return datetime.strptime(value, "%Y-%m-%d %H:%M:%S")
+
+
+def ingest_fit_file(
+    con: sqlite3.Connection,
+    archive_dir: Path,
+    path: Path,
+    index: dict[str, dict],
+) -> str:
+    """Ingest one FIT file. Returns 'ingested' | 'skipped'."""
+    artifact_id, digest = archive_file(con, archive_dir, "fit", path)
+    if source_dive_exists(con, "garmin", digest):
+        return "skipped"
+
+    stream = Stream.from_file(str(path))
+    decoder = Decoder(stream)
+    messages, errors = decoder.read(convert_datetimes_to_dates=False)
+    if errors:
+        raise ValueError(f"FIT decode errors in {path.name}: {errors}")
+
+    records = messages.get("record_mesgs", [])
+    # A file can carry summaries but no sample stream (rare device hiccup);
+    # keep it as a header-only source dive rather than losing the dive.
+    if records:
+        t0 = records[0]["timestamp"]
+    else:
+        session = (messages.get("session_mesgs") or [{}])[0]
+        file_id_msg = (messages.get("file_id_mesgs") or [{}])[0]
+        t0 = session.get("start_time") or file_id_msg.get("time_created")
+        if t0 is None:
+            raise ValueError(f"no records and no start time in {path.name}")
+
+    file_id = (messages.get("file_id_mesgs") or [{}])[0]
+    activity_id = str(file_id.get("serial_number", ""))
+    # Garmin Connect's activityId is not in the file; recover it from the
+    # index by matching start time, else from a '<...>_<id>.fit' filename.
+    entry = None
+    stem_id = path.stem.rsplit("_", 1)[-1]
+    if stem_id.isdigit() and stem_id in index:
+        entry = index[stem_id]
+        activity_id = stem_id
+    else:
+        start_utc_probe = datetime.fromtimestamp(
+            t0 + FIT_EPOCH_OFFSET, tz=timezone.utc
+        ).strftime("%Y-%m-%d %H:%M:%S")
+        for aid, e in index.items():
+            if e.get("startTimeGMT") == start_utc_probe:
+                entry, activity_id = e, aid
+                break
+
+    start_dt_utc = datetime.fromtimestamp(t0 + FIT_EPOCH_OFFSET, tz=timezone.utc)
+    start_utc = start_dt_utc.strftime("%Y-%m-%d %H:%M:%S")
+    start_local, utc_offset_min = start_utc, None
+    if entry:
+        gmt = _parse_connect_time(entry["startTimeGMT"])
+        local = _parse_connect_time(entry["startTimeLocal"])
+        utc_offset_min = int((local - gmt).total_seconds() // 60)
+        start_local = entry["startTimeLocal"]
+    else:
+        # activity message carries the device's local timestamp
+        act = (messages.get("activity_mesgs") or [{}])[0]
+        if "local_timestamp" in act and "timestamp" in act:
+            utc_offset_min = int((act["local_timestamp"] - act["timestamp"]) // 60)
+            start_local = (
+                start_dt_utc + timedelta(minutes=utc_offset_min)
+            ).strftime("%Y-%m-%d %H:%M:%S")
+
+    header = {"connect_index": entry}
+    for key, msgs in messages.items():
+        if isinstance(key, str) and key.endswith("_mesgs") and key not in NON_HEADER_KEYS:
+            header[key] = _jsonable(msgs)
+
+    summaries = messages.get("dive_summary_mesgs") or []
+    session = (messages.get("session_mesgs") or [{}])[0]
+    duration = entry.get("duration") if entry else session.get("total_elapsed_time")
+    max_depth = max((r.get("depth") or 0 for r in records), default=None)
+    if max_depth is None and session.get("max_depth") is not None:
+        max_depth = session["max_depth"]
+    avg_depth = next(
+        (s.get("avg_depth") for s in summaries if s.get("avg_depth") is not None), None
+    )
+    mode = entry.get("activityType", {}).get("typeKey") if entry else None
+
+    intervals = [
+        records[i + 1]["timestamp"] - records[i]["timestamp"]
+        for i in range(min(len(records) - 1, 50))
+    ]
+    interval_ms = int(sorted(intervals)[len(intervals) // 2] * 1000) if intervals else None
+
+    cur = con.execute(
+        "INSERT INTO source_dives (source, artifact_id, source_key, content_hash,"
+        " decoder_version, start_time_utc, start_time_local, utc_offset_min,"
+        " duration_s, max_depth_m, avg_depth_m, sample_interval_ms, mode, header_json)"
+        " VALUES ('garmin',?,?,?,?,?,?,?,?,?,?,?,?,?)",
+        (
+            artifact_id,
+            activity_id,
+            digest,
+            DECODER_VERSION,
+            start_utc,
+            start_local,
+            utc_offset_min,
+            duration,
+            max_depth,
+            avg_depth,
+            interval_ms,
+            mode,
+            json.dumps(_jsonable(header), separators=(",", ":")),
+        ),
+    )
+    dive_id = cur.lastrowid
+
+    sample_rows = []
+    for rec in records:
+        row = {col: None for col in RECORD_COLUMNS.values()}
+        extra = {}
+        lat = lon = None
+        for k, v in rec.items():
+            if k == "timestamp":
+                continue
+            elif k == "position_lat":
+                lat = v * SEMICIRCLE
+            elif k == "position_long":
+                lon = v * SEMICIRCLE
+            elif isinstance(k, str) and k in RECORD_COLUMNS:
+                row[RECORD_COLUMNS[k]] = v
+            else:
+                extra[str(k)] = _jsonable(v)
+        sample_rows.append(
+            (
+                dive_id,
+                float(rec["timestamp"] - t0),
+                row["depth_m"], row["temp_c"], row["heart_rate"],
+                row["absolute_pressure"], lat, lon,
+                row["next_stop_depth_m"], row["next_stop_time_s"], row["tts_s"],
+                row["ndl_s"], row["cns_pct"], row["n2_load"], row["po2"],
+                row["ascent_rate"], row["air_time_remaining_s"],
+                row["pressure_sac"], row["volume_sac"], row["rmv"],
+                json.dumps(extra, separators=(",", ":")) if extra else None,
+            )
+        )
+    con.executemany(
+        "INSERT INTO garmin_samples (source_dive_id, t_s, depth_m, temp_c, heart_rate,"
+        " absolute_pressure, lat, lon, next_stop_depth_m, next_stop_time_s, tts_s,"
+        " ndl_s, cns_pct, n2_load, po2, ascent_rate, air_time_remaining_s,"
+        " pressure_sac, volume_sac, rmv, extra_json)"
+        " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+        sample_rows,
+    )
+
+    for gi, gas in enumerate(messages.get("dive_gas_mesgs") or []):
+        mode_str = str(gas.get("mode", ""))
+        circuit = {"open_circuit": "oc", "closed_circuit": "cc"}.get(mode_str)
+        con.execute(
+            "INSERT INTO gases (source_dive_id, gas_index, o2_pct, he_pct, circuit,"
+            " enabled, used, source_json) VALUES (?,?,?,?,?,?,?,?)",
+            (
+                dive_id,
+                gas.get("message_index", gi),
+                gas.get("oxygen_content"),
+                gas.get("helium_content"),
+                circuit,
+                1 if gas.get("status") == "enabled" else 0,
+                None,
+                json.dumps(_jsonable(gas), separators=(",", ":")),
+            ),
+        )
+
+    event_rows = []
+    for kind, key in (("fit_event", "event_mesgs"), ("fit_dive_alarm", "dive_alarm_mesgs")):
+        for msg in messages.get(key) or []:
+            ts = msg.get("timestamp")
+            event_rows.append(
+                (
+                    dive_id,
+                    float(ts - t0) if isinstance(ts, (int, float)) else None,
+                    kind,
+                    json.dumps(_jsonable(msg), separators=(",", ":")),
+                )
+            )
+    con.executemany(
+        "INSERT INTO events (source_dive_id, t_s, kind, payload_json) VALUES (?,?,?,?)",
+        event_rows,
+    )
+
+    undecoded = []
+    for key, msgs in messages.items():
+        if isinstance(key, str) and key.endswith("_mesgs"):
+            continue
+        for seq, msg in enumerate(msgs):
+            undecoded.append(
+                (
+                    dive_id,
+                    "fit_mesg",
+                    str(key),
+                    seq,
+                    None,
+                    json.dumps(_jsonable(msg), separators=(",", ":")),
+                )
+            )
+    con.executemany(
+        "INSERT INTO undecoded_payloads (source_dive_id, container, type_key, seq,"
+        " payload, fields_json) VALUES (?,?,?,?,?,?)",
+        undecoded,
+    )
+
+    return "ingested"
+
+
+def ingest_garmin_dir(
+    con: sqlite3.Connection,
+    archive_dir: Path,
+    fit_dir: Path,
+    index_name: str = "dives_index.json",
+) -> dict:
+    index_path = fit_dir / index_name
+    index = load_index(index_path)
+    if index_path.exists():
+        archive_file(con, archive_dir, "garmin_index", index_path)
+
+    stats = {"ingested": 0, "skipped": 0, "failed": 0}
+    for path in sorted(fit_dir.glob("*.fit")):
+        try:
+            with con:
+                result = ingest_fit_file(con, archive_dir, path, index)
+            stats[result] += 1
+        except Exception as e:
+            stats["failed"] += 1
+            print(f"FAILED {path.name}: {type(e).__name__}: {e}")
+    return stats

bottomtime/ingest/shearwater_db.py

@@ -0,0 +1,231 @@
+"""Ingest a Shearwater Cloud database (dive_data.db / 'Export Database').
+
+Each dive's complete native log lives in log_data.data_bytes_1 as an sw-pnf
+blob (decoded by bottomtime.decode.pnf); dive_details supplies the metadata
+layer (gas plan JSON, EndGF99, site/notes when present). The blob is stored
+verbatim in source_dives.raw_blob in addition to the archived .db, so every
+Shearwater byte is preserved twice over.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import sqlite3
+from datetime import datetime, timezone
+from pathlib import Path
+
+from ..decode import pnf
+from ..store.db import source_dive_exists
+from .archive import archive_file
+
+
+def _details_row(con: sqlite3.Connection, dive_id: str) -> dict:
+    row = con.execute(
+        "SELECT * FROM dive_details WHERE DiveId = ?", (dive_id,)
+    ).fetchone()
+    if row is None:
+        return {}
+    return {k: row[k] for k in row.keys()}
+
+
+def ingest_shearwater_db(
+    con: sqlite3.Connection, archive_dir: Path, db_path: Path
+) -> dict:
+    archive_file(con, archive_dir, "shearwater_db", db_path)
+
+    src = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
+    src.row_factory = sqlite3.Row
+
+    stats = {"ingested": 0, "skipped": 0, "failed": 0}
+    rows = src.execute(
+        "SELECT log_id, file_name, format, format_version,"
+        " calculated_values_from_samples, data_bytes_1, data_bytes_2, data_bytes_3"
+        " FROM log_data"
+    ).fetchall()
+
+    artifact_id = con.execute(
+        "SELECT id FROM raw_artifacts WHERE kind='shearwater_db' AND original_path=?"
+        " ORDER BY id DESC LIMIT 1",
+        (str(db_path),),
+    ).fetchone()[0]
+
+    for row in rows:
+        try:
+            with con:
+                result = _ingest_log(con, artifact_id, src, row)
+            stats[result] += 1
+        except Exception as e:
+            stats["failed"] += 1
+            print(f"FAILED {row['file_name']}: {type(e).__name__}: {e}")
+
+    src.close()
+    return stats
+
+
+def _ingest_log(
+    con: sqlite3.Connection,
+    artifact_id: int,
+    src: sqlite3.Connection,
+    row: sqlite3.Row,
+) -> str:
+    if row["format"] != "sw-pnf":
+        raise ValueError(f"unsupported log format {row['format']!r}")
+
+    blob = row["data_bytes_1"]
+    digest = hashlib.sha256(blob).hexdigest()
+    if source_dive_exists(con, "shearwater", digest):
+        return "skipped"
+
+    dive = pnf.decode(blob)
+    details = _details_row(src, row["log_id"])
+    b2 = json.loads(row["data_bytes_2"]) if row["data_bytes_2"] else {}
+    b3 = json.loads(row["data_bytes_3"]) if row["data_bytes_3"] else {}
+    calc = (
+        json.loads(row["calculated_values_from_samples"])
+        if row["calculated_values_from_samples"]
+        else {}
+    )
+
+    serial = details.get("SerialNumber")
+    if not serial and "computer_serial" in dive.header:
+        serial = f"{dive.header['computer_serial']:08X}"
+    dive_number = details.get("DiveNumber") or b3.get("DiveNumber")
+    source_key = f"{serial}#{dive_number}"
+
+    # DIVE_START_TIME is the local wall clock stored as a fake-UTC epoch.
+    start_epoch = b2.get("DIVE_START_TIME") or b3.get("StartTime")
+    start_local = datetime.fromtimestamp(start_epoch, tz=timezone.utc).strftime(
+        "%Y-%m-%d %H:%M:%S"
+    )
+
+    header = {
+        "pnf": dive.header,
+        "log_data": {
+            "log_id": row["log_id"],
+            "file_name": row["file_name"],
+            "format": row["format"],
+            "format_version": row["format_version"],
+            "data_bytes_2": b2,
+            "data_bytes_3": b3,
+            "calculated_values_from_samples": calc,
+        },
+        "dive_details": details,
+    }
+
+    cur = con.execute(
+        "INSERT INTO source_dives (source, artifact_id, source_key, content_hash,"
+        " decoder_version, start_time_utc, start_time_local, utc_offset_min,"
+        " duration_s, max_depth_m, avg_depth_m, sample_interval_ms, mode,"
+        " header_json, raw_blob)"
+        " VALUES ('shearwater',?,?,?,?,NULL,?,NULL,?,?,?,?,?,?,?)",
+        (
+            artifact_id,
+            source_key,
+            digest,
+            pnf.DECODER_VERSION,
+            start_local,
+            dive.header.get("divetime_s"),
+            dive.header.get("max_depth_m"),
+            calc.get("AverageDepth"),
+            dive.sample_interval_ms,
+            dive.header.get("divemode"),
+            json.dumps(header, separators=(",", ":")),
+            blob,
+        ),
+    )
+    dive_id = cur.lastrowid
+
+    con.executemany(
+        "INSERT INTO shearwater_samples (source_dive_id, t_s, depth_m, temp_c,"
+        " stop_depth_m, stop_or_ndl_min, tts_min, in_deco, avg_ppo2, o2_pct, he_pct,"
+        " setpoint, cns_pct, gf99, ceiling_m, sensor1_raw, sensor2_raw, sensor3_raw,"
+        " sensor1_ppo2, sensor2_ppo2, sensor3_ppo2, battery_v, status_flags,"
+        " tank0_psi, tank1_psi, gas_time_min, extra_json)"
+        " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+        [
+            (
+                dive_id, s.t_s, s.depth_m, s.temp_c,
+                s.stop_depth_m, s.stop_or_ndl_min, s.tts_min, s.in_deco,
+                s.avg_ppo2, s.o2_pct, s.he_pct,
+                s.setpoint, s.cns_pct, s.gf99, s.ceiling_m,
+                s.sensor1_raw, s.sensor2_raw, s.sensor3_raw,
+                s.sensor1_ppo2, s.sensor2_ppo2, s.sensor3_ppo2,
+                s.battery_v, s.status_flags,
+                s.tank0_psi, s.tank1_psi, s.gas_time_min,
+                json.dumps(s.extra, separators=(",", ":")) if s.extra else None,
+            )
+            for s in dive.samples
+        ],
+    )
+
+    for gas in dive.gases:
+        if gas.o2_pct == 0 and gas.he_pct == 0:
+            continue
+        con.execute(
+            "INSERT INTO gases (source_dive_id, gas_index, o2_pct, he_pct, circuit,"
+            " enabled, used, source_json) VALUES (?,?,?,?,?,?,NULL,NULL)",
+            (
+                dive_id,
+                gas.index,
+                gas.o2_pct,
+                gas.he_pct,
+                "cc" if gas.diluent else "oc",
+                1 if gas.enabled else 0,
+            ),
+        )
+
+    tank_profile = details.get("TankProfileData")
+    if tank_profile:
+        try:
+            profile = json.loads(tank_profile)
+        except (TypeError, ValueError):
+            profile = None
+        for seg in (profile or {}).get("GasProfiles") or []:
+            con.execute(
+                "INSERT INTO gas_segments (source_dive_id, start_s, end_s, o2_pct,"
+                " he_pct, circuit, avg_depth_m, source_json) VALUES (?,?,?,?,?,?,?,?)",
+                (
+                    dive_id,
+                    seg.get("StartTimeInSeconds"),
+                    seg.get("EndTimeInSeconds"),
+                    seg.get("O2Percent"),
+                    seg.get("HePercent"),
+                    seg.get("CircuitMode"),
+                    seg.get("AverageDepthInMeters"),
+                    json.dumps(seg, separators=(",", ":")),
+                ),
+            )
+
+    con.executemany(
+        "INSERT INTO events (source_dive_id, t_s, kind, payload_json) VALUES (?,?,?,?)",
+        [
+            (
+                dive_id,
+                e.t_s,
+                "pnf_info_event",
+                json.dumps(
+                    {
+                        "event": e.event,
+                        "timestamp": e.timestamp,
+                        "w1": e.w1,
+                        "w2": e.w2,
+                        "raw_hex": e.raw_hex,
+                    },
+                    separators=(",", ":"),
+                ),
+            )
+            for e in dive.events
+        ],
+    )
+
+    con.executemany(
+        "INSERT INTO undecoded_payloads (source_dive_id, container, type_key, seq,"
+        " payload, fields_json) VALUES (?,?,?,?,?,NULL)",
+        [
+            (dive_id, "pnf_record", type_key, seq, raw)
+            for type_key, seq, raw in dive.unknown_records
+        ],
+    )
+
+    return "ingested"

bottomtime/ingest/shearwater_xml.py

@@ -0,0 +1,92 @@
+"""Parse Shearwater Cloud 'Shearwater XML' exports.
+
+Used as an independent cross-check of the PNF decoder (verify), not as a
+third sample series: the XML is a lossy rendering of the same native log.
+Files declare encoding="utf-16" but are written as ASCII; parse accordingly.
+"""
+
+from __future__ import annotations
+
+import re
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+# AI columns carry text sentinels when no transmitter is paired
+_TEXT_SENTINELS = {"AI is off", "N/A", "Not paired", "No comms", ""}
+
+_NUMERIC_SAMPLE_FIELDS = (
+    "currentTime",
+    "currentDepth",
+    "firstStopDepth",
+    "firstStopTime",
+    "ttsMins",
+    "averagePPO2",
+    "fractionO2",
+    "fractionHe",
+    "currentNdl",
+    "waterTemp",
+    "batteryVoltage",
+    "sensor1Millivolts",
+    "sensor2Millivolts",
+    "sensor3Millivolts",
+    "sac",
+    "sad",
+)
+
+
+def _num(text: str | None) -> float | None:
+    if text is None or text.strip() in _TEXT_SENTINELS:
+        return None
+    try:
+        return float(text)
+    except ValueError:
+        return None
+
+
+def parse_xml(path: Path) -> dict:
+    text = path.read_text(encoding="ascii", errors="replace")
+    text = re.sub(r"^<\?xml[^>]*\?>", "", text, count=1)
+    root = ET.fromstring(text)
+    log = root.find("diveLog")
+
+    header = {
+        el.tag: (el.text or "").strip()
+        for el in log
+        if el.tag != "diveLogRecords" and len(el) == 0
+    }
+
+    samples = []
+    for rec in log.find("diveLogRecords") or []:
+        sample = {}
+        for el in rec:
+            if el.tag in _NUMERIC_SAMPLE_FIELDS:
+                sample[el.tag] = _num(el.text)
+            else:
+                sample[el.tag] = (el.text or "").strip()
+        samples.append(sample)
+
+    return {"path": str(path), "header": header, "samples": samples}
+
+
+def normalize_start(value: str) -> str:
+    """Normalize a Shearwater startDate to 'YYYY-MM-DD HH:MM:SS'.
+
+    Accepts the export's native 'dd-MM-yyyy HH:mm:ss' as well as ISO
+    'yyyy-MM-ddTHH:mm:ss' (files repaired for Garmin Connect upload)."""
+    value = value.strip().replace("T", " ")
+    m = re.match(r"^(\d{2})-(\d{2})-(\d{4}) (\d{2}:\d{2}:\d{2})$", value)
+    if m:
+        return f"{m.group(3)}-{m.group(2)}-{m.group(1)} {m.group(4)}"
+    return value
+
+
+def load_xml_dir(xml_dir: Path) -> dict[str, dict]:
+    """Map normalized start wall-clock -> parsed XML for every export.
+
+    Keyed by start time, not dive number: the on-unit dive counter can be
+    reset, so numbers repeat across a collection."""
+    dives = {}
+    for path in sorted(xml_dir.glob("*.xml")):
+        parsed = parse_xml(path)
+        dives[normalize_start(parsed["header"]["startDate"])] = parsed
+    return dives

bottomtime/match/__init__.py

@@ -0,0 +1 @@
+