clarity-api-sdk-python 0.4.3__tar.gz → 0.4.4__tar.gz

{clarity_api_sdk_python-0.4.3 → clarity_api_sdk_python-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clarity-api-sdk-python
-Version: 0.4.3
+Version: 0.4.4
 Summary: A Python SDK to connect to the CTI Clarity API server.
 Author-email: "Chesapeake Technology Inc." <support@chesapeaketech.com>
 Project-URL: Homepage, https://github.com/chesapeake-tech/clarity-api-sdk-python

{clarity_api_sdk_python-0.4.3 → clarity_api_sdk_python-0.4.4}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "clarity-api-sdk-python"
-version = "0.4.3"
+version = "0.4.4"
 authors = [
   { name="Chesapeake Technology Inc.", email="support@chesapeaketech.com" },
 ]

{clarity_api_sdk_python-0.4.3 → clarity_api_sdk_python-0.4.4}/src/clarity_api_sdk_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clarity-api-sdk-python
-Version: 0.4.3
+Version: 0.4.4
 Summary: A Python SDK to connect to the CTI Clarity API server.
 Author-email: "Chesapeake Technology Inc." <support@chesapeaketech.com>
 Project-URL: Homepage, https://github.com/chesapeake-tech/clarity-api-sdk-python

{clarity_api_sdk_python-0.4.3 → clarity_api_sdk_python-0.4.4}/src/clarity_api_sdk_python.egg-info/SOURCES.txt

@@ -19,6 +19,7 @@ src/cti/cli/__init__.py
 src/cti/cli/__main__.py
 src/cti/cli/client.py
 src/cti/cli/main.py
+src/cti/events/__init__.py
 src/cti/logger/__init__.py
 src/cti/logger/logger.py
 src/cti/model/__init__.py

{clarity_api_sdk_python-0.4.3 → clarity_api_sdk_python-0.4.4}/src/cti/api/sonar_wiz_async_api.py

@@ -1902,6 +1902,27 @@ class SonarWizAsyncApi:
         response.raise_for_status()
         return [FinalProduct.model_validate(item) for item in response.json()]
 
+    async def delete_final_product(self, final_product_id: UUID | str) -> None:
+        """Hard-delete a final product row.
+
+        Used by ``xs_pipeline`` to clear superseded ``FinalProduct`` rows
+        for a (survey, product_type) before inserting the fresh one — so
+        the latest run is the only product the client UI auto-picks.
+        S3 artifacts at the row's ``file_path`` are not touched here;
+        callers that need full cleanup should remove the S3 prefix
+        separately.
+
+        Args:
+            final_product_id: Final product UUID or string identifier.
+
+        Raises:
+            httpx.HTTPStatusError: 404 if the product was already gone.
+        """
+        response = await self._client.delete(
+            f"/api/v1/final-products/{final_product_id}"
+        )
+        response.raise_for_status()
+
     async def update_final_product(
         self, final_product_id: UUID | str, final_product: FinalProductUpdate
     ) -> FinalProduct:

{clarity_api_sdk_python-0.4.3 → clarity_api_sdk_python-0.4.4}/src/cti/cli/main.py

@@ -113,67 +113,20 @@ def upload(
         f"Uploaded: {result.raw_file_id} ({result.file_name}, {result.file_size} bytes)"
     )
 
-    # Auto-create device mappings
+    # Stream-to-device mapping is handled by ``ftm automap`` after upload.
+    # Auto-mapping at upload time was guessing stream URIs (e.g.
+    # ``xtf://sidescan(source=ping_header,pair=0)``) that don't match what
+    # the engine's scan phase actually emits in the manifest, so the
+    # rows it created weren't usable downstream. Run ``ftm automap`` after
+    # upload — it reads the actual scan manifest and replicates the UI's
+    # "Generate auto-platform" + "Auto-mapping" buttons.
     if not no_mappings:
-        try:
-            # Get the survey's platform and its devices
-            platform = api.get_first_survey_platform(survey_id=survey)
-            devices = api.get_platform_devices(platform_id=platform.platform_id)
-
-            # Look up device types to identify sidescan and GNSS
-            dt_list = api.list_device_types()
-            dt_by_id = {dt.device_type_id: dt for dt in dt_list}
-
-            # Get survey timezone for mapping
-            survey_obj = api.get_survey(survey)
-            tz = survey_obj.timezone_name
-
-            mappings_created = 0
-            for device in devices:
-                dt = dt_by_id.get(device.device_type_id)
-                if not dt:
-                    continue
-                if dt.enum_name == "sidescan_sonar":
-                    stream_id = _identifier_from_uri(
-                        sidescan_stream, ("samples", "timestamp")
-                    )
-                elif dt.enum_name == "gnss_gps_position_sensor":
-                    stream_id = _identifier_from_uri(
-                        gnss_stream, ("position", "timestamp")
-                    )
-                else:
-                    continue
-
-                api.create_raw_file_device_mapping(
-                    RawFileDeviceMappingCreate(
-                        raw_file_id=result.raw_file_id,
-                        device_id=device.device_id,
-                        source_stream_identifier=stream_id,
-                        input_srid=4326,
-                        input_timezone=tz,
-                    )
-                )
-                typer.echo(
-                    f"Mapped: {device.name} → stream {stream_id.source.selected}"
-                )
-                mappings_created += 1
-
-            if mappings_created == 0:
-                typer.echo(
-                    "Warning: no sidescan/GNSS devices found on survey platform — no mappings created",
-                    err=True,
-                )
-        except httpx.HTTPStatusError as e:
-            if e.response.status_code == 404:
-                typer.echo(
-                    "Warning: no platform found on survey — skipping device mappings. Run 'ftm init' first.",
-                    err=True,
-                )
-            else:
-                typer.echo(
-                    f"Warning: failed to create device mappings: {_get_error_detail(e)}",
-                    err=True,
-                )
+        typer.echo(
+            "Note: per-stream device mappings are no longer created at upload time. "
+            "Run `ftm automap <survey>` after upload (waits for prescan, then mirrors "
+            "the UI's Auto-platform + Auto-mapping buttons).",
+            err=True,
+        )
 
     _output(
         {
@@ -190,6 +143,281 @@ def upload(
     )
 
 
+# =============================================================================
+# Stream type → seeded DeviceType.enum_name mapping.
+# Mirrors clarity-client/src/lib/auto-platform/stream-device-mapping.ts —
+# keep in sync with that table. Multiple stream types may collapse to
+# the same device type (geographic + projected nav both → GNSS).
+# =============================================================================
+_STREAM_TO_DEVICE_ENUM: dict[str, str] = {
+    "sidescan": "sidescan_sonar",
+    "geographic_navigation": "gnss_gps_position_sensor",
+    "projected_navigation": "gnss_gps_position_sensor",
+    "attitude": "imu",
+    "altitude": "single_beam_echosounder_sbes",
+    "depth": "depth_sensor",
+    "cable_counter": "sheave_block",
+}
+
+
+@app.command()
+def automap(
+    survey: UUID = typer.Argument(..., help="Survey ID to map streams for"),
+    timeout: float = typer.Option(
+        180.0,
+        "--timeout",
+        help="Seconds to wait for prescan to finish across all raw files.",
+    ),
+    output_json: bool = typer.Option(False, "--json", "-j"),
+) -> None:
+    """Auto-platform + auto-map streams for every raw file in a survey.
+
+    Mirrors the clarity-client UI flow operators run after upload:
+
+      1. Wait for the prescan phase to complete on every raw file
+         (the engine writes the parsed stream manifest to
+         ``ProcessingLog.result`` for each file as soon as scan finishes).
+      2. Discover the per-file stream manifests, derive the auto-platform
+         spec from them, and create the platform + one device per stream
+         type — equivalent to clicking "Generate auto-platform" on the
+         Platform page.
+      3. For every (raw_file, stream) in the manifests, find the matching
+         device on the auto-platform (by ``DeviceType.enum_name``) and
+         POST a ``RawFileDeviceMapping`` carrying the scan manifest's
+         exact ``source`` + ``fields_by_source`` shapes — equivalent to
+         clicking "Auto-mapping" on the stream-mapping page.
+
+    Idempotent: existing platforms / devices / mappings are reused; the
+    server's unique-constraint dedup matches the planner's "first wins"
+    rule on ``(raw_file, device)``.
+    """
+    import time
+    from cti.model.device import DeviceCreate
+    from cti.model.platform import PlatformCreate
+    from cti.model.raw_file_device_mapping import (
+        RawFileDeviceMappingCreate,
+        SourceSelection,
+        StreamIdentifier,
+    )
+
+    api = get_api()
+
+    survey_obj = api.get_survey(survey)
+    raw_files = api.get_raw_files(survey)
+    if not raw_files:
+        typer.echo(f"Error: survey {survey} has no raw files; upload first", err=True)
+        raise typer.Exit(1)
+
+    typer.echo(
+        f"Survey {survey_obj.name}: {len(raw_files)} raw file(s); "
+        f"waiting up to {timeout:.0f}s for prescan to finish..."
+    )
+
+    # ── 1. Wait for prescan ─────────────────────────────────────────────
+    deadline = time.monotonic() + timeout
+    manifests: dict[UUID, dict] = {}
+    pending = {rf.raw_file_id for rf in raw_files}
+    while pending and time.monotonic() < deadline:
+        for raw_file_id in list(pending):
+            logs = api.list_processing_logs(
+                raw_file_id=raw_file_id, processing_step="scan"
+            )
+            done = next(
+                (
+                    l for l in logs
+                    if l.status == "complete" and l.result
+                ),
+                None,
+            )
+            if done:
+                try:
+                    manifests[raw_file_id] = json.loads(done.result)
+                except (ValueError, TypeError) as exc:
+                    typer.echo(
+                        f"Error: scan manifest for {raw_file_id} is not valid JSON: {exc}",
+                        err=True,
+                    )
+                    raise typer.Exit(1)
+                pending.discard(raw_file_id)
+        if pending:
+            time.sleep(2.0)
+
+    if pending:
+        typer.echo(
+            f"Error: prescan still pending for {len(pending)} file(s) after {timeout:.0f}s",
+            err=True,
+        )
+        raise typer.Exit(1)
+
+    typer.echo(f"Prescan complete: {len(manifests)} manifest(s) loaded")
+
+    # ── 2. Generate auto-platform (clicks "Generate auto-platform") ─────
+    file_format = next(
+        (m.get("file_format", "").upper() for m in manifests.values() if m.get("file_format")),
+        "MIXED",
+    )
+    auto_name = f"Auto-{file_format}"
+
+    existing_platforms = [
+        p for p in api.list_platforms() if p.survey_id == survey
+    ]
+    auto_platform = next((p for p in existing_platforms if p.name == auto_name), None)
+
+    device_types = api.list_device_types()
+    dt_by_enum = {dt.enum_name: dt for dt in device_types}
+
+    if auto_platform is None:
+        platform_types = api.list_platform_types()
+        if not platform_types:
+            typer.echo("Error: no PlatformType seeded; cannot create auto-platform", err=True)
+            raise typer.Exit(1)
+        auto_platform = api.create_platform(
+            PlatformCreate(
+                survey_id=survey,
+                platform_type_id=platform_types[0].platform_type_id,
+                name=auto_name,
+                description=f"Auto-generated for {file_format} files",
+            )
+        )
+        typer.echo(f"Created auto-platform: {auto_name} ({auto_platform.platform_id})")
+    else:
+        typer.echo(f"Reusing auto-platform: {auto_name} ({auto_platform.platform_id})")
+
+    # Aggregate which DeviceType enums we need across all manifests.
+    needed_enums: set[str] = set()
+    for manifest in manifests.values():
+        for stream in manifest.get("streams", []) or []:
+            stype = stream.get("stream_type")
+            target_enum = _STREAM_TO_DEVICE_ENUM.get(stype) if stype else None
+            if target_enum and target_enum in dt_by_enum:
+                needed_enums.add(target_enum)
+
+    # Existing devices on the auto-platform — first-wins per stream type.
+    existing_devices = api.get_platform_devices(platform_id=auto_platform.platform_id)
+    device_by_enum: dict[str, "Device"] = {}  # type: ignore[name-defined]
+    for d in existing_devices:
+        dt = next((t for t in device_types if t.device_type_id == d.device_type_id), None)
+        if dt and dt.enum_name not in device_by_enum:
+            device_by_enum[dt.enum_name] = d
+
+    # Create devices for any missing enum.
+    created_devices = 0
+    for enum_name in sorted(needed_enums):
+        if enum_name in device_by_enum:
+            continue
+        dt = dt_by_enum[enum_name]
+        # Defaults match the JS auto-platform planner
+        # (clarity-client/src/lib/auto-platform/plan.ts): all offsets
+        # zero, channel 1, is_towed=false. Refinement is deferred —
+        # operators can edit these in the UI after auto-platform creation.
+        new_device = api.create_device(
+            DeviceCreate(
+                platform_id=auto_platform.platform_id,
+                device_type_id=dt.device_type_id,
+                name=dt.name,
+                description=f"Auto-generated for {file_format} {enum_name} stream",
+                channel=1,
+                offset_x=0.0,
+                offset_y=0.0,
+                offset_z=0.0,
+                offset_heading=0.0,
+                offset_pitch=0.0,
+                offset_roll=0.0,
+                latency=0.0,
+                is_towed=False,
+            )
+        )
+        device_by_enum[enum_name] = new_device
+        created_devices += 1
+        typer.echo(f"  + device: {dt.name} ({enum_name})")
+
+    typer.echo(
+        f"Auto-platform devices: {len(device_by_enum)} total ({created_devices} new)"
+    )
+
+    # ── 3. Auto-map streams (clicks "Auto-mapping") ─────────────────────
+    tz = survey_obj.timezone_name
+    mappings_created = 0
+    mappings_skipped = 0
+    for raw_file_id, manifest in manifests.items():
+        # Existing mappings for this file — dedup by device_id (matches
+        # server's unique constraint on (raw_file, device)).
+        existing = api.get_raw_file_device_mappings(raw_file_id=raw_file_id)
+        existing_device_ids = {m.device_id for m in existing}
+        used_device_ids: set = set()
+
+        for stream in manifest.get("streams", []) or []:
+            stype = stream.get("stream_type")
+            target_enum = _STREAM_TO_DEVICE_ENUM.get(stype) if stype else None
+            if not target_enum:
+                continue
+            device = device_by_enum.get(target_enum)
+            if device is None:
+                continue
+            if device.device_id in existing_device_ids or device.device_id in used_device_ids:
+                mappings_skipped += 1
+                continue
+
+            # Build StreamIdentifier from manifest's source + fields_by_source.
+            src = stream.get("source") or {}
+            uri = src.get("selected") if isinstance(src, dict) else None
+            if not uri:
+                continue
+            available = src.get("available", [uri]) if isinstance(src, dict) else [uri]
+            fields_by_source_raw = stream.get("fields_by_source") or {}
+            fields_by_source: dict = {}
+            for fs_uri, roles in fields_by_source_raw.items():
+                role_dict: dict = {}
+                for role_name, role_val in (roles or {}).items():
+                    if isinstance(role_val, dict):
+                        ravail = role_val.get("available", [role_name])
+                        rsel = role_val.get("selected", role_name)
+                    else:
+                        ravail = [str(role_val)]
+                        rsel = str(role_val)
+                    role_dict[role_name] = SourceSelection(
+                        available=list(ravail), selected=rsel
+                    )
+                fields_by_source[fs_uri] = role_dict
+
+            stream_id = StreamIdentifier(
+                source=SourceSelection(available=list(available), selected=uri),
+                fields_by_source=fields_by_source,
+            )
+
+            api.create_raw_file_device_mapping(
+                RawFileDeviceMappingCreate(
+                    raw_file_id=raw_file_id,
+                    device_id=device.device_id,
+                    source_stream_identifier=stream_id,
+                    input_srid=4326,
+                    input_timezone=tz,
+                )
+            )
+            used_device_ids.add(device.device_id)
+            mappings_created += 1
+            typer.echo(
+                f"  + mapping: {raw_file_id} {stype} → {device.name}"
+            )
+
+    typer.echo(
+        f"Done. {mappings_created} mapping(s) created, {mappings_skipped} skipped (already existed)."
+    )
+
+    _output(
+        {
+            "platform_id": str(auto_platform.platform_id),
+            "platform_name": auto_name,
+            "devices_total": len(device_by_enum),
+            "devices_created": created_devices,
+            "mappings_created": mappings_created,
+            "mappings_skipped": mappings_skipped,
+            "files_processed": len(manifests),
+        },
+        output_json,
+    )
+
+
 @app.command()
 def process(
     survey: UUID = typer.Argument(..., help="Survey ID to process"),

clarity_api_sdk_python-0.4.4/src/cti/events/__init__.py

@@ -0,0 +1,152 @@
+"""Typed event classes for structured logging.
+
+Each class is the canonical definition of a log event the dashboard or
+QA harness programs against. Construct an instance and ``emit()`` it
+through any ``cti.logger``-compatible logger::
+
+    from cti.events import TaskCompleted, emit
+
+    emit(
+        TaskCompleted(
+            job_id=jid,
+            phase="ingest",
+            task_type="ingest",
+            worker_pid=os.getpid(),
+            duration_sec=elapsed,
+            bytes_processed=output.bytes,
+            pings_processed=output.pings,
+        ),
+        logger,
+    )
+
+The dataclass IS the contract — pyright catches missing or wrong-typed
+fields at the call site, and ``from cti.events import <Tab>`` discovers
+every defined event. The string carried in ``event_name`` (a
+``ClassVar``, so excluded from ``asdict()``) is what shows up in the log
+stream — kept snake-case so existing dashboard log-search prefixes
+(``task.``, ``module.``) still match.
+
+The set is small on purpose: only events something programs against
+structurally. Free-form ``logger.info("xs_pipeline_v2_uploaded",
+extra={...})`` remains the right shape for diagnostic chatter that
+nothing consumes by name.
+"""
+
+from dataclasses import asdict, dataclass
+from typing import ClassVar
+
+
+def emit(event, logger) -> None:
+    """Emit a typed event through the given structlog-style logger.
+
+    Looks up the per-class ``level`` (info / error / warning / debug)
+    and calls the matching method, so failure events stay at ERROR and
+    pass through level filters the dashboard depends on.
+    """
+    getattr(logger, event.level)(event.event_name, extra=asdict(event))
+
+
+@dataclass(frozen=True)
+class TaskStarted:
+    """A worker task has begun executing inside a phase's pool."""
+
+    event_name: ClassVar[str] = "task.started"
+    level: ClassVar[str] = "info"
+
+    job_id: str
+    phase: str
+    task_type: str
+    worker_pid: int
+    raw_file_id: str | None = None
+    file_name: str | None = None
+    file_size_bytes: int | None = None
+
+
+@dataclass(frozen=True)
+class TaskCompleted:
+    """A worker task finished successfully."""
+
+    event_name: ClassVar[str] = "task.completed"
+    level: ClassVar[str] = "info"
+
+    job_id: str
+    phase: str
+    task_type: str
+    worker_pid: int
+    duration_sec: float
+    raw_file_id: str | None = None
+    file_name: str | None = None
+    file_size_bytes: int | None = None
+    bytes_processed: int | None = None
+    pings_processed: int | None = None
+    batch_size: int | None = None
+
+
+@dataclass(frozen=True)
+class TaskFailed:
+    """A worker task raised an exception. Re-raised after emit."""
+
+    event_name: ClassVar[str] = "task.failed"
+    level: ClassVar[str] = "error"
+
+    job_id: str
+    phase: str
+    task_type: str
+    worker_pid: int
+    duration_sec: float
+    error: str
+    traceback: str
+    raw_file_id: str | None = None
+    file_name: str | None = None
+    file_size_bytes: int | None = None
+
+
+@dataclass(frozen=True)
+class ModuleStarted:
+    """A pipeline module began executing on the orchestrator thread."""
+
+    event_name: ClassVar[str] = "module.started"
+    level: ClassVar[str] = "info"
+
+    module: str
+    phase: str
+    job_id: str | None = None
+
+
+@dataclass(frozen=True)
+class ModuleCompleted:
+    """A pipeline module completed successfully."""
+
+    event_name: ClassVar[str] = "module.completed"
+    level: ClassVar[str] = "info"
+
+    module: str
+    phase: str
+    duration_sec: float
+    job_id: str | None = None
+
+
+@dataclass(frozen=True)
+class ModuleFailed:
+    """A pipeline module raised an exception. Re-raised after emit."""
+
+    event_name: ClassVar[str] = "module.failed"
+    level: ClassVar[str] = "error"
+
+    module: str
+    phase: str
+    duration_sec: float
+    error: str
+    traceback: str
+    job_id: str | None = None
+
+
+__all__ = [
+    "ModuleCompleted",
+    "ModuleFailed",
+    "ModuleStarted",
+    "TaskCompleted",
+    "TaskFailed",
+    "TaskStarted",
+    "emit",
+]