sift-stack-py 0.17.0.dev1__tar.gz → 0.17.0.dev2__tar.gz

{sift_stack_py-0.17.0.dev1 → sift_stack_py-0.17.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sift_stack_py
-Version: 0.17.0.dev1
+Version: 0.17.0.dev2
 Summary: Python client library for the Sift API
 Maintainer-email: Sift Software Engineers <engineering@siftstack.com>
 License: Copyright (c) 2024 Azimuth Industries Inc.
@@ -46,6 +46,7 @@ Requires-Dist: types-requests~=2.25
 Requires-Dist: googleapis-common-protos>=1.60
 Requires-Dist: protoc-gen-openapiv2>=0.0.1
 Requires-Dist: filelock~=3.13
+Requires-Dist: tomli~=2.0; python_version < "3.11"
 Provides-Extra: all
 Requires-Dist: cffi~=1.14; extra == "all"
 Requires-Dist: h5py~=3.11; extra == "all"

{sift_stack_py-0.17.0.dev1 → sift_stack_py-0.17.0.dev2}/lib/sift_client/_internal/low_level_wrappers/_test_results_log.py

@@ -143,9 +143,13 @@ class _ReplayState:
 
 @dataclass
 class ReplayResult:
-    """Result of replaying a log file."""
+    """Result of replaying a log file.
 
-    report: TestReport
+    ``report`` is None on an incremental resume tick that uploaded only steps or
+    measurements; the report itself was created on an earlier tick.
+    """
+
+    report: TestReport | None = None
     steps: list[TestStep] = field(default_factory=list)
     measurements: list[TestMeasurement] = field(default_factory=list)
 

{sift_stack_py-0.17.0.dev1 → sift_stack_py-0.17.0.dev2}/lib/sift_client/_internal/low_level_wrappers/test_results.py

@@ -1072,13 +1072,17 @@ class TestResultsLowLevelClient(LowLevelClientBase, WithGrpcClient):
         id_map: dict[str, str],
         state: _ReplayState,
     ) -> None:
-        if state.report is None:
-            raise ValueError("UpdateTestReport found before CreateTestReport")
         request = UpdateTestReportRequest()
         json_format.Parse(json_str, request)
         request.test_report.test_report_id = self._map_id(
             id_map, request.test_report.test_report_id
         )
+        # Batch/simulate replays the whole log in order, so a missing report means
+        # the log is malformed. Incremental replay may have created the report on an
+        # earlier tick (its real ID lives in id_map), so state.report is legitimately
+        # None here -- the mapped ID is enough to issue the update.
+        if simulate and state.report is None:
+            raise ValueError("UpdateTestReport found before CreateTestReport")
         state.report = await self.update_test_report(
             request=request, simulate=simulate, existing=state.report
         )
@@ -1203,6 +1207,7 @@ class TestResultsLowLevelClient(LowLevelClientBase, WithGrpcClient):
         next tick.
         """
         tracking = LogTracking.load(log_path)
+        resuming = tracking.last_uploaded_line > 0
         id_map = tracking.id_map
         state = _ReplayState()
 
@@ -1221,7 +1226,10 @@ class TestResultsLowLevelClient(LowLevelClientBase, WithGrpcClient):
             tracking.last_uploaded_line += 1
             tracking.save(log_path)
 
-        if state.report is None:
+        # On a resume tick the CreateTestReport line was consumed on an earlier
+        # tick, so state.report is expected to be None; the report already exists
+        # on the server. Only a genuine first pass over an empty log is an error.
+        if state.report is None and not resuming:
             raise ValueError("No CreateTestReport found in log file")
 
         return ReplayResult(

sift_stack_py-0.17.0.dev2/lib/sift_client/_internal/pyproject_config.py

@@ -0,0 +1,84 @@
+"""Loader for the ``[tool.sift]`` table in a project's ``pyproject.toml``.
+
+The pytest plugin consumes this loader to resolve report-content config (under
+``[tool.sift.pytest.report]``) and SDK-level fallbacks (URIs under
+``[tool.sift]``). A malformed or missing ``pyproject.toml`` returns ``{}`` so a
+bad config file never aborts the session — the plugin falls back to its
+built-in defaults and surfaces a single warning.
+"""
+
+from __future__ import annotations
+
+import warnings
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+# ``tomllib`` landed in 3.11; ``tomli`` is the same parser packaged for older
+# interpreters and is declared as a conditional install dep on 3.8-3.10.
+try:
+    import tomllib  # type: ignore[import-not-found,unused-ignore]
+except ImportError:  # pragma: no cover - exercised on 3.8-3.10 only
+    import tomli as tomllib  # type: ignore[no-redef,import-not-found,unused-ignore]
+
+if TYPE_CHECKING:
+    import pytest
+
+
+# Bound the upward walk so a misconfigured environment can't trigger an
+# unbounded filesystem traversal looking for a project root that isn't there.
+_MAX_PARENT_WALK = 3
+
+
+def _find_pyproject(config: pytest.Config) -> Path | None:
+    """Locate the active project's ``pyproject.toml``.
+
+    Order:
+    1. ``config.inipath`` when it is itself a ``pyproject.toml`` (the common
+       case: project uses ``[tool.pytest.ini_options]`` so pytest loaded the
+       ini settings directly from pyproject).
+    2. ``<config.rootpath>/pyproject.toml``.
+    3. A bounded walk upward from ``rootpath`` for monorepo layouts where
+       pytest's rootdir is a subdirectory and the project pyproject lives
+       higher up.
+    """
+    inipath = config.inipath
+    if inipath is not None and inipath.name == "pyproject.toml" and inipath.is_file():
+        return inipath
+    cur = Path(config.rootpath).resolve()
+    candidate = cur / "pyproject.toml"
+    if candidate.is_file():
+        return candidate
+    for _ in range(_MAX_PARENT_WALK):
+        cur = cur.parent
+        candidate = cur / "pyproject.toml"
+        if candidate.is_file():
+            return candidate
+    return None
+
+
+def load_tool_sift(config: pytest.Config) -> dict[str, Any]:
+    """Return the parsed ``[tool.sift]`` table from the project's pyproject.toml.
+
+    Returns ``{}`` when no pyproject is discoverable, when the file omits the
+    ``[tool.sift]`` table, or when parsing fails. A parse / IO failure emits a
+    single :class:`SiftPytestPluginWarning` so the session continues with
+    defaults rather than aborting on a malformed file.
+    """
+    pyproject = _find_pyproject(config)
+    if pyproject is None:
+        return {}
+    try:
+        with pyproject.open("rb") as fh:
+            data = tomllib.load(fh)
+    except (OSError, tomllib.TOMLDecodeError) as exc:
+        # Deferred import: ``pytest_plugin`` imports this loader, so a
+        # top-level import here would close the cycle at module load time.
+        from sift_client.pytest_plugin import SiftPytestPluginWarning
+
+        warnings.warn(
+            f"Failed to read {pyproject} for [tool.sift]: {type(exc).__name__}: {exc}",
+            SiftPytestPluginWarning,
+            stacklevel=2,
+        )
+        return {}
+    return (data.get("tool") or {}).get("sift") or {}

sift_stack_py-0.17.0.dev2/lib/sift_client/_internal/util/hdf5.py

@@ -0,0 +1,271 @@
+"""HDF5 schema detection.
+
+HDF5 files have no single canonical layout, so detection is parameterized
+by an ``Hdf5Schema``: ``ONE_D`` (per-group time dataset + sibling 1D values,
+with an ancestor walk-up), ``TWO_D`` (``[N, 2]`` datasets where col 0 is
+time), or ``COMPOUND`` (struct-like datasets whose first field is time).
+Each detector walks every dataset in the file recursively; datasets that
+don't fit the chosen schema are not included in the resulting config."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Callable
+
+import h5py
+import numpy as np
+
+from sift_client._internal.util.numpy_types import numpy_to_sift_type
+from sift_client.sift_types.data_import import (
+    DataTypeKey,
+    Hdf5DataColumn,
+    Hdf5ImportConfig,
+)
+
+# Heuristic attribute names for channel metadata, in priority order. The
+# first non-empty value found on a dataset wins; missing attributes resolve
+# to empty strings.
+_NAME_ATTRS = ("Name", "name", "Title", "title", "Sensor", "sensor", "Channel", "channel")
+_UNIT_ATTRS = ("Unit", "unit", "Units", "units")
+_DESCRIPTION_ATTRS = ("Description", "description")
+
+# Per-group time dataset names, case-insensitive, in priority order.
+_TIME_DATASET_NAMES = ("time", "timestamp", "timestamps", "ts")
+
+
+def _read_string_attr(dataset: h5py.Dataset, candidates: tuple[str, ...]) -> str:
+    """Return the first non-empty string attribute among `candidates`."""
+    for name in candidates:
+        if name not in dataset.attrs:
+            continue
+        value = dataset.attrs[name]
+        if isinstance(value, bytes):
+            value = value.decode("utf-8", errors="replace")
+        if isinstance(value, str) and value:
+            return value
+        # h5py returns multi-element string attrs as ndarrays; take the first.
+        if isinstance(value, np.ndarray) and value.size > 0:
+            first = value.flat[0]
+            if isinstance(first, bytes):
+                first = first.decode("utf-8", errors="replace")
+            if isinstance(first, str) and first:
+                return first
+    return ""
+
+
+def _read_channel_metadata(dataset: h5py.Dataset) -> tuple[str, str, str]:
+    """Return ``(name, units, description)`` discovered from HDF5 attributes."""
+    return (
+        _read_string_attr(dataset, _NAME_ATTRS),
+        _read_string_attr(dataset, _UNIT_ATTRS),
+        _read_string_attr(dataset, _DESCRIPTION_ATTRS),
+    )
+
+
+def _is_compound(dataset: h5py.Dataset) -> bool:
+    return dataset.dtype.names is not None and len(dataset.dtype.names) > 1
+
+
+def _is_1d_non_compound(dataset: h5py.Dataset) -> bool:
+    return not _is_compound(dataset) and len(dataset.shape) == 1
+
+
+def _is_2d_n_by_2(dataset: h5py.Dataset) -> bool:
+    return not _is_compound(dataset) and len(dataset.shape) == 2 and dataset.shape[1] == 2
+
+
+def _path_to_channel_name(path: str) -> str:
+    """Sift renders dotted names hierarchically, so ``group1/current`` becomes
+    ``group1.current``, with ``current`` shown under a ``group1`` folder."""
+    return path.replace("/", ".")
+
+
+def _make_name_deduper() -> Callable[[str, str], str]:
+    """Return a callable that resolves duplicate channel names by appending
+    the dataset's dotted path. First claim of a name wins; later claims of
+    the same name get the fallback suffix appended."""
+    used: set[str] = set()
+
+    def dedupe(base_name: str, fallback_suffix: str) -> str:
+        name = f"{base_name}.{fallback_suffix}" if base_name in used else base_name
+        used.add(name)
+        return name
+
+    return dedupe
+
+
+def _collect_datasets(h5file: h5py.File) -> list[h5py.Dataset]:
+    """Recursively walk every dataset in the file."""
+    out: list[h5py.Dataset] = []
+
+    def visit(_name: str, obj: object) -> None:
+        if isinstance(obj, h5py.Dataset):
+            out.append(obj)
+
+    h5file.visititems(visit)
+    return out
+
+
+def _group_by_parent(datasets: list[h5py.Dataset]) -> dict[str, list[h5py.Dataset]]:
+    """Group datasets by their parent group path (``""`` for root-level)."""
+    out: dict[str, list[h5py.Dataset]] = {}
+    for ds in datasets:
+        out.setdefault(ds.name.lstrip("/").rpartition("/")[0], []).append(ds)
+    return out
+
+
+def _resolve_ancestor_time(group_path: str, per_group_time: dict[str, str]) -> str:
+    """Return the closest-ancestor time dataset path for ``group_path``,
+    walking up to the root. Empty string if no ancestor has one."""
+    cursor = group_path
+    while True:
+        found = per_group_time.get(cursor)
+        if found:
+            return found
+        if cursor == "":
+            return ""
+        slash = cursor.rfind("/")
+        cursor = "" if slash < 0 else cursor[:slash]
+
+
+def _build_one_d_configs(datasets: list[h5py.Dataset]) -> list[Hdf5DataColumn]:
+    """1D non-compound schema: at each group, pick a time dataset (by name)
+    and pair every other 1D dataset in that group as a value channel.
+    Datasets that aren't 1D non-compound are not included."""
+
+    def identify_time_dataset(group: list[h5py.Dataset]) -> h5py.Dataset | None:
+        """Pick the group's time dataset by leaf name, case-insensitive, in
+        priority order. Returns ``None`` if no candidate matches; callers
+        fall back to an ancestor group's time before giving up."""
+        for candidate in _TIME_DATASET_NAMES:
+            for ds in group:
+                if ds.name.rsplit("/", 1)[-1].lower() == candidate:
+                    return ds
+        return None
+
+    columns: list[Hdf5DataColumn] = []
+    dedupe = _make_name_deduper()
+
+    one_d = [ds for ds in datasets if _is_1d_non_compound(ds)]
+    grouped = _group_by_parent(one_d)
+
+    # First pass: each group's own time dataset (if any).
+    per_group_time: dict[str, str] = {}
+    for group_path, group in grouped.items():
+        time_ds = identify_time_dataset(group)
+        if time_ds is not None:
+            per_group_time[group_path] = time_ds.name.lstrip("/")
+
+    for group_path, group in grouped.items():
+        own_time_path = per_group_time.get(group_path)
+        time_path = own_time_path or _resolve_ancestor_time(group_path, per_group_time)
+        for ds in group:
+            ds_path = ds.name.lstrip("/")
+            if own_time_path and ds_path == own_time_path:
+                continue
+            name, units, description = _read_channel_metadata(ds)
+            fallback = _path_to_channel_name(ds_path)
+            columns.append(
+                Hdf5DataColumn(
+                    name=dedupe(name or fallback, fallback),
+                    data_type=numpy_to_sift_type(ds.dtype),
+                    units=units,
+                    description=description,
+                    time_dataset=time_path,
+                    value_dataset=ds_path,
+                    time_index=0,
+                    value_index=0,
+                )
+            )
+
+    return columns
+
+
+def _build_two_d_configs(datasets: list[h5py.Dataset]) -> list[Hdf5DataColumn]:
+    """2D schema: every dataset with shape ``[N, 2]`` becomes one channel
+    (col 0 = time, col 1 = value). Other shapes are not included."""
+    columns: list[Hdf5DataColumn] = []
+    dedupe = _make_name_deduper()
+
+    for ds in datasets:
+        if not _is_2d_n_by_2(ds):
+            continue
+        ds_path = ds.name.lstrip("/")
+        name, units, description = _read_channel_metadata(ds)
+        fallback = _path_to_channel_name(ds_path)
+        columns.append(
+            Hdf5DataColumn(
+                name=dedupe(name or fallback, fallback),
+                data_type=numpy_to_sift_type(ds.dtype),
+                units=units,
+                description=description,
+                time_dataset=ds_path,
+                value_dataset=ds_path,
+                time_index=0,
+                value_index=1,
+            )
+        )
+
+    return columns
+
+
+def _build_compound_configs(datasets: list[h5py.Dataset]) -> list[Hdf5DataColumn]:
+    """Compound schema: every compound dataset becomes one channel per
+    non-time member. First member is time. Non-compound datasets are not included."""
+    columns: list[Hdf5DataColumn] = []
+    dedupe = _make_name_deduper()
+
+    for ds in datasets:
+        if not _is_compound(ds):
+            continue
+        field_names = ds.dtype.names
+        assert field_names is not None  # guaranteed by _is_compound
+        time_field = field_names[0]
+        value_fields = field_names[1:]
+        ds_path = ds.name.lstrip("/")
+        name, units, description = _read_channel_metadata(ds)
+        dataset_name = name or _path_to_channel_name(ds_path)
+
+        for value_field in value_fields:
+            base_name = f"{dataset_name}.{value_field}" if len(value_fields) > 1 else dataset_name
+            fallback_suffix = f"{_path_to_channel_name(ds_path)}.{value_field}"
+            columns.append(
+                Hdf5DataColumn(
+                    name=dedupe(base_name, fallback_suffix),
+                    data_type=numpy_to_sift_type(ds.dtype[value_field]),
+                    units=units,
+                    description=description,
+                    time_dataset=ds_path,
+                    value_dataset=ds_path,
+                    time_index=0,
+                    value_index=0,
+                    time_field=time_field,
+                    value_field=value_field,
+                )
+            )
+
+    return columns
+
+
+_BUILDERS: dict[DataTypeKey, Callable[[list[h5py.Dataset]], list[Hdf5DataColumn]]] = {
+    DataTypeKey.HDF5_ONE_D: _build_one_d_configs,
+    DataTypeKey.HDF5_TWO_D: _build_two_d_configs,
+    DataTypeKey.HDF5_COMPOUND: _build_compound_configs,
+}
+
+
+def detect_hdf5_config(file_path: str | Path, data_type_key: DataTypeKey) -> Hdf5ImportConfig:
+    """Detect an HDF5 import config under the given variant. Datasets that
+    don't fit the chosen variant are not included. ``time_format`` is always
+    left unset: HDF5 timestamps aren't self-describing, so the caller must set
+    ``config.time_format`` before importing."""
+    if data_type_key not in _BUILDERS:
+        raise ValueError(
+            f"detect_hdf5_config requires an HDF5 DataTypeKey variant "
+            f"(HDF5_ONE_D, HDF5_TWO_D, or HDF5_COMPOUND); got {data_type_key}."
+        )
+    path = Path(file_path)
+    with h5py.File(path, "r") as h5file:
+        columns = _BUILDERS[data_type_key](_collect_datasets(h5file))
+
+    return Hdf5ImportConfig(asset_name="", data=columns)