pyoncat 2.7__tar.gz → 2.9__tar.gz

{pyoncat-2.7 → pyoncat-2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyoncat
-Version: 2.7
+Version: 2.9
 Summary: A Python Client for ONCat (the ORNL Neutron Catalog).
 Keywords: ornl,neutron,metadata,catalog
 Author: ONCat Support
@@ -30,7 +30,7 @@ The `pyoncat` package serves as a Python client to more easily interact with the
 
 ### Requirements
 
-- Python version: `>=3.7, <4`
+- Python version: `>=3.9, <4`
 - Python packages: `requests`
 - Optional packages for authenticated API access: `oauthlib`, `requests-oauthlib`
 

{pyoncat-2.7 → pyoncat-2.9}/pyoncat/README.md

@@ -10,7 +10,7 @@ The `pyoncat` package serves as a Python client to more easily interact with the
 
 ### Requirements
 
-- Python version: `>=3.7, <4`
+- Python version: `>=3.9, <4`
 - Python packages: `requests`
 - Optional packages for authenticated API access: `oauthlib`, `requests-oauthlib`
 

{pyoncat-2.7 → pyoncat-2.9}/pyoncat/__init__.py

@@ -34,7 +34,7 @@ from pyoncat.reduction import ReductionFileBuilder
 if TYPE_CHECKING:
     import threading
 
-__version__ = "2.7"
+__version__ = "2.9"
 
 # Sent on every request PyONCat makes, API calls and token requests alike,
 # so server logs can tell PyONCat traffic from other clients.
@@ -58,6 +58,7 @@ __all__ = [
     "REAUTH_NEVER",
     "REAUTH_PROMPT",
     "RESOURCE_OWNER_CREDENTIALS_FLOW",
+    "ConflictError",
     "DeviceAuthorizationCancelled",
     "DeviceAuthorizationChallenge",
     "DeviceAuthorizationDenied",
@@ -409,6 +410,7 @@ class ONCat:
         Cycle: Cycle
         Datafile: Datafile
         Reduction: Reduction
+        Dataset: Dataset
         Simulation: Simulation
         User: User
         Task: Task
@@ -560,6 +562,9 @@ class ONCat:
                 if error.response.status_code == 404:
                     msg = f'Could not find resource at "{full_url}"'
                     raise NotFoundError(msg) from error
+                if error.response.status_code == 409:
+                    msg = _conflict_message(error.response)
+                    raise ConflictError(msg) from error
                 msg = f'Error: "{error!s}"'
                 raise PyONCatError(msg) from error
 
@@ -763,7 +768,7 @@ class ONCat:
 
         Revocation uses the configured public `client_id` and no secret
         (the device-authorization client is a public client, RFC 8628
-        / ADR-0058).
+        / ADR-000058).
 
         Per RFC 7009 §2.2 the revocation endpoint returns HTTP 200 even for
         a token that is already invalid, so a 200 confirms the request was
@@ -932,12 +937,13 @@ class ONCat:
             device_authorization_timeout: float | None
                 Device Authorization Grant only.  An overall caller
                 deadline in seconds for the sign-in, distinct from the
-                RFC 8628 server-side `expires_in`.  When set, polling
-                stops at the deadline with `DeviceAuthorizationTimeout`.
-                Polling also stops no later than the server's own
-                `expires_in` (the effective deadline is the earlier of
-                the two).  `None` (the default) means no overall deadline
-                -- today's behavior.
+                RFC 8628 server-side `expires_in`.  Polling stops at the
+                earlier of the two.  When the caller deadline is reached
+                first, the sign-in raises `DeviceAuthorizationTimeout`;
+                when the server's `expires_in` is reached first (or at
+                the same moment, or no caller deadline is set), it raises
+                `DeviceAuthorizationExpired`.  `None` (the default) means
+                no overall caller deadline.
 
             http_timeout: float | None
                 Device Authorization Grant only.  The connect/read HTTP
@@ -1138,7 +1144,7 @@ class ONCat:
             device_auth_body["scope"] = scope_str
 
         # The caller's `device_authorization_timeout` bounds the WHOLE sign-in
-        # (ADR-0097), so the deadline clock starts BEFORE the initial
+        # (ADR-000097), so the deadline clock starts BEFORE the initial
         # device-authorization POST. A slow initial POST therefore counts
         # against the overall budget. `http_timeout` still caps each
         # individual POST; the caller deadline is the cumulative bound.
@@ -1207,14 +1213,22 @@ class ONCat:
         # The effective deadline is the earlier of the caller's overall
         # `device_authorization_timeout` (clocked from before the
         # initial POST) and the server's own `expires_in` (clocked from
-        # now, once known).
+        # now, once known). Which one it is decides the outcome raised
+        # when it passes: the server expiry means the device code is dead
+        # (`DeviceAuthorizationExpired`, the same outcome as a server
+        # `expired_token`), the caller deadline means the caller gave up
+        # (`DeviceAuthorizationTimeout`). On a tie the device code is dead
+        # either way, so the server expiry wins.
         server_expiry = (
             monotonic() + expires_in if expires_in is not None else None
         )
-        deadlines = [
-            d for d in (caller_deadline, server_expiry) if d is not None
-        ]
-        deadline = min(deadlines) if deadlines else None
+        deadline = caller_deadline
+        deadline_is_server_expiry = False
+        if server_expiry is not None and (
+            caller_deadline is None or server_expiry <= caller_deadline
+        ):
+            deadline = server_expiry
+            deadline_is_server_expiry = True
 
         self._dispatch_device_authorization_challenge(
             prompt,
@@ -1236,10 +1250,18 @@ class ONCat:
 
         while True:
             self._raise_if_cancelled(cancel_event)
-            self._raise_if_past_deadline(deadline, monotonic)
+            self._raise_if_past_deadline(
+                deadline,
+                monotonic,
+                server_expiry=deadline_is_server_expiry,
+            )
             sleep(self._poll_wait_seconds(interval, deadline, monotonic))
             self._raise_if_cancelled(cancel_event)
-            self._raise_if_past_deadline(deadline, monotonic)
+            self._raise_if_past_deadline(
+                deadline,
+                monotonic,
+                server_expiry=deadline_is_server_expiry,
+            )
 
             try:
                 token_response = requests.post(
@@ -1372,14 +1394,32 @@ class ONCat:
 
     @staticmethod
     def _raise_if_past_deadline(
-        deadline: float | None, monotonic: Callable[[], float]
+        deadline: float | None,
+        monotonic: Callable[[], float],
+        *,
+        server_expiry: bool = False,
     ) -> None:
-        if deadline is not None and monotonic() >= deadline:
+        """Raise once `deadline` has passed.
+
+        `server_expiry` says the deadline is the device code's own
+        `expires_in`, so passing it raises `DeviceAuthorizationExpired`;
+        otherwise it is the caller's `device_authorization_timeout` and
+        passing it raises `DeviceAuthorizationTimeout`.
+        """
+        if deadline is None or monotonic() < deadline:
+            return
+        if server_expiry:
             msg = (
-                "Device authorization timed out before the user completed "
-                "the sign-in."
+                "Device authorization expired: the device code's "
+                "expires_in lapsed before the user completed the sign-in."
             )
-            raise DeviceAuthorizationTimeout(msg)
+            raise DeviceAuthorizationExpired(msg)
+        msg = (
+            "Device authorization timed out: the caller's "
+            "device_authorization_timeout lapsed before the user completed "
+            "the sign-in."
+        )
+        raise DeviceAuthorizationTimeout(msg)
 
     def _dispatch_device_authorization_challenge(
         self,
@@ -1798,6 +1838,57 @@ class Reduction(
     _NAMESPACE = "api"
 
 
+class Dataset(_ListableONCatMixin):
+    """The Datasets of an instrument: groups of reductions that belong
+    together, read-only.
+
+    Every cataloged reduction belongs to exactly one Dataset, and every
+    Dataset to exactly one experiment.  ONCat decides a reduction's
+    Dataset when the reduction file is ingested and stores its
+    identifier as the reduction's `indexed.dataset`.  The identifier
+    takes one of two forms: `<experiment>/<name>`, such as
+    `IPTS-20841/my-dataset`, for the reductions of an experiment whose
+    reduction files give the same `dataset.name` (see
+    `ReductionFileBuilder.set_dataset_name`); and the reduction file's
+    location, such as
+    `/SNS/REF_M/IPTS-20841/shared/autoreduce/REF_M_32025.json`, for any
+    other reduction, autoreduced or not, which is a Dataset of its own.
+    Datasets exist only on instruments where ONCat builds them; on any
+    other instrument `list` returns an empty list.
+
+    The ONCat website calls a named Dataset a Reduction Set and an
+    unnamed one a Reduction; both are this resource.
+
+    A Dataset record holds `id`, `name` (`None` when unnamed),
+    `facility`, `instrument`, `experiment`, `reductions` and `runs`
+    (each `{ranges, min, max, count}`: `ranges` lists `[first, last]`
+    pairs of reduction keys or run numbers), `earliest` and `latest`
+    (each `{created, ingested, modified}`), and `metadata`, a summary of
+    the reductions' metadata whose leaves hold `min`, `max`, `values`
+    (a list of strings, or the string `"TRUNCATED"` once a path has too
+    many distinct strings to list), and `booleans` (the distinct
+    booleans at the path, sorted: `[False]`, `[True]` or
+    `[False, True]`; present only when a reduction holds a boolean
+    there).
+
+    `retrieve` takes an identifier in either form, and `list`
+    takes the `experiment`, `name`, and `run` filters and paging
+    parameters; `run` keeps the Datasets whose `runs` ranges include
+    that run, whatever their experiment.
+    List a Dataset's reductions with `Reduction.list(..., dataset=...)`:
+
+        dataset = oncat.Dataset.retrieve(
+            "IPTS-20841/my-dataset", facility="SNS", instrument="REF_M"
+        )
+        reductions = oncat.Reduction.list(
+            facility="SNS", instrument="REF_M", dataset=dataset.id
+        )
+    """
+
+    _RESOURCE_ENDPOINT = "datasets"
+    _NAMESPACE = "api"
+
+
 class Simulation(_ControllerONCatMixin, _ListableONCatMixin):
     _RESOURCE_ENDPOINT = "simulations"
     _NAMESPACE = "api"
@@ -2004,6 +2095,18 @@ def _yield_nodes(d, path, include_branches):
             yield node_path
 
 
+def _conflict_message(response):
+    """Return the message a 409 response body carries, which names the
+    state that refused the request, or the response's status line when
+    the body holds none."""
+    try:
+        body = response.json()
+    except ValueError:
+        body = None
+    message = body.get("message") if isinstance(body, dict) else None
+    return message or f'Error: "{response.status_code} {response.reason}"'
+
+
 ########################################################################
 # Exceptions
 ########################################################################
@@ -2054,6 +2157,16 @@ class BadRequestError(PyONCatError):
     """
 
 
+class ConflictError(PyONCatError):
+    """An error that is raised when a request conflicts with the current
+    state of the item it acts on, such as submitting a DOI that is no
+    longer drafting.
+
+    The message is the API's explanation. The same request can succeed
+    once the item's state allows it, so check the item before retrying.
+    """
+
+
 class DeviceAuthorizationError(PyONCatError):
     """An error that is raised when the OAuth 2.0 Device Authorization
     Grant (RFC 8628) flow cannot complete (for example, the user did not
@@ -2072,28 +2185,32 @@ class DeviceAuthorizationError(PyONCatError):
 # The four outcome subclasses below intentionally omit the `Error` suffix
 # (triggering N818): the names are the public 2.6 device-authorization
 # API contract
-# (ADR-0097), spelled to read as outcomes the caller catches by reason --
+# (ADR-000097), spelled to read as outcomes the caller catches by reason --
 # Denied / Expired / Cancelled / Timeout. They subclass
 # `DeviceAuthorizationError`, so the suffixed base type still catches them.
-class DeviceAuthorizationDenied(DeviceAuthorizationError):  # ruff: ignore[error-suffix-on-exception-name] - public API name (ADR-0097)
+class DeviceAuthorizationDenied(DeviceAuthorizationError):  # ruff: ignore[error-suffix-on-exception-name] - public API name (ADR-000097)
     """Raised when the user explicitly denied the sign-in request
     (the IdP returned `access_denied`, RFC 8628 §3.5)."""
 
 
-class DeviceAuthorizationExpired(DeviceAuthorizationError):  # ruff: ignore[error-suffix-on-exception-name] - public API name (ADR-0097)
+class DeviceAuthorizationExpired(DeviceAuthorizationError):  # ruff: ignore[error-suffix-on-exception-name] - public API name (ADR-000097)
     """Raised when the device code expired before the user approved the
-    sign-in (the IdP returned `expired_token`, RFC 8628 §3.5)."""
+    sign-in: the IdP returned `expired_token` (RFC 8628 §3.5), or the
+    server's `expires_in` lapsed no later than the caller's
+    `device_authorization_timeout`. The code is dead, so a new sign-in
+    must start from a fresh device code."""
 
 
-class DeviceAuthorizationCancelled(DeviceAuthorizationError):  # ruff: ignore[error-suffix-on-exception-name] - public API name (ADR-0097)
+class DeviceAuthorizationCancelled(DeviceAuthorizationError):  # ruff: ignore[error-suffix-on-exception-name] - public API name (ADR-000097)
     """Raised when the caller cancelled the sign-in in progress via the
     `cancel_event` passed to `login()`."""
 
 
-class DeviceAuthorizationTimeout(DeviceAuthorizationError):  # ruff: ignore[error-suffix-on-exception-name] - public API name (ADR-0097)
+class DeviceAuthorizationTimeout(DeviceAuthorizationError):  # ruff: ignore[error-suffix-on-exception-name] - public API name (ADR-000097)
     """Raised when the sign-in did not complete before the overall
-    caller deadline (`device_authorization_timeout`) or the server's own
-    `expires_in`, whichever came first."""
+    caller deadline (`device_authorization_timeout`) and that deadline
+    came strictly before the server's own `expires_in`. A server expiry
+    reached first raises `DeviceAuthorizationExpired` instead."""
 
 
 class DeviceAuthorizationNetworkError(DeviceAuthorizationError):

{pyoncat-2.7 → pyoncat-2.9}/pyoncat/reduction.py

@@ -10,6 +10,51 @@ import sys
 import traceback
 from typing import Any
 
+# PyONCat is installed without the rest of this repository, so the
+# constants below copy ONCat's rules instead of importing them.
+
+# ONCat takes a reduction file's facility and instrument from the first
+# two folders of the file's location.
+_LOCATION_REGEX = re.compile(
+    r"^/(?P<facility>HFIR|SNS)/(?P<instrument>[A-Z0-9_]+)/"
+)
+
+# ONCat's pattern for an instrument-level shared file: a file under
+# /<FAC>/<INST>/shared/ with no IPTS-<n> folder on the way.  ONCat
+# accepts such a file as an input only when <FAC> and <INST> are those
+# of the reduction file's own location.
+_SHARED_INPUT_REGEX = re.compile(
+    r"^/(?P<facility>HFIR|SNS)/(?P<instrument>[A-Z0-9_]*)/shared/"
+    r"(?:(?!IPTS-\d+/)[^/]+/)*[^/]+$"
+)
+
+# A raw input inside an experiment: /<FAC>/<INST>/IPTS-<n>/.../<name>.
+_RAW_INPUT_REGEX = re.compile(
+    r"^/(?P<facility>SNS|HFIR)/"
+    r"(?P<instrument>[A-Z0-9_]+)/"
+    r"(?P<experiment>IPTS-\d+)/"
+    r"(?:[^/]+/)*"
+    r"(?P<name>[^/]+)$"
+)
+
+_TRAILING_RUN_NUMBER_REGEX = re.compile(
+    r"_(\d+)(?:_event|_histo)?\.nxs(?:\.h5)?$"
+)
+
+# ONCat refuses a Dataset name with any of these segments, the text
+# between two "/": a web address rewrites them, so the Dataset could not
+# be opened by its address.
+_REFUSED_DATASET_NAME_SEGMENTS = ("", ".", "..")
+
+
+def _fspath(path: str | os.PathLike[str]) -> str:
+    """Return os.fspath(path) for a path object, and path itself otherwise.
+
+    The builder stores, validates, and writes locations as str, so a
+    pathlib.Path gives the same result as its str.  Any other value,
+    bytes included, passes through as the caller gave it."""
+    return os.fspath(path) if isinstance(path, os.PathLike) else path
+
 
 class ReductionFileBuilder:
     """Builder for creating JSON Reduction Files for ONCat cataloging.
@@ -30,8 +75,9 @@ class ReductionFileBuilder:
         5. Write the JSON file to disk
 
     All mutator methods return `self` for convenient method chaining.
-    Use `write_safe()` in autoreduction scripts to ensure cataloging
-    failures do not break the reduction.
+    `write_safe()` never raises, so a failed write does not stop the rest
+    of the script; its stderr output marks an SNS autoreduction job as
+    failed.
 
     Example usage:
 
@@ -75,29 +121,46 @@ class ReductionFileBuilder:
         """
         self._user = user
         self._metadata = metadata if metadata is not None else {}
+        # Kept apart from metadata, so set_metadata() and
+        # update_metadata() never drop a name set_dataset_name() set.
+        self._dataset: dict[str, Any] | None = None
         self._input_files: list[dict[str, Any]] = []
         self._output_files: list[dict[str, Any]] = []
         self._validation_errors: list[str] = []
 
     def add_input(
         self,
-        location: str,
+        location: str | os.PathLike[str],
         type: str,
         purpose: str,
         fields: dict[str, str] | None = None,
-        relative_to: str | None = None,
+        relative_to: str | os.PathLike[str] | None = None,
     ) -> ReductionFileBuilder:
         """Add an input file to the reduction.
 
+        An entry identical to one already added (same resolved location,
+        type, purpose, and fields) is skipped, so a second reduction pass
+        in the same job can add the same files again.
+
         Arguments:
         ---------
-            location: str
+            location: str | os.PathLike[str]
                 Path to the input file.  Must be absolute, or relative if
-                relative_to is provided.
+                relative_to is provided.  An input may also be a shared
+                file of the reduction's own instrument, under
+                /<FAC>/<INST>/shared/ where <FAC> and <INST> are the first
+                two folders of the reduction file's location (for example
+                a grouping file); ONCat stores such a file by reference
+                only, listing it in the reduction without cataloging it as
+                a datafile.  validate() and write() refuse a shared file of
+                another facility or instrument whenever the reduction
+                file's location is known: the path passed to write(), or
+                the path write() derives from the first raw sample-data
+                input.
 
             type: str
                 File type.  Must be one of: "raw", "processed",
-                "user-provided", "history".
+                "user-provided", "history", "simulation".
 
             purpose: str
                 File purpose in kebab-case (e.g., "sample-data",
@@ -109,7 +172,7 @@ class ReductionFileBuilder:
                 "source.path": "dest_path" for extracting metadata.
                 Primarily used for processed files.
 
-            relative_to: str | None
+            relative_to: str | os.PathLike[str] | None
                 Base directory for relative paths.  If provided and
                 location is relative, will be resolved to absolute path.
 
@@ -142,7 +205,10 @@ class ReductionFileBuilder:
             )
 
         """
-        resolved_location = self._resolve_path(location, relative_to)
+        resolved_location = self._resolve_path(
+            _fspath(location),
+            None if relative_to is None else _fspath(relative_to),
+        )
 
         file_dict: dict[str, Any] = {
             "location": resolved_location,
@@ -154,19 +220,25 @@ class ReductionFileBuilder:
         if fields:
             file_dict["fields"] = fields
 
-        self._input_files.append(file_dict)
+        # The reduction file schema requires unique entries in each list
+        if file_dict not in self._input_files:
+            self._input_files.append(file_dict)
         return self
 
     def add_output(
         self,
-        location: str,
+        location: str | os.PathLike[str],
         type: str,
         purpose: str,
         fields: dict[str, str] | None = None,
-        relative_to: str | None = None,
+        relative_to: str | os.PathLike[str] | None = None,
     ) -> ReductionFileBuilder:
         """Add an output file to the reduction.
 
+        An entry identical to one already added (same resolved location,
+        type, purpose, and fields) is skipped, so a second reduction pass
+        in the same job can add the same files again.
+
         The 'fields' parameter is primarily used for output files to extract
         specific metadata from processed NeXus files, for example:
 
@@ -177,13 +249,13 @@ class ReductionFileBuilder:
 
         Arguments:
         ---------
-            location: str
+            location: str | os.PathLike[str]
                 Path to the output file.  Must be absolute, or relative if
                 relative_to is provided.
 
             type: str
                 File type.  Must be one of: "raw", "processed",
-                "user-provided", "history".
+                "user-provided", "history", "simulation".
 
             purpose: str
                 File purpose in kebab-case (e.g., "reduced-data",
@@ -195,7 +267,7 @@ class ReductionFileBuilder:
                 "source.path": "dest_path" for extracting metadata.
                 Commonly used for processed output files.
 
-            relative_to: str | None
+            relative_to: str | os.PathLike[str] | None
                 Base directory for relative paths.  If provided and
                 location is relative, will be resolved to absolute path.
 
@@ -221,7 +293,10 @@ class ReductionFileBuilder:
             )
 
         """
-        resolved_location = self._resolve_path(location, relative_to)
+        resolved_location = self._resolve_path(
+            _fspath(location),
+            None if relative_to is None else _fspath(relative_to),
+        )
 
         file_dict: dict[str, Any] = {
             "location": resolved_location,
@@ -233,7 +308,9 @@ class ReductionFileBuilder:
         if fields:
             file_dict["fields"] = fields
 
-        self._output_files.append(file_dict)
+        # The reduction file schema requires unique entries in each list
+        if file_dict not in self._output_files:
+            self._output_files.append(file_dict)
         return self
 
     def set_metadata(self, metadata: dict[str, Any]) -> ReductionFileBuilder:
@@ -316,6 +393,59 @@ class ReductionFileBuilder:
         """
         return self._metadata.copy()
 
+    def set_dataset_name(self, name: str) -> ReductionFileBuilder:
+        """Set the dataset name, written as the reduction file's
+        top-level "dataset": {"name": name}.
+
+        Reductions in one experiment that carry the same dataset name
+        belong to one Dataset, which the ONCat website calls a Reduction
+        Set.  Names count exactly as written, so names that differ only
+        in case or spacing make two sets.  A reduction without a name is
+        a Dataset of its own, which the website calls a Reduction.
+
+        The name is kept apart from the metadata, so set_metadata() and
+        update_metadata() leave it in place, whichever order they are
+        called in.
+
+        Arguments:
+        ---------
+            name: str
+                Dataset name.  Must be a non-empty string in which no
+                part between slashes is empty, "." or "..", since a web
+                address rewrites such a part: "a/b", ".a" and "..." are
+                names, but "..", "a/../b", "a//b", "/a" and "a/" are not.
+
+        Returns:
+        -------
+            Self for method chaining.  An invalid name is not set (any
+            name set earlier is kept); the error is stored internally
+            and surfaced by validate().
+
+        Example:
+            builder.set_dataset_name("my-dataset")
+            builder.set_metadata({"ei": 100.0})
+            builder.to_dict()["dataset"]  # {"name": "my-dataset"}
+
+        """
+        if not isinstance(name, str) or not name:
+            msg = f"Dataset name must be a non-empty string: {name!r}"
+            self._validation_errors.append(msg)
+            return self
+
+        if any(
+            segment in _REFUSED_DATASET_NAME_SEGMENTS
+            for segment in name.split("/")
+        ):
+            msg = (
+                "Dataset name must have no empty, '.' or '..' part between "
+                f"slashes: {name!r}"
+            )
+            self._validation_errors.append(msg)
+            return self
+
+        self._dataset = {"name": name}
+        return self
+
     def clear_outputs(self) -> ReductionFileBuilder:
         """Remove all output files from the builder.
 
@@ -351,11 +481,19 @@ class ReductionFileBuilder:
         - At least one output file present
         - User is set and non-empty
         - Metadata is a dict
-        - If auto-deriving path: at least one input is type="raw" and
-          purpose="sample-data"
         - All paths must be absolute and canonical (`/SNS/...` or
           `/HFIR/...` for IPTS directories, or `/SNS/...` or `/HFIR/...`
           for shared directories like /SNS/ARCS/shared/...)
+        - An input under an instrument's shared directory,
+          /<FAC>/<INST>/shared/, belongs to the facility and instrument
+          that are the first two folders of the reduction file's
+          location.  validate() knows that location only as the path
+          write() derives from the first raw sample-data input, and
+          leaves this check to ONCat when it cannot derive one.
+          write(path) checks against the path it is given.
+
+        A raw sample-data input is not required: write() needs one only
+        to derive a path when it is given none.
 
         Does NOT validate:
 
@@ -378,51 +516,7 @@ class ReductionFileBuilder:
                 builder.write()
 
         """
-        errors = list(self._validation_errors)  # Copy accumulated errors
-
-        # Check for inputs and outputs
-        if not self._input_files:
-            errors.append("At least one input file is required")
-
-        if not self._output_files:
-            errors.append("At least one output file is required")
-
-        # Check user
-        if not self._user:
-            errors.append("User must be set and non-empty")
-
-        # Check metadata is a dict
-        if not isinstance(self._metadata, dict):
-            errors.append("Metadata must be a dictionary")
-
-        # Check for raw sample-data input (needed for auto-derivation)
-        has_raw_sample = any(
-            f.get("type") == "raw" and f.get("purpose") == "sample-data"
-            for f in self._input_files
-        )
-        if not has_raw_sample:
-            errors.append(
-                "At least one input file with type='raw' and "
-                "purpose='sample-data' is required for path auto-derivation"
-            )
-
-        # Validate canonical paths
-        for file_list, file_type in [
-            (self._input_files, "input"),
-            (self._output_files, "output"),
-        ]:
-            for file_dict in file_list:
-                location = file_dict.get("location", "")
-                if not location.startswith(
-                    "/SNS/"
-                ) and not location.startswith("/HFIR/"):
-                    msg = (
-                        f"Path must be canonical (start with /SNS/ or "
-                        f"/HFIR/): {location} ({file_type} file)"
-                    )
-                    errors.append(msg)
-
-        return errors
+        return self._validate(self._derived_location())
 
     def to_dict(self) -> dict[str, Any]:
         """Preview the JSON structure that will be written (without timestamp).
@@ -431,6 +525,8 @@ class ReductionFileBuilder:
         -------
             Dictionary representing the reduction file structure.  Does not
             include the 'created' timestamp which is generated on write().
+            Includes "dataset" only after set_dataset_name() has set a
+            name, or after load() of a file whose name it accepts.
 
         Example:
             import json
@@ -438,22 +534,32 @@ class ReductionFileBuilder:
             print(json.dumps(preview, indent=2))
 
         """
-        return {
+        content = {
             "version": "1.0",
             "input_files": self._input_files,
             "output_files": self._output_files,
             "user": self._user,
             "metadata": self._metadata,
         }
+        if self._dataset is not None:
+            content["dataset"] = self._dataset
+
+        return content
 
-    def write(self, path: str | None = None, overwrite: bool = True) -> str:
+    def write(
+        self,
+        path: str | os.PathLike[str] | None = None,
+        overwrite: bool = True,
+    ) -> str:
         """Write the reduction file to disk.
 
         Behavior:
 
-        1. Validates builder state via validate()
-        2. Generates RFC3339 timestamp with timezone
-        3. Derives path if not provided (from first raw sample-data input)
+        1. Validates builder state as validate() does, checking shared
+           inputs against path, or against the derived path when path
+           is None
+        2. Derives path if not provided (from first raw sample-data input)
+        3. Generates RFC3339 timestamp with timezone
         4. Creates parent directories if needed
         5. Writes JSON file with 4-space indentation, sorted keys
         6. Returns absolute path to written file
@@ -462,12 +568,19 @@ class ReductionFileBuilder:
 
             /{facility}/{inst}/{experiment}/shared/autoreduce/{inst}_{run}.json
 
+        {facility}, {inst}, and {experiment} are the first three folders
+        of the raw sample-data input's path.  {run} is the digits after a
+        leading "{inst}_" in its file name (REF_M_32025.nxs.h5 is run
+        32025), otherwise the digits before an optional _event or _histo
+        and the NeXus extension.
+
         Arguments:
         ---------
-            path: str | None
+            path: str | os.PathLike[str] | None
                 Optional explicit path to write to.  If None, auto-derives
                 path from first input file with type="raw" and
-                purpose="sample-data".
+                purpose="sample-data", and raises ValueError when there is
+                no such input.
 
             overwrite: bool
                 Whether to overwrite existing file.  Default: True
@@ -479,7 +592,9 @@ class ReductionFileBuilder:
         Raises:
         ------
             ValueError
-                If validation fails or path cannot be auto-derived.
+                If validation fails, or if path is None and cannot be
+                auto-derived: "no raw sample-data input to derive a path
+                from; pass a path" when there is no raw sample-data input.
 
             FileExistsError
                 If file exists and overwrite=False.
@@ -499,8 +614,13 @@ class ReductionFileBuilder:
             path = builder.write(overwrite=False)  # Raises if exists
 
         """
+        if path is not None:
+            path = _fspath(path)
+
         # 1. Validate
-        errors = self.validate()
+        errors = self._validate(
+            path if path is not None else self._derived_location()
+        )
         if errors:
             error_list = "\n".join(f"  - {e}" for e in errors)
             msg = f"Validation failed:\n{error_list}"
@@ -516,14 +636,7 @@ class ReductionFileBuilder:
             raise FileExistsError(msg)
 
         # 4. Build JSON structure
-        content = {
-            "version": "1.0",
-            "input_files": self._input_files,
-            "output_files": self._output_files,
-            "created": self._generate_timestamp(),
-            "user": self._user,
-            "metadata": self._metadata,
-        }
+        content = {**self.to_dict(), "created": self._generate_timestamp()}
 
         # 5. Create directories
         os.makedirs(os.path.dirname(path), exist_ok=True)
@@ -535,17 +648,26 @@ class ReductionFileBuilder:
         return path
 
     def write_safe(
-        self, path: str | None = None, overwrite: bool = True
+        self,
+        path: str | os.PathLike[str] | None = None,
+        overwrite: bool = True,
     ) -> str | None:
-        """Write the reduction file to disk with error handling.  Never raises.
+        """Write the reduction file like write(), but never raise.
 
-        This is a convenience wrapper around write() that catches all
-        exceptions and prints them to stderr.  Use this in autoreduction
-        scripts where cataloging failures should not break the reduction.
+        write_safe() writes the same file to the same path as write(),
+        and catches every exception write() raises, so the rest of the
+        script keeps running.  On failure it prints the error and its
+        traceback to stderr and returns None.
+
+        write_safe() only writes the file; it does not catalog it.  At
+        SNS, the post-processing agent catalogs the reduction file after
+        the autoreduction job succeeds.  In an SNS autoreduction job, the
+        stderr output of a failed write marks the job as failed: the web
+        monitor shows it as failed, and the reduction is not cataloged.
 
         Arguments:
         ---------
-            path: str | None
+            path: str | os.PathLike[str] | None
                 Same as write().
 
             overwrite: bool
@@ -556,32 +678,34 @@ class ReductionFileBuilder:
             Absolute path to written file on success, None on failure.
 
         Example:
-            # In autoreduction script - safe, won't crash reduction
+            # In an autoreduction script: a failed write does not stop
+            # the script, and its stderr output marks the job as failed
             path = builder.write_safe()
             if path:
-                print(f"Reduction cataloged: {path}")
+                print(f"Reduction file written: {path}")
             else:
-                print("Warning: Cataloging failed (see stderr)")
+                print("Warning: reduction file not written (see stderr)")
 
         """
         try:
             return self.write(path, overwrite)
         except Exception as e:  # ruff: ignore[blind-except]
-            # Intentionally catch all exceptions - design requirement for
-            # autoreduction scripts where cataloging failures should never
-            # break the reduction workflow
+            # Intentionally catch every exception so a failed write never
+            # stops the rest of the autoreduction script.  The stderr
+            # output below is what marks an SNS autoreduction job as
+            # failed.
             msg = f"Warning: Failed to catalog reduction: {e}"
             print(msg, file=sys.stderr)
             traceback.print_exc()
             return None
 
     @classmethod
-    def load(cls, path: str) -> ReductionFileBuilder:
+    def load(cls, path: str | os.PathLike[str]) -> ReductionFileBuilder:
         """Load an existing reduction file for modification.
 
         Arguments:
         ---------
-            path: str
+            path: str | os.PathLike[str]
                 Path to existing JSON reduction file.
 
         Returns:
@@ -596,6 +720,10 @@ class ReductionFileBuilder:
             ValueError
                 If file is not valid JSON or missing required fields.
 
+        The file's dataset name goes through set_dataset_name()'s check.
+        A name it refuses is not loaded, validate() reports it, and
+        write() raises.
+
         Example:
             # Load and modify existing file
             builder = ReductionFileBuilder.load(
@@ -607,6 +735,7 @@ class ReductionFileBuilder:
             builder.write()  # Overwrites original file
 
         """
+        path = _fspath(path)
         if not os.path.exists(path):
             msg = f"Reduction file not found: {path}"
             raise FileNotFoundError(msg)
@@ -623,6 +752,11 @@ class ReductionFileBuilder:
 
         # Create builder
         builder = cls(user=content["user"], metadata=content["metadata"])
+        dataset = content.get("dataset")
+        if dataset is not None:
+            builder.set_dataset_name(
+                dataset.get("name") if isinstance(dataset, dict) else None
+            )
 
         # Add files (without fields if not present)
         for f in content["input_files"]:
@@ -666,12 +800,82 @@ class ReductionFileBuilder:
         self._validation_errors.append(msg)
         return location
 
+    def _validate(self, reduction_location: str | None) -> list[str]:
+        """Return validate()'s errors for a reduction file at
+        reduction_location; with None, shared inputs are not checked."""
+        errors = list(self._validation_errors)  # Copy accumulated errors
+
+        # Check for inputs and outputs
+        if not self._input_files:
+            errors.append("At least one input file is required")
+
+        if not self._output_files:
+            errors.append("At least one output file is required")
+
+        # Check user
+        if not self._user:
+            errors.append("User must be set and non-empty")
+
+        # Check metadata is a dict
+        if not isinstance(self._metadata, dict):
+            errors.append("Metadata must be a dictionary")
+
+        # Validate canonical paths
+        for file_list, file_type in [
+            (self._input_files, "input"),
+            (self._output_files, "output"),
+        ]:
+            for file_dict in file_list:
+                location = file_dict.get("location", "")
+                if not location.startswith(
+                    "/SNS/"
+                ) and not location.startswith("/HFIR/"):
+                    msg = (
+                        f"Path must be canonical (start with /SNS/ or "
+                        f"/HFIR/): {location} ({file_type} file)"
+                    )
+                    errors.append(msg)
+
+        # Check shared inputs belong to the reduction file's instrument
+        owner = (
+            _LOCATION_REGEX.match(os.fspath(reduction_location))
+            if reduction_location is not None
+            else None
+        )
+        if owner:
+            facility = owner.group("facility")
+            instrument = owner.group("instrument")
+            for file_dict in self._input_files:
+                shared = _SHARED_INPUT_REGEX.match(
+                    file_dict.get("location", "")
+                )
+                if shared and (
+                    shared.group("facility") != facility
+                    or shared.group("instrument") != instrument
+                ):
+                    msg = (
+                        f"Shared input must belong to the reduction "
+                        f"file's instrument, under "
+                        f"/{facility}/{instrument}/shared/: "
+                        f"{file_dict['location']} (input file)"
+                    )
+                    errors.append(msg)
+
+        return errors
+
+    def _derived_location(self) -> str | None:
+        """Return the path write() derives, or None when it cannot
+        derive one."""
+        try:
+            return self._derive_path()
+        except ValueError:
+            return None
+
     def _derive_path(self) -> str:
         """Auto-derive JSON file path from first raw sample-data input.
 
-        Self-contained: duplicates a minimal regex for raw NeXus paths to
-        extract facility, instrument, IPTS, and run_number. No imports from
-        other parts of this repository.
+        Raises ValueError when there is no such input, or when its path
+        names no facility, instrument, experiment, and run.
         """
         # Find first raw sample-data input
         raw_sample = None
@@ -681,25 +885,16 @@ class ReductionFileBuilder:
                 break
 
         if not raw_sample:
-            msg = (
-                "Cannot auto-derive path: no input file with type='raw' "
-                "and purpose='sample-data'. Either add one or provide "
-                "explicit path."
-            )
+            msg = "no raw sample-data input to derive a path from; pass a path"
             raise ValueError(msg)
 
-        raw_nexus_regex = re.compile(
-            r"^/(?P<facility>SNS|HFIR)/"
-            r"(?P<instrument>[A-Z0-9_]+)/"
-            r"IPTS-(?P<ipts>\d+)/"
-            r"(?:[^/]+/)*"
-            r"[^/_]*_(?P<run_number>\d+)"
-            r"(?:_event|_histo)?"
-            r"\.(?:nxs(?:\.h5)?)$"
+        match = _RAW_INPUT_REGEX.match(raw_sample)
+        run_number = (
+            self._run_number(match.group("name"), match.group("instrument"))
+            if match
+            else None
         )
-
-        match = raw_nexus_regex.search(raw_sample)
-        if not match:
+        if match is None or run_number is None:
             msg = (
                 f"Cannot parse facility/instrument/experiment from: "
                 f"{raw_sample}"
@@ -708,15 +903,25 @@ class ReductionFileBuilder:
 
         facility = match.group("facility")
         instrument = match.group("instrument")
-        ipts = match.group("ipts")
-        run_number = match.group("run_number")
-        experiment = f"IPTS-{ipts}"
+        experiment = match.group("experiment")
 
         return (
             f"/{facility}/{instrument}/{experiment}/shared/autoreduce/"
             f"{instrument}_{run_number}.json"
         )
 
+    @staticmethod
+    def _run_number(name: str, instrument: str) -> str | None:
+        """Return the run in a raw file's name, read as ONCat reads a
+        reduction file's name: the digits after a leading
+        "<instrument>_" (REF_M_32025_event.nxs is run 32025), otherwise
+        the digits before an optional _event or _histo and the NeXus
+        extension; None when neither is there."""
+        run = re.match(
+            rf"{re.escape(instrument)}_(\d+)", name
+        ) or _TRAILING_RUN_NUMBER_REGEX.search(name)
+        return run.group(1) if run else None
+
     @staticmethod
     def _generate_timestamp() -> str:
         """Generate RFC3339 timestamp with timezone."""

{pyoncat-2.7 → pyoncat-2.9}/pyproject.toml

@@ -1,13 +1,13 @@
 [build-system]
-requires = ["poetry"]
-build-backend = "poetry.masonry.api"
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
 
 # Poetry requires it's own condfiguration because it predates the
 # "[project]" standard.  See:
 # https://github.com/python-poetry/roadmap/issues/3
 [tool.poetry]
 name = "pyoncat"
-version = "2.7"
+version = "2.9"
 description = "A Python Client for ONCat (the ORNL Neutron Catalog)."
 authors = ["ONCat Support <oncat-support@ornl.gov>"]
 readme = ["pyoncat/README.md", "pyoncat/LICENSE"]