opentrons 8.6.0a1__py3-none-any.whl

opentrons/protocol_reader/file_identifier.py

@@ -0,0 +1,284 @@
+"""File identifier interface."""
+
+import json
+from dataclasses import dataclass
+from typing import Any, Dict, Sequence, Union, Optional
+
+import anyio
+
+from opentrons_shared_data.robot.types import RobotType
+from opentrons_shared_data.errors.exceptions import EnumeratedError, PythonException
+
+from opentrons.protocols.api_support.definitions import MAX_SUPPORTED_VERSION
+from opentrons.protocols.api_support.types import APIVersion
+from opentrons.protocols import parse
+from opentrons.protocols.types import MalformedPythonProtocolError, PythonProtocol
+
+from .file_reader_writer import BufferedFile
+from .protocol_files_invalid_error import ProtocolFilesInvalidError
+from .protocol_source import Metadata
+
+
+JsonDict = Dict[str, Any]
+
+
+@dataclass(frozen=True)
+class IdentifiedJsonMain:
+    """A file identified as a JSON protocol's main .json file."""
+
+    original_file: BufferedFile
+    """The original file that this was identified from."""
+
+    unvalidated_json: JsonDict
+    """The parsed JSON contents.
+
+    Believed, but not confirmed at this point, to conform to one of our JSON protocol
+    schemas.
+    """
+
+    schema_version: int
+    """The JSON protocol schema that this file is believed to conform to."""
+
+    robot_type: RobotType
+    """The type of robot on which this protocol is meant to run."""
+
+    metadata: Metadata
+    """The protocol metadata extracted from this file."""
+
+
+@dataclass(frozen=True)
+class IdentifiedPythonMain:
+    """A file identified as a Python protocol's main .py file."""
+
+    original_file: BufferedFile
+    """The original file that this was identified from."""
+
+    api_level: APIVersion
+    """The Python Protocol API apiLevel declared by the Python source."""
+
+    robot_type: RobotType
+    """The type of robot on which this protocol is meant to run."""
+
+    metadata: Metadata
+    """The protocol metadata extracted from this file."""
+
+
+@dataclass(frozen=True)
+class IdentifiedLabwareDefinition:
+    """A file identified as a labware definition."""
+
+    original_file: BufferedFile
+    """The original file that this was identified from."""
+
+    unvalidated_json: JsonDict
+    """The parsed JSON contents.
+
+    Believed, but not confirmed at this point, to conform to our labware definition
+    schema v2.
+    """
+
+
+@dataclass(frozen=True)
+class IdentifiedData:
+    """A file identified as a user-defined data file."""
+
+    original_file: BufferedFile
+    """The original file that this was identified from."""
+
+
+IdentifiedFile = Union[
+    IdentifiedJsonMain,
+    IdentifiedPythonMain,
+    IdentifiedLabwareDefinition,
+    IdentifiedData,
+]
+
+
+class FileIdentificationError(ProtocolFilesInvalidError):
+    """Raised when FileIdentifier detects an invalid file."""
+
+    def __init__(
+        self,
+        message: str,
+        detail: Optional[Dict[str, Any]] = None,
+        wrapping: Optional[Sequence[EnumeratedError]] = None,
+        only_message: bool = False,
+    ) -> None:
+        super().__init__(message=message)
+        self._only_message = only_message
+
+    def __str__(self) -> str:
+        """Special stringifier to conform to expecations about python protocol errors."""
+        if self._only_message:
+            return self.message
+        else:
+            return super().__str__()
+
+
+class FileIdentifier:
+    """File identifier interface."""
+
+    @staticmethod
+    async def identify(
+        files: Sequence[BufferedFile], python_parse_mode: parse.PythonParseMode
+    ) -> Sequence[IdentifiedFile]:
+        """Identify the type and extract basic information from each file.
+
+        This is intended to take ≲1 second per protocol on an OT-2, so it can extract
+        basic information about all stored protocols relatively quickly. Fully parsing
+        and validating protocols can take 10-100x longer, so that's left to other units,
+        for only when it's really needed.
+        """
+        return [await _identify(file, python_parse_mode) for file in files]
+
+
+async def _identify(
+    file: BufferedFile, python_parse_mode: parse.PythonParseMode
+) -> IdentifiedFile:
+    lower_file_name = file.name.lower()
+    if lower_file_name.endswith(".json"):
+        return await _analyze_json(json_file=file)
+    elif lower_file_name.endswith(".py"):
+        return _analyze_python_protocol(
+            py_file=file, python_parse_mode=python_parse_mode
+        )
+    elif lower_file_name.endswith(".csv") or lower_file_name.endswith(".txt"):
+        return IdentifiedData(original_file=file)
+    else:
+        raise FileIdentificationError(
+            message=f"{file.name} has an unrecognized file extension.",
+            detail={"type": "bad-file-extension", "file": file.name},
+        )
+
+
+async def _analyze_json(
+    json_file: BufferedFile,
+) -> Union[IdentifiedJsonMain, IdentifiedLabwareDefinition]:
+    try:
+        json_contents = await anyio.to_thread.run_sync(json.loads, json_file.contents)
+    except json.JSONDecodeError as e:
+        raise FileIdentificationError(
+            message=f"{json_file.name} is not valid JSON. {str(e)}",
+            detail={"type": "invalid-json", "file": json_file.name},
+            wrapping=[PythonException(e)],
+        ) from e
+
+    if _json_seems_like_labware(json_contents):
+        return IdentifiedLabwareDefinition(
+            original_file=json_file,
+            unvalidated_json=json_contents,
+        )
+    elif _json_seems_like_protocol(json_contents):
+        return _analyze_json_protocol(
+            original_file=json_file,
+            json_contents=json_contents,
+        )
+    else:
+        raise FileIdentificationError(
+            message=f"{json_file.name} is not a known Opentrons format.",
+            detail={"type": "no-schema-match", "file": json_file.name},
+        )
+
+
+def _json_seems_like_labware(json: JsonDict) -> bool:
+    # "ordering" and "wells" are required properties in our labware schema v2.
+    return "ordering" in json and "wells" in json
+
+
+def _json_seems_like_protocol(json: JsonDict) -> bool:
+    # "schemaVersion" and "commands" are required properties in all of our JSON
+    # protocol schemas since schema v3. (v7 is the latest at the time of writing.)
+    #
+    # When we stop supporting v3 files, we can look at "$otSharedSchema" instead,
+    # which is more precise.
+    return "schemaVersion" in json and "commands" in json
+
+
+def _analyze_json_protocol(
+    original_file: BufferedFile, json_contents: JsonDict
+) -> IdentifiedJsonMain:
+    error_message = f"{original_file.name} is not a valid JSON protocol."
+
+    try:
+        metadata = json_contents["metadata"]
+        schema_version = json_contents["schemaVersion"]
+        robot_type = json_contents["robot"]["model"]
+    except KeyError as e:
+        raise FileIdentificationError(
+            message=error_message,
+            detail={"kind": "missing-json-metadata", "missing-key": str(e)},
+            wrapping=[PythonException(e)],
+        ) from e
+
+    # todo(mm, 2022-12-22): A JSON protocol file's metadata is not quite just an
+    # arbitrary dict: its fields are supposed to follow a schema. Should we validate
+    # this metadata against that schema instead of doing this simple isinstance() check?
+    if not isinstance(metadata, dict):
+        raise FileIdentificationError(
+            message=error_message, detail={"kind": "json-metadata-not-object"}
+        )
+
+    if not isinstance(schema_version, int):
+        raise FileIdentificationError(
+            message=error_message,
+            detail={
+                "kind": "json-schema-version-not-int",
+                "schema-version": schema_version,
+            },
+        )
+
+    if robot_type not in ("OT-2 Standard", "OT-3 Standard"):
+        raise FileIdentificationError(
+            message=error_message,
+            detail={"kind": "bad-json-protocol-robot-type", "robot-type": robot_type},
+        )
+
+    return IdentifiedJsonMain(
+        original_file=original_file,
+        unvalidated_json=json_contents,
+        schema_version=schema_version,
+        robot_type=robot_type,
+        metadata=metadata,
+    )
+
+
+def _analyze_python_protocol(
+    py_file: BufferedFile,
+    python_parse_mode: parse.PythonParseMode,
+) -> IdentifiedPythonMain:
+    try:
+        parsed = parse.parse(
+            protocol_file=py_file.contents,
+            filename=py_file.name,
+            python_parse_mode=python_parse_mode,
+        )
+    except MalformedPythonProtocolError as e:
+        raise FileIdentificationError(
+            message=e.short_message,
+            detail={"kind": "malformed-python-protocol"},
+            wrapping=[PythonException(e)],
+            only_message=True,
+        ) from e
+
+    # We know this should never be a JsonProtocol. Help out the type-checker.
+    assert isinstance(
+        parsed, PythonProtocol
+    ), "Parsing a Python file returned something other than a Python protocol."
+
+    if parsed.api_level > MAX_SUPPORTED_VERSION:
+        raise FileIdentificationError(
+            message=(
+                f"API version {parsed.api_level} is not supported by this "
+                f"robot software. Please either reduce your requested API "
+                f"version or update your robot."
+            ),
+            detail={"kind": "future-api-version", "api-version": str(parsed.api_level)},
+            only_message=True,
+        )
+
+    return IdentifiedPythonMain(
+        original_file=py_file,
+        metadata=parsed.metadata or {},
+        robot_type=parsed.robot_type,
+        api_level=parsed.api_level,
+    )

opentrons/protocol_reader/file_reader_writer.py

@@ -0,0 +1,90 @@
+"""Input file reading."""
+
+import os
+import pathlib
+import platform
+from dataclasses import dataclass
+from typing import List, Optional, Sequence, Union
+
+import anyio
+
+from .input_file import AbstractInputFile
+from .protocol_files_invalid_error import ProtocolFilesInvalidError
+
+
+@dataclass(frozen=True)
+class BufferedFile:
+    """A file that has been read into memory."""
+
+    name: str
+    contents: bytes
+    path: Optional[pathlib.Path]
+
+
+class FileReadError(ProtocolFilesInvalidError):
+    """An error raised if input files cannot be read."""
+
+
+class FileReaderWriter:
+    """Input file reader/writer interface."""
+
+    @staticmethod
+    async def read(
+        files: Sequence[Union[AbstractInputFile, pathlib.Path]]
+    ) -> List[BufferedFile]:
+        """Read a set of input files into memory."""
+        return [await _read_file(input_file=file) for file in files]
+
+    @staticmethod
+    async def write(directory: pathlib.Path, files: Sequence[BufferedFile]) -> None:
+        """Write a set of previously buffered files to disk."""
+        await anyio.Path(directory).mkdir(parents=True, exist_ok=True)
+
+        for file in files:
+            path = directory / file.name
+            await _write_and_fsync_file(path=path, contents=file.contents)
+
+        await _fsync_directory(path=directory)
+
+
+async def _read_file(
+    input_file: Union[AbstractInputFile, pathlib.Path]
+) -> BufferedFile:
+    if isinstance(input_file, pathlib.Path):
+        path: Optional[pathlib.Path] = input_file
+        filename = input_file.name
+        contents = await anyio.Path(input_file).read_bytes()
+    elif not input_file.filename:
+        raise FileReadError("File was missing a name")
+    else:
+        path = None
+        filename = input_file.filename
+        async with anyio.wrap_file(input_file.file) as f:
+            contents = await f.read()
+
+    return BufferedFile(
+        name=filename,
+        contents=contents,
+        path=path,
+    )
+
+
+async def _write_and_fsync_file(path: pathlib.Path, contents: bytes) -> None:
+    async with await anyio.open_file(path, "wb") as file:
+        await file.write(contents)
+        await file.flush()
+        await anyio.to_thread.run_sync(os.fsync, file.wrapped.fileno())
+
+
+async def _fsync_directory(path: pathlib.Path) -> None:
+    def _fsync_directory_sync() -> None:
+        # We can't fsync directories on Windows because doing os.open() on them raises
+        # PermissionError. https://stackoverflow.com/questions/21785127
+        if platform.system() != "Windows":
+            fd = os.open(path, os.O_RDONLY)
+            try:
+                os.fsync(fd)
+            finally:
+                os.close(fd)
+
+    await anyio.to_thread.run_sync(_fsync_directory_sync)

opentrons/protocol_reader/input_file.py

@@ -0,0 +1,16 @@
+"""Input file value objects."""
+from __future__ import annotations
+from typing import BinaryIO, Protocol
+
+
+class AbstractInputFile(Protocol):
+    """An individual file to be read as part of a protocol.
+
+    Properties:
+        filename: The basename, including extension, of the file.
+        file: A [file](https://docs.python.org/3/glossary.html#term-file-object)
+            providing the contents of the protocol to be read.
+    """
+
+    filename: str
+    file: BinaryIO

opentrons/protocol_reader/protocol_files_invalid_error.py

@@ -0,0 +1,6 @@
+# noqa: D100
+from opentrons_shared_data.errors.exceptions import InvalidProtocolData
+
+
+class ProtocolFilesInvalidError(InvalidProtocolData):
+    """Raised when the input to a ProtocolReader is not a well-formed protocol."""

opentrons/protocol_reader/protocol_reader.py

@@ -0,0 +1,188 @@
+"""Read relevant protocol information from a set of files."""
+from pathlib import Path
+from typing import Optional, Sequence
+
+from opentrons.protocols.parse import PythonParseMode
+
+from .file_identifier import (
+    FileIdentifier,
+    IdentifiedFile,
+    IdentifiedJsonMain,
+    IdentifiedPythonMain,
+    IdentifiedLabwareDefinition,
+    IdentifiedData,
+)
+from .role_analyzer import RoleAnalyzer, RoleAnalysis
+from .file_format_validator import FileFormatValidator
+from .file_reader_writer import FileReaderWriter, BufferedFile
+from .file_hasher import FileHasher
+from .protocol_source import (
+    ProtocolSource,
+    ProtocolSourceFile,
+    ProtocolConfig,
+    ProtocolFileRole,
+    JsonProtocolConfig,
+    PythonProtocolConfig,
+)
+
+
+class ProtocolReader:
+    """Collaborator to turn a set of files into a protocol object."""
+
+    def __init__(
+        self,
+        file_reader_writer: Optional[FileReaderWriter] = None,
+        file_identifier: Optional[FileIdentifier] = None,
+        role_analyzer: Optional[RoleAnalyzer] = None,
+        file_format_validator: Optional[FileFormatValidator] = None,
+        file_hasher: Optional[FileHasher] = None,
+    ) -> None:
+        """Initialize the reader with its dependencies.
+
+        Arguments:
+            file_reader_writer: Input file reader/writer. Default impl. used if None.
+            file_identifier: File identifier. Default impl. used if None.
+            role_analyzer: File role analyzer. Default impl. used if None.
+            file_format_validator: File format validator. Default impl. used if None.
+            file_hasher: File hasher. Default impl. used if None.
+        """
+        self._file_reader_writer = file_reader_writer or FileReaderWriter()
+        self._file_identifier = file_identifier or FileIdentifier()
+        self._role_analyzer = role_analyzer or RoleAnalyzer()
+        self._file_format_validator = file_format_validator or FileFormatValidator()
+        self._file_hasher = file_hasher or FileHasher()
+
+    async def save(
+        self,
+        files: Sequence[BufferedFile],
+        directory: Path,
+        content_hash: str,
+    ) -> ProtocolSource:
+        """Compute a `ProtocolSource` from buffered files and save them as files.
+
+        The input is parsed and statically analyzed to ensure it's basically
+        well-formed. For example, labware definition files must conform to the labware
+        definition schema.
+
+        Arguments:
+            files: List buffered files. Do not attempt to reuse any objects
+                in this list once they've been passed to the ProtocolReader.
+            directory: Name of the directory to create and place files in.
+
+        Returns:
+            A ProtocolSource describing the validated protocol.
+
+        Raises:
+            ProtocolFilesInvalidError: Input file list given to the reader
+                could not be validated as a protocol.
+        """
+        identified_files = await self._file_identifier.identify(
+            files, python_parse_mode=PythonParseMode.NORMAL
+        )
+        role_analysis = self._role_analyzer.analyze(identified_files)
+        await self._file_format_validator.validate(role_analysis.all_files)
+
+        files_to_write = [f.original_file for f in role_analysis.all_files]
+        await self._file_reader_writer.write(directory=directory, files=files_to_write)
+
+        main_file = directory / role_analysis.main_file.original_file.name
+        output_files = [
+            ProtocolSourceFile(
+                path=directory / f.original_file.name, role=self._map_file_role(f)
+            )
+            for f in role_analysis.all_files
+        ]
+
+        return ProtocolSource(
+            directory=directory,
+            main_file=main_file,
+            content_hash=content_hash,
+            files=output_files,
+            config=self._map_config(role_analysis),
+            robot_type=role_analysis.main_file.robot_type,
+            metadata=role_analysis.main_file.metadata,
+        )
+
+    async def read_saved(
+        self,
+        files: Sequence[Path],
+        directory: Optional[Path],
+        files_are_prevalidated: bool = False,
+        python_parse_mode: PythonParseMode = PythonParseMode.NORMAL,
+    ) -> ProtocolSource:
+        """Compute a `ProtocolSource` from protocol source files on the filesystem.
+
+        The input is parsed and statically analyzed to ensure it's basically
+        well-formed. For example, labware definition files must conform to the labware
+        definition schema.
+
+        Arguments:
+            files: The files comprising the protocol.
+            directory: Passed through to `ProtocolSource.directory`. Otherwise unused.
+            files_are_prevalidated: Assume that the input files are valid. Skip full
+                parsing and validation, doing only the minimum required to extract the
+                stuff for the `ProtocolSource`. This can be 10-100x faster, but you
+                should only do it with protocols that have already been validated
+                by this module.
+            python_parse_mode: See the documentation in `PythonParseMode`.
+
+        Returns:
+            A `ProtocolSource` describing the validated protocol.
+
+        Raises:
+            ProtocolFilesInvalidError: Input file list given to the reader
+                could not be validated as a protocol.
+        """
+        buffered_files = await self._file_reader_writer.read(files)
+        identified_files = await self._file_identifier.identify(
+            files=buffered_files,
+            python_parse_mode=python_parse_mode,
+        )
+        role_analysis = self._role_analyzer.analyze(identified_files)
+        if not files_are_prevalidated:
+            await self._file_format_validator.validate(role_analysis.all_files)
+
+        # We know these paths will not be None because we supplied real Paths,
+        # not AbstractInputFiles, to FileReaderWriter.
+        # TODO(mc, 2022-04-01): these asserts are a bit awkward,
+        # consider restructuring so they're not needed.
+        assert isinstance(role_analysis.main_file.original_file.path, Path)
+        assert all(
+            isinstance(f.original_file.path, Path) for f in role_analysis.all_files
+        )
+
+        main_file = role_analysis.main_file.original_file.path
+        content_hash = await self._file_hasher.hash(buffered_files)
+
+        output_files = [
+            ProtocolSourceFile(path=f.original_file.path, role=self._map_file_role(f))  # type: ignore[arg-type]
+            for f in role_analysis.all_files
+        ]
+
+        return ProtocolSource(
+            directory=directory,
+            main_file=main_file,
+            content_hash=content_hash,
+            files=output_files,
+            config=self._map_config(role_analysis),
+            robot_type=role_analysis.main_file.robot_type,
+            metadata=role_analysis.main_file.metadata,
+        )
+
+    @staticmethod
+    def _map_file_role(file: IdentifiedFile) -> ProtocolFileRole:
+        if isinstance(file, (IdentifiedJsonMain, IdentifiedPythonMain)):
+            return ProtocolFileRole.MAIN
+        elif isinstance(file, IdentifiedLabwareDefinition):
+            return ProtocolFileRole.LABWARE
+        elif isinstance(file, IdentifiedData):
+            return ProtocolFileRole.DATA
+
+    @staticmethod
+    def _map_config(protocol_info: RoleAnalysis) -> ProtocolConfig:
+        if isinstance(protocol_info.main_file, IdentifiedJsonMain):
+            return JsonProtocolConfig(
+                schema_version=protocol_info.main_file.schema_version
+            )
+        elif isinstance(protocol_info.main_file, IdentifiedPythonMain):
+            return PythonProtocolConfig(api_version=protocol_info.main_file.api_level)