sift-stack-py 0.3.2__py3-none-any.whl

sift_py/data_import/status.py

@@ -0,0 +1,113 @@
+import time
+from datetime import datetime
+from enum import Enum
+from typing import Optional, Union
+from urllib.parse import urljoin
+
+import requests
+from pydantic import BaseModel, ConfigDict, field_validator
+from pydantic.alias_generators import to_camel
+from pydantic_core import PydanticCustomError
+from typing_extensions import Self
+
+from sift_py.rest import SiftRestConfig, compute_uri
+
+
+class DataImportStatusType(Enum):
+    """Status of the data import."""
+
+    SUCCEEDED = "DATA_IMPORT_STATUS_SUCCEEDED"
+    PENDING = "DATA_IMPORT_STATUS_PENDING"
+    IN_PROGRESS = "DATA_IMPORT_STATUS_IN_PROGRESS"
+    FAILED = "DATA_IMPORT_STATUS_FAILED"
+
+    @classmethod
+    def from_str(cls, val: str) -> Optional[Self]:
+        try:
+            return cls(val)
+        except ValueError:
+            return None
+
+    def as_human_str(self) -> str:
+        return self.value
+
+
+class DataImport(BaseModel):
+    """Metadata regarding the data import."""
+
+    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+
+    data_import_id: str
+    created_date: datetime
+    modified_date: datetime
+    source_url: str = ""
+    status: Union[str, DataImportStatusType]
+    error_message: str = ""
+    csv_config: dict
+
+    @field_validator("status", mode="before")
+    @classmethod
+    def convert_status(cls, raw: Union[str, DataImportStatusType]) -> DataImportStatusType:
+        if isinstance(raw, DataImportStatusType):
+            return raw
+        elif isinstance(raw, str):
+            value = DataImportStatusType.from_str(raw)
+            if value is not None:
+                return value
+
+        raise PydanticCustomError(
+            "invalid_data_import_error", f"Invalid data import status: {raw}."
+        )
+
+
+class DataImportService:
+    """
+    Service used to retrive information about a particular data import.
+    """
+
+    STATUS_PATH = "/api/v1/data-imports"
+    _data_import_id: str
+
+    def __init__(self, restconf: SiftRestConfig, data_import_id: str):
+        base_uri = compute_uri(restconf)
+        self._data_import_id = data_import_id
+        self._status_uri = urljoin(base_uri, self.STATUS_PATH)
+        self._apikey = restconf["apikey"]
+
+    def get_data_import(self) -> DataImport:
+        """
+        Returns information about the data import.
+        """
+        response = requests.get(
+            url=f"{self._status_uri}/{self._data_import_id}",
+            headers={"Authorization": f"Bearer {self._apikey}"},
+        )
+        response.raise_for_status()
+
+        data = response.json().get("dataImport")
+        data_import = DataImport(**data)
+        return data_import
+
+    def wait_until_complete(self) -> DataImport:
+        """
+        Blocks until the data import is completed. Check the status to determine
+        if the import was successful or not.
+        """
+        polling_interval = 1
+        while True:
+            data_import = self.get_data_import()
+            status: DataImportStatusType = data_import.status  # type: ignore
+            if status in [
+                DataImportStatusType.SUCCEEDED,
+                DataImportStatusType.FAILED,
+            ]:
+                return data_import
+            elif status in [
+                DataImportStatusType.PENDING,
+                DataImportStatusType.IN_PROGRESS,
+            ]:
+                pass
+            else:
+                raise Exception(f"Unknown status: {status}")
+            time.sleep(polling_interval)
+            polling_interval = min(polling_interval * 2, 60)

sift_py/data_import/tdms.py

@@ -0,0 +1,206 @@
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
+try:
+    from nptdms import (  # type: ignore
+        ChannelObject,
+        RootObject,
+        TdmsChannel,
+        TdmsFile,
+        TdmsWriter,
+        types,
+    )
+except ImportError as e:
+    raise RuntimeError(
+        "The npTDMS package is required to use the TDMS upload service. "
+        "Please include this dependency in your project by specifying `sift-stack-py[tdms]`."
+    ) from e
+
+from sift_py.data_import._config import DataColumn, TimeColumn
+from sift_py.data_import.config import CsvConfig
+from sift_py.data_import.csv import CsvUploadService
+from sift_py.data_import.status import DataImportService
+from sift_py.data_import.tempfile import NamedTemporaryFile
+from sift_py.data_import.time_format import TimeFormatType
+from sift_py.ingestion.channel import ChannelDataType
+from sift_py.rest import SiftRestConfig
+
+TDMS_TO_SIFT_TYPES = {
+    types.Boolean: ChannelDataType.BOOL,
+    types.Int8: ChannelDataType.INT_32,
+    types.Int16: ChannelDataType.INT_32,
+    types.Int32: ChannelDataType.INT_32,
+    types.Int64: ChannelDataType.INT_64,
+    types.Uint8: ChannelDataType.UINT_32,
+    types.Uint16: ChannelDataType.UINT_32,
+    types.Uint32: ChannelDataType.UINT_32,
+    types.Uint64: ChannelDataType.UINT_64,
+    types.SingleFloat: ChannelDataType.FLOAT,
+    types.DoubleFloat: ChannelDataType.DOUBLE,
+}
+
+
+class TdmsUploadService:
+    """
+    Service to upload TDMS files.
+    """
+
+    _csv_upload_service: CsvUploadService
+
+    def __init__(self, rest_conf: SiftRestConfig):
+        self._csv_upload_service = CsvUploadService(rest_conf)
+
+    def upload(
+        self,
+        path: Union[str, Path],
+        asset_name: str,
+        group_into_components: bool = False,
+        ignore_errors: bool = False,
+        run_name: Optional[str] = None,
+        run_id: Optional[str] = None,
+    ) -> DataImportService:
+        """
+        Uploads the TDMS file pointed to by `path` to the specified asset.
+
+        Set `group_into_components` to True if you want to upload the TDMS groups as
+        a Sift Component.
+
+        If `ignore_errors` is True will skip channels without timing information.
+
+        Override `run_name` to specify the name of the run to create for this data. Default is None.
+        Override `run_id` to specify the id of the run to add this data to. Default is None.
+        """
+        posix_path = Path(path) if isinstance(path, str) else path
+
+        if not posix_path.is_file():
+            raise Exception(f"Provided path, '{path}', does not point to a regular file.")
+
+        with NamedTemporaryFile(mode="w", suffix=".csv") as temp_file:
+            valid_channels = self._convert_to_csv(path, temp_file.name, ignore_errors)
+            if not valid_channels:
+                raise Exception(f"No valid channels remaining in {path}")
+
+            csv_config = self._create_csv_config(
+                valid_channels, asset_name, group_into_components, run_name, run_id
+            )
+            return self._csv_upload_service.upload(temp_file.name, csv_config)
+
+    def _convert_to_csv(
+        self, src_path: Union[str, Path], dst_path: Union[str, Path], ignore_errors: bool
+    ) -> List[TdmsChannel]:
+        """Converts the TDMS file to a temporary CSV on disk that we will upload.
+
+        Returns the valid channels after parsing the TDMS file. Valid channels contain
+        timing information.
+        """
+
+        def contains_timing(channel: TdmsChannel) -> bool:
+            """Returns true if the TDMS Channel contains timing information."""
+            return all(
+                [
+                    "wf_increment" in channel.properties,
+                    "wf_start_time" in channel.properties,
+                    "wf_start_offset" in channel.properties,
+                ]
+            )
+
+        def normalize_name(channel_name: str) -> str:
+            """Normalize channel names by invalid characters."""
+            return " ".join(channel_name.replace("/", " ").split())
+
+        src_file = TdmsFile(src_path)
+
+        original_groups = src_file.groups()
+        valid_channels: List[ChannelObject] = []
+        for group in original_groups:
+            for channel in group.channels():
+                if contains_timing(channel):
+                    new_channel = ChannelObject(
+                        group=normalize_name(channel.group_name),
+                        channel=normalize_name(channel.name),
+                        data=channel.data,
+                        properties=channel.properties,
+                    )
+                    valid_channels.append(new_channel)
+                else:
+                    if ignore_errors:
+                        print(
+                            f"{group.name}:{channel.name} does not contain timing information. Skipping."
+                        )
+                    else:
+                        raise Exception(
+                            f"{group.name}:{channel.name} does not contain timing information. "
+                            "Set `ignore_errors` to True to skip channels without timing information."
+                        )
+
+        # Write out the new TDMS file with invalid channels removed, then convert to csv.
+        with NamedTemporaryFile(mode="w") as f:
+            with TdmsWriter(f.name) as tdms_writer:
+                root_object = RootObject(src_file.properties)
+                tdms_writer.write_segment([root_object] + original_groups + valid_channels)
+
+            filtered_tdms_file = TdmsFile(f.name)
+            df = filtered_tdms_file.as_dataframe(time_index=True, absolute_time=True)
+            df.to_csv(dst_path, encoding="utf-8")
+
+        return [channel for group in filtered_tdms_file.groups() for channel in group.channels()]
+
+    def _create_csv_config(
+        self,
+        channels: List[TdmsChannel],
+        asset_name: str,
+        group_into_components: bool,
+        run_name: Optional[str] = None,
+        run_id: Optional[str] = None,
+    ) -> CsvConfig:
+        """Construct a CsvConfig based on metadata within the TDMS file."""
+        data_config: Dict[int, DataColumn] = {}
+        # Data columns start in column 2 (1-indexed)
+        first_data_column = 2
+        for i, channel in enumerate(channels):
+            try:
+                data_type = TDMS_TO_SIFT_TYPES[channel.data_type].as_human_str(api_format=True)
+            except KeyError:
+                data_type = None
+
+            if data_type is None:
+                raise Exception(f"{channel.name} data type not supported: {channel.data_type}")
+
+            extra_info = ""
+            for k, v in channel.properties.items():
+                # Skip these since the csv config has dedicated fields for them.
+                if k in ["description", "unit_string"]:
+                    continue
+                # Must convert datetime to a string
+                elif k == "wf_start_time":
+                    v = str(v)
+                extra_info += f"{k}: {v}\n"
+
+            channel_config = DataColumn(
+                name=channel.name,
+                data_type=data_type,
+                description=f"{channel.properties.get('description')}\n{extra_info}",
+                units=channel.properties.get("unit_string") or "",
+            )
+            if group_into_components and channel.group_name:
+                channel_config.component = channel.group_name
+
+            data_config[first_data_column + i] = channel_config
+
+        config_info = {
+            "asset_name": asset_name,
+            "first_data_row": first_data_column,
+            "time_column": TimeColumn(
+                format=TimeFormatType.ABSOLUTE_DATETIME,
+                column_number=1,
+            ),
+            "data_columns": data_config,
+        }
+
+        if run_name is not None:
+            config_info["run_name"] = run_name
+
+        if run_id is not None:
+            config_info["run_id"] = run_id
+
+        return CsvConfig(config_info)

sift_py/data_import/tempfile.py

@@ -0,0 +1,30 @@
+import os
+import tempfile
+from pathlib import Path
+
+
+class NamedTemporaryFile:
+    """
+    Created a named temporary file.
+
+    Works on both Windows and Unix systems.
+
+    See https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file
+    for more information on Windows compatibility.
+    """
+
+    def __init__(self, mode, suffix=""):
+        self.temp_dir = tempfile.mkdtemp()
+        self.name = Path(self.temp_dir) / f"tempfile{suffix}"
+        self.file = open(self.name, mode)
+
+    def __enter__(self):
+        return self.file
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.file.close()
+        try:
+            os.remove(self.name)
+            os.rmdir(self.temp_dir)
+        except FileNotFoundError:
+            pass

sift_py/data_import/time_format.py

@@ -0,0 +1,39 @@
+from enum import Enum
+from typing import Optional
+
+from typing_extensions import Self
+
+
+class TimeFormatType(Enum):
+    ABSOLUTE_RFC3339 = "TIME_FORMAT_ABSOLUTE_RFC3339"
+    ABSOLUTE_DATETIME = "TIME_FORMAT_ABSOLUTE_DATETIME"
+    ABSOLUTE_UNIX_SECONDS = "TIME_FORMAT_ABSOLUTE_UNIX_SECONDS"
+    ABSOLUTE_UNIX_MILLISECONDS = "TIME_FORMAT_ABSOLUTE_UNIX_MILLISECONDS"
+    ABSOLUTE_UNIX_MICROSECONDS = "TIME_FORMAT_ABSOLUTE_UNIX_MICROSECONDS"
+    ABSOLUTE_UNIX_NANOSECONDS = "TIME_FORMAT_ABSOLUTE_UNIX_NANOSECONDS"
+    RELATIVE_NANOSECONDS = "TIME_FORMAT_RELATIVE_NANOSECONDS"
+    RELATIVE_MICROSECONDS = "TIME_FORMAT_RELATIVE_MICROSECONDS"
+    RELATIVE_MILLISECONDS = "TIME_FORMAT_RELATIVE_MILLISECONDS"
+    RELATIVE_SECONDS = "TIME_FORMAT_RELATIVE_SECONDS"
+    RELATIVE_MINUTES = "TIME_FORMAT_RELATIVE_MINUTES"
+    RELATIVE_HOURS = "TIME_FORMAT_RELATIVE_HOURS"
+
+    @classmethod
+    def from_str(cls, val: str) -> Optional[Self]:
+        try:
+            return cls(val)
+        except ValueError:
+            return None
+
+    def as_human_str(self) -> str:
+        return self.value
+
+    def is_relative(self) -> bool:
+        return self in [
+            self.RELATIVE_NANOSECONDS,
+            self.RELATIVE_MICROSECONDS,
+            self.RELATIVE_MILLISECONDS,
+            self.RELATIVE_SECONDS,
+            self.RELATIVE_MINUTES,
+            self.RELATIVE_HOURS,
+        ]

sift_py/error.py

@@ -0,0 +1,11 @@
+class SiftError(Exception):
+    """
+    These exceptions are raised when something totally unexpected occurs and is
+    meant to indicate that the error is likely not caused by the user, but rather,
+    the library itself. These errors should be reported to Sift.
+    """
+
+    msg: str
+
+    def __init__(self, msg: str):
+        super().__init__(f"{msg}\nPlease notify Sift.")

sift_py/file_attachment/__init__.py

@@ -0,0 +1,88 @@
+"""
+This module contains services to facilitate uploading and downloading file attachments.
+It also provides utilities to easily query all file attachments for a given entity
+which could be a run, annotation, or annotation logs. File attachment deletion is also supported.
+
+Once files have been attached, they should be viewable on the Sift application, attached to their
+respective entities. Below are various examples on how to leverage the `sift_py.file_attachment.service.FileAttachmentService`.
+
+## Initializing the file attachment service
+
+Unlike other services throughout `sift_py`, the `sift_py.file_attachment.service.FileAttachmentService` does rely on both
+REST and gRPC APIs, so with that in mind we can initialize our service like so:
+
+```python
+from sift_py.grpc.transport import SiftChannelConfig, use_sift_channel
+from sift_py.file_attachment.service import FileAttachmentService
+from sift_py.file_attachment.entity import Entity, EntityType
+from sift_py.file_attachment.metadata import ImageMetadata
+from sift_py.rest import SiftRestConfig
+
+from sift.remote_files.v1.remote_files_pb2 import GetRemoteFileRequest
+from sift.remote_files.v1.remote_files_pb2_grpc import RemoteFileServiceStub
+
+rest_config: SiftRestConfig = {
+    # Be sure to exclude the "https://" or "http://" scheme out of the uri
+    "uri": rest_base_uri,
+    "apikey": apikey,
+}
+
+sift_channel_config = SiftChannelConfig(uri=grpc_base_uri, apikey=apikey)
+
+with use_sift_channel(sift_channel_config) as channel:
+    file_attachment_service = FileAttachmentService(channel, rest_config)
+    ...
+```
+
+With the service initialized we can now interact with the file attachments API.
+
+## Various Examples
+
+For demonstrative purposes we will upload an `mp4` file and attach to a run of `run_id`.
+Once it is uploaded we will query all file attachments for a particular run and re-download
+what we just uploaded.
+
+```python
+
+from sift_py.grpc.transport import SiftChannelConfig, use_sift_channel
+from sift_py.file_attachment.service import FileAttachmentService
+from sift_py.file_attachment.entity import Entity, EntityType
+from sift_py.file_attachment.metadata import VideoMetadata
+from sift_py.rest import SiftRestConfig
+
+from sift.remote_files.v1.remote_files_pb2 import GetRemoteFileRequest
+from sift.remote_files.v1.remote_files_pb2_grpc import RemoteFileServiceStub
+
+...
+
+with use_sift_channel(sift_channel_config) as channel:
+    file_attachment_service = FileAttachmentService(channel, rest_config)
+
+    run = entity=Entity(
+        entity_id=run_id, # some arbitrary run ID that refers to an existing run
+        entity_type=EntityType.RUN,
+    )
+
+    # uploading the file attachment and attaching it to a run of `run_id`.
+    remote_file = file_attachment_service.upload_attachment(
+        path="path/to/foo.mp4",
+        entity=run,
+        # Metatadata.. optional but recommended for optimal viewing in the application
+        metadata=VideoMetadata(height=2160, width=3840, duration_seconds=5.5, timestamp=datetime(2024, 10, 19, 2, 22, 22),
+        description="thrusters getting too hot" ,
+    )
+
+    # retrieving all of the file attachments for our run
+    all_file_attachments = file_attachment_service.retrieve_attachments(run)
+
+    # downloading our file_attachment and saving it to our current working dir
+    file_attachment_service.download_attachment(remote_file)
+
+    # downloading our file_attachment and saving it somewhere else with a different name
+    file_attachment_service.download_attachment(remote_file, "somewhere/else/foo.mp4")
+
+    # deleting out file attachment from Sift
+    file_attachment_service.delete_file_attachments(remote_file_1, remote_file_2, remote_file_etc)
+```
+
+"""

sift_py/file_attachment/_internal/download.py

@@ -0,0 +1,13 @@
+import io
+from pathlib import Path
+
+import requests
+
+
+def download_remote_file(url: str, out: Path):
+    with requests.get(url, stream=True) as req:
+        req.raise_for_status()
+        with open(out, "wb") as output_file:
+            for chunk in req.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE):
+                if chunk:
+                    output_file.write(chunk)

sift_py/file_attachment/_internal/upload.py

@@ -0,0 +1,100 @@
+import mimetypes
+from pathlib import Path
+from typing import Any, Dict, Optional, Tuple, Union
+from urllib.parse import urljoin
+
+import requests
+from requests_toolbelt import MultipartEncoder
+
+from sift_py._internal.convert.json import to_json
+from sift_py.file_attachment.entity import Entity
+from sift_py.file_attachment.metadata import Metadata
+from sift_py.rest import SiftRestConfig, compute_uri
+
+
+class UploadService:
+    UPLOAD_PATH = "/api/v0/remote-files/upload"
+    UPLOAD_BULK_PATH = "/api/v0/remote-files/upload:bulk"
+
+    _upload_uri: str
+    _upload_bulk_uri: str
+    _apikey: str
+
+    def __init__(self, restconf: SiftRestConfig):
+        base_uri = compute_uri(restconf)
+        self._upload_uri = urljoin(base_uri, self.UPLOAD_PATH)
+        self._upload_bulk_uri = urljoin(base_uri, self.UPLOAD_BULK_PATH)
+        self._apikey = restconf["apikey"]
+
+    def upload_attachment(
+        self,
+        path: Union[str, Path],
+        entity: Entity,
+        metadata: Optional[Metadata] = None,
+        description: Optional[str] = None,
+        organization_id: Optional[str] = None,
+    ) -> str:
+        posix_path = Path(path) if isinstance(path, str) else path
+
+        if not posix_path.is_file():
+            raise Exception(f"Provided path, '{path}', does not point to a regular file.")
+
+        file_name, mimetype, content_encoding = self.__class__._mime_and_content_type_from_path(
+            posix_path
+        )
+
+        if not mimetype:
+            raise Exception(f"The MIME-type of '{posix_path}' could not be computed.")
+
+        with open(path, "rb") as file:
+            form_fields: Dict[str, Any] = {
+                "entityId": entity.entity_id,
+                "entityType": entity.entity_type.value,
+            }
+
+            if content_encoding:
+                form_fields["file"] = (
+                    file_name,
+                    file,
+                    mimetype,
+                    {
+                        "Content-Encoding": content_encoding,
+                    },
+                )
+            else:
+                form_fields["file"] = (file_name, file, mimetype)
+
+            if metadata:
+                form_fields["metadata"] = to_json(metadata)
+
+            if organization_id:
+                form_fields["organizationId"] = organization_id
+
+            if description:
+                form_fields["description"] = description
+
+            form_data = MultipartEncoder(fields=form_fields)
+
+            # https://github.com/requests/toolbelt/issues/312
+            # Issue above is reason for the type ignoring
+            response = requests.post(
+                url=self._upload_uri,
+                data=form_data,  # type: ignore
+                headers={
+                    "Authorization": f"Bearer {self._apikey}",
+                    "Content-Type": form_data.content_type,
+                },
+            )
+
+            if response.status_code != 200:
+                raise Exception(
+                    f"Request failed with status code {response.status_code} ({response.reason})."
+                )
+
+            return response.json().get("remoteFile").get("remoteFileId")
+
+    @staticmethod
+    def _mime_and_content_type_from_path(path: Path) -> Tuple[str, Optional[str], Optional[str]]:
+        file_name = path.name
+        mime, encoding = mimetypes.guess_type(path)
+        return file_name, mime, encoding