sift-stack-py 0.3.2__py3-none-any.whl

sift_py/data_import/_tdms_test.py

@@ -0,0 +1,238 @@
+import json
+from typing import Any, Dict, List, Optional
+
+import pandas as pd
+import pytest
+from nptdms import TdmsFile, types  # type: ignore
+from pytest_mock import MockFixture
+
+from sift_py.data_import.tdms import TdmsUploadService
+from sift_py.rest import SiftRestConfig
+
+
+class MockTdmsChannel:
+    def __init__(
+        self,
+        name: str,
+        group_name: str,
+        properties: Optional[Dict[str, Any]] = None,
+        data: Optional[List[int]] = None,
+        data_type: type = types.Int32,
+    ):
+        self.name: str = name
+        self.group_name: str = group_name
+        self.properties: Optional[Dict[str, str]] = properties or {}
+        self.data: Optional[List[int]] = data or []
+        self.data_type: type = data_type
+
+
+class MockTdmsGroup:
+    def __init__(self, name, channels: List[MockTdmsChannel]):
+        self.name: str = name
+        self.path: str = f"/'{name}'"
+        self._channels: List[MockTdmsChannel] = channels
+        self.properties: Optional[Dict[str, str]] = {}
+
+    def channels(self) -> List[MockTdmsChannel]:
+        return self._channels
+
+
+class MockTdmsFile:
+    def __init__(self, groups: List[MockTdmsGroup]):
+        self._groups: List[MockTdmsGroup] = groups
+        self.properties: Dict[str, str] = {}
+
+    def groups(self) -> List[MockTdmsGroup]:
+        return self._groups
+
+    def as_dataframe(self, *_, **__):
+        return pd.DataFrame()
+
+
+class MockResponse:
+    def __init__(self):
+        self.status_code = 200
+        self.text = json.dumps({"uploadUrl": "some_url.com", "dataImportId": "123-123-123"})
+
+    def json(self) -> dict:
+        return json.loads(self.text)
+
+
+@pytest.fixture
+def mock_tdms_file():
+    mock_tdms_groups = [
+        MockTdmsGroup(
+            name=f"Group {g}",
+            channels=[
+                MockTdmsChannel(
+                    name=f"Test/channel_{c}",
+                    group_name=f"Group {g}",
+                    data=[1, 2, 3],
+                    properties={
+                        "wf_start_time": 0,
+                        "wf_increment": 0.1,
+                        "wf_start_offset": 0,
+                        "extra": "info",
+                    },
+                )
+                for c in range(5)
+            ],
+        )
+        for g in range(5)
+    ]
+
+    return MockTdmsFile(mock_tdms_groups)
+
+
+rest_config: SiftRestConfig = {
+    "uri": "some_uri.com",
+    "apikey": "123123123",
+}
+
+
+def test_tdms_upload_service_upload_validate_path(mocker: MockFixture):
+    mock_path_is_file = mocker.patch("sift_py.data_import.tdms.Path.is_file")
+    mock_path_is_file.return_value = False
+
+    with pytest.raises(Exception, match="does not point to a regular file"):
+        svc = TdmsUploadService(rest_config)
+        svc.upload("some_data.tdms", "asset_name")
+
+
+def test_tdms_upload_success(mocker: MockFixture, mock_tdms_file: MockTdmsFile):
+    mock_path_is_file = mocker.patch("sift_py.data_import.tdms.Path.is_file")
+    mock_path_is_file.return_value = True
+
+    mock_requests_post = mocker.patch("sift_py.data_import.csv.requests.post")
+    mock_requests_post.return_value = MockResponse()
+
+    def mock_tdms_file_constructor(path):
+        """The first call should always return the mocked object since
+        it is mocking a call to open the orignal tdms file.
+
+        The second call should return a real TdmsFile since the unit
+        test will actually create one with filtered channels.
+        """
+        print(path)
+        if path == "some_tdms.tdms":
+            return mock_tdms_file
+        else:
+            return TdmsFile(path)
+
+    mocker.patch("sift_py.data_import.tdms.TdmsFile", mock_tdms_file_constructor)
+
+    svc = TdmsUploadService(rest_config)
+
+    def get_csv_config(mock, n):
+        """Return the CSV config that was created and uploaded under the hood."""
+        return json.loads(mock_requests_post.call_args_list[n].kwargs["data"])["csv_config"]
+
+    # Test without grouping
+    svc.upload("some_tdms.tdms", "asset_name")
+    config = get_csv_config(mock_requests_post, 0)
+    expected_config: Dict[str, Any] = {
+        "asset_name": "asset_name",
+        "run_name": "",
+        "run_id": "",
+        "first_data_row": 2,
+        "time_column": {
+            "format": "TIME_FORMAT_ABSOLUTE_DATETIME",
+            "column_number": 1,
+            "relative_start_time": None,
+        },
+        "data_columns": {},
+    }
+    for i in range(5):
+        for j in range(5):
+            expected_config["data_columns"][str(2 + (i * 5) + j)] = {
+                "name": f"Test channel_{j}",
+                "data_type": "CHANNEL_DATA_TYPE_INT_32",
+                "component": "",
+                "units": "",
+                "description": "None\nwf_start_time: 0\nwf_increment: 0.1\nwf_start_offset: 0\nextra: info\n",
+                "enum_types": [],
+                "bit_field_elements": [],
+            }
+    assert config == expected_config
+
+    # Test with grouping
+    svc.upload("some_tdms.tdms", "asset_name", group_into_components=True)
+    config = get_csv_config(mock_requests_post, 2)
+    for i in range(5):
+        for j in range(5):
+            expected_config["data_columns"][str(2 + (i * 5) + j)]["component"] = f"Group {i}"
+    assert config == expected_config
+
+    # Test with run information
+    svc.upload(
+        "some_tdms.tdms",
+        "asset_name",
+        group_into_components=True,
+        run_name="Run Name",
+        run_id="Run ID",
+    )
+    expected_config["run_name"] = "Run Name"
+    expected_config["run_id"] = "Run ID"
+    config = get_csv_config(mock_requests_post, 4)
+    assert config == expected_config
+
+
+def test_tdms_upload_ignore_errors(mocker: MockFixture):
+    mock_path_is_file = mocker.patch("sift_py.data_import.tdms.Path.is_file")
+    mock_path_is_file.return_value = True
+
+    mocker.patch("sift_py.data_import.tdms.TdmsWriter")
+
+    mock_requests_post = mocker.patch("sift_py.data_import.csv.requests.post")
+    mock_requests_post.return_value = MockResponse()
+
+    # Start with all invalid channels
+    mock_tdms_groups = [
+        MockTdmsGroup(
+            name=f"Group {g}",
+            channels=[MockTdmsChannel(f"channel_{c}", f"Group {g}") for c in range(5)],
+        )
+        for g in range(5)
+    ]
+    mock_tdms_file = MockTdmsFile(mock_tdms_groups)
+
+    def mock_tdms_file_constructor(path):
+        """The first call should always return the mocked object since
+        it is mocking a call to open the orignal tdms file.
+
+        The second call should return a real TdmsFile since the unit
+        test will actually create one with filtered channels.
+        """
+        print(path)
+        if path == "some_tdms.tdms":
+            return mock_tdms_file
+        else:
+            return TdmsFile(path)
+
+    mocker.patch("sift_py.data_import.tdms.TdmsFile", mock_tdms_file_constructor)
+
+    svc = TdmsUploadService(rest_config)
+
+    with pytest.raises(Exception, match="does not contain timing information"):
+        svc.upload("some_tdms.tdms", "asset_name")
+
+    with pytest.raises(Exception, match="No valid channels remaining"):
+        svc.upload("some_tdms.tdms", "asset_name", ignore_errors=True)
+
+
+def test_tdms_upload_unknown_data_type(mocker: MockFixture, mock_tdms_file: MockTdmsFile):
+    mock_path_is_file = mocker.patch("sift_py.data_import.tdms.Path.is_file")
+    mock_path_is_file.return_value = True
+
+    mocker.patch("sift_py.data_import.tdms.TdmsWriter")
+
+    mock_requests_post = mocker.patch("sift_py.data_import.csv.requests.post")
+    mock_requests_post.return_value = MockResponse()
+
+    mock_tdms_file.groups()[0].channels()[0].data_type = types.ComplexDoubleFloat
+    mocker.patch("sift_py.data_import.tdms.TdmsFile").return_value = mock_tdms_file
+
+    svc = TdmsUploadService(rest_config)
+
+    with pytest.raises(Exception, match="data type not supported"):
+        svc.upload("some_tdms.tdms", "asset_name")

sift_py/data_import/ch10.py

@@ -0,0 +1,157 @@
+import json
+from typing import Any, Dict, Optional
+
+import requests
+
+from sift_py.data_import.config import CsvConfig
+from sift_py.data_import.csv import CsvUploadService
+from sift_py.data_import.status import DataImportService
+from sift_py.data_import.time_format import TimeFormatType
+
+
+class BaseCh10File:
+    """
+    Base class for uploading IRIG Chapter 10/Chapter 11 files.
+
+    Implement a concrete version of this class that parses a ch10 stream and returns
+    a csv row of data on each iteration.
+
+    Set `gzip` to `True` if sending a compressed stream.
+
+    Example:
+    ```python
+
+    class Ch10(BaseCh10File):
+
+        def __init__(self, path):
+            self.file = open(path, "rb")
+            self.initialize_csv_data_columns = None
+
+        def initialize_csv_data_columns(self):
+            self.csv_config_data_columns = self.process_ch10_computer_f1_packet()
+
+        def process_ch10_computer_f1_packet(self) -> Dict[int, dict]:
+            # Processes the first Computer F1 packet
+            # and returns the measurements as a dict.
+            ...
+
+        def process_ch10_pcm_packet(self) -> str:
+            # Processed the data packets and returns
+            # a CSV row.
+            ...
+
+        def __next__(self) -> str:
+            # On all iterations, return data for the CSV file.
+            if end_of_file:
+                raise StopIteration()
+            else:
+                return self.process_ch10_data_packet()
+    ```
+    """
+
+    csv_config_data_columns: Dict[int, dict]
+    gzip: bool = False
+
+    def initialize_csv_data_columns(self) -> None:
+        """
+        Must populate the `csv_config_data_columns` attribute
+        that is the data_columns entry in the CsvConfig.
+
+        See the Sift data_import module or API docs for the schema.
+        """
+        raise NotImplementedError
+
+    def __iter__(self):
+        return self
+
+    def __next__(self) -> str:
+        raise NotImplementedError
+
+
+class Ch10UploadService(CsvUploadService):
+    """Service to upload ch10 files."""
+
+    def upload(  # type: ignore
+        self,
+        ch10_file: BaseCh10File,
+        asset_name: str,
+        time_format: TimeFormatType = TimeFormatType.ABSOLUTE_UNIX_NANOSECONDS,
+        run_name: Optional[str] = None,
+        run_id: Optional[str] = None,
+    ) -> DataImportService:
+        """
+        Uploads the ch10 file to the specified asset.
+
+        Override `time_format` to specify the time data format. Default is `TimeFormatType.ABSOLUTE_UNIX_NANOSECONDS`.
+        Override `run_name` to specify the name of the run to create for this data. Default is None.
+        Override `run_id` to specify the id of the run to add this data to. Default is None.
+        """
+        ch10_file.initialize_csv_data_columns()
+
+        assert getattr(ch10_file, "csv_config_data_columns"), (
+            "`csv_config_data_columns` was not set correctly on the first iteration"
+        )
+
+        config_info: Dict[str, Any] = {
+            "asset_name": asset_name,
+            "first_data_row": 2,
+            "time_column": {
+                "format": time_format,
+                "column_number": 1,
+            },
+            "data_columns": ch10_file.csv_config_data_columns,
+        }
+        if run_name:
+            config_info["run_name"] = run_name
+
+        if run_id:
+            config_info["run_id"] = run_name
+
+        csv_config = CsvConfig(config_info)
+
+        response = requests.post(
+            url=self._upload_uri,
+            headers={
+                "Authorization": f"Bearer {self._apikey}",
+                "Content-Encoding": "application/octet-stream",
+            },
+            data=json.dumps({"csv_config": csv_config.to_dict()}),
+        )
+
+        if response.status_code != 200:
+            raise Exception(
+                f"Config file upload request failed with status code {response.status_code}. {response.text}"
+            )
+
+        try:
+            upload_info = response.json()
+        except (json.decoder.JSONDecodeError, KeyError) as e:
+            raise Exception(f"Invalid response: {response.text}.\n{e}")
+
+        try:
+            upload_url: str = upload_info["uploadUrl"]
+            data_import_id: str = upload_info["dataImportId"]
+        except KeyError as e:
+            raise Exception(
+                f"Response missing required keys: {e}. This is unexpected. Please reach out to the Sift team about this error."
+            )
+
+        headers = {
+            "Authorization": f"Bearer {self._apikey}",
+        }
+
+        if ch10_file.gzip:
+            headers["Content-Encoding"] = "gzip"
+
+        response = requests.post(
+            url=upload_url,
+            headers=headers,
+            data=ch10_file,
+        )
+
+        if response.status_code != 200:
+            raise Exception(
+                f"Data file upload request failed with status code {response.status_code}. {response.text}"
+            )
+
+        return DataImportService(self._rest_conf, data_import_id)

sift_py/data_import/config.py

@@ -0,0 +1,19 @@
+from typing import Any, Dict
+
+from sift_py.data_import._config import CsvConfigImpl
+
+
+class CsvConfig:
+    """
+    Defines the CSV config for data imports.
+    """
+
+    def __init__(self, config_info: Dict[str, Any]):
+        self._config_info = config_info
+        self._csv_config = CsvConfigImpl(**self._config_info)
+
+    def to_json(self) -> str:
+        return self._csv_config.model_dump_json()
+
+    def to_dict(self) -> Dict[str, Any]:
+        return self._csv_config.model_dump()

sift_py/data_import/csv.py

@@ -0,0 +1,259 @@
+import json
+import mimetypes
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple, Union, cast
+from urllib.parse import urljoin, urlparse
+
+import pandas as pd
+import requests
+
+from sift_py.data_import.config import CsvConfig
+from sift_py.data_import.status import DataImportService
+from sift_py.data_import.time_format import TimeFormatType
+from sift_py.ingestion.channel import ChannelDataType
+from sift_py.rest import SiftRestConfig, compute_uri
+
+
+class CsvUploadService:
+    UPLOAD_PATH = "/api/v1/data-imports:upload"
+    URL_PATH = "/api/v1/data-imports:url"
+
+    _rest_conf: SiftRestConfig
+    _upload_uri: str
+    _url_uri: str
+    _apikey: str
+
+    def __init__(self, rest_conf: SiftRestConfig):
+        self._rest_conf = rest_conf
+        base_uri = compute_uri(rest_conf)
+        self._apikey = rest_conf["apikey"]
+        self._upload_uri = urljoin(base_uri, self.UPLOAD_PATH)
+        self._url_uri = urljoin(base_uri, self.URL_PATH)
+
+    def upload(
+        self,
+        path: Union[str, Path],
+        csv_config: CsvConfig,
+    ) -> DataImportService:
+        """
+        Uploads the CSV file pointed to by `path` using a custom CSV config.
+        """
+        content_encoding = self._validate_file_type(path)
+
+        response = requests.post(
+            url=self._upload_uri,
+            headers={
+                "Authorization": f"Bearer {self._apikey}",
+                "Content-Encoding": "application/octet-stream",
+            },
+            data=json.dumps({"csv_config": csv_config.to_dict()}),
+        )
+
+        if response.status_code != 200:
+            raise Exception(
+                f"Config file upload request failed with status code {response.status_code}. {response.text}"
+            )
+
+        try:
+            upload_info = response.json()
+        except (json.decoder.JSONDecodeError, KeyError):
+            raise Exception(f"Invalid response: {response.text}")
+
+        try:
+            upload_url: str = upload_info["uploadUrl"]
+            data_import_id: str = upload_info["dataImportId"]
+        except KeyError as e:
+            raise Exception(f"Response missing required keys: {e}")
+
+        with open(path, "rb") as f:
+            headers = {
+                "Authorization": f"Bearer {self._apikey}",
+                "Content-Encoding": content_encoding,
+            }
+
+            response = requests.post(
+                url=upload_url,
+                headers=headers,
+                data=f,
+            )
+
+            if response.status_code != 200:
+                raise Exception(
+                    f"Data file upload request failed with status code {response.status_code}. {response.text}"
+                )
+
+            return DataImportService(self._rest_conf, data_import_id)
+
+    def upload_from_url(
+        self,
+        url: str,
+        csv_config: CsvConfig,
+    ) -> DataImportService:
+        """
+        Uploads the CSV file pointed to by `url` using a custom CSV config.
+        """
+        parsed_url = urlparse(url)
+        if parsed_url.scheme not in ["s3", "http", "https"]:
+            raise Exception(
+                f"Invalid URL scheme: '{parsed_url.scheme}'. Only S3 and HTTP(S) URLs are supported."
+            )
+
+        headers = {"Authorization": f"Bearer {self._apikey}"}
+
+        response = requests.post(
+            url=self._url_uri,
+            headers=headers,
+            data=json.dumps(
+                (
+                    {
+                        "url": url,
+                        "csv_config": csv_config.to_dict(),
+                    }
+                )
+            ),
+        )
+
+        if response.status_code != 200:
+            raise Exception(
+                f"URL upload request failed with status code {response.status_code}. {response.text}"
+            )
+
+        try:
+            upload_info = response.json()
+        except (json.decoder.JSONDecodeError, KeyError) as e:
+            raise Exception(f"Invalid response: {e}")
+
+        try:
+            data_import_id: str = upload_info["dataImportId"]
+        except KeyError as e:
+            raise Exception(f"Response missing required keys: {e}")
+
+        return DataImportService(self._rest_conf, data_import_id)
+
+    def simple_upload(
+        self,
+        asset_name: str,
+        path: Union[str, Path],
+        first_data_row: int = 2,
+        time_column: int = 1,
+        time_format: TimeFormatType = TimeFormatType.ABSOLUTE_DATETIME,
+        run_name: Optional[str] = None,
+        run_id: Optional[str] = None,
+        units_row: Optional[int] = None,
+        descriptions_row: Optional[int] = None,
+        relative_start_time: Optional[str] = None,
+    ) -> DataImportService:
+        """
+        Uploads the CSV file pointed to by `path` to the specified asset. This function will
+        infer the data types and assume certain things about how the data is formatted. See the options
+        below for what parameters can be overridden. Use `upload` if you need to specify a custom CSV config.
+
+        Override `first_data_row` to specify which is the first row with data. Default is 2.
+        Override `time_column` to specify which column contains timestamp information. Default is 1.
+        Override `time_format` to specify the time data format. Default is `TimeFormatType.ABSOLUTE_DATETIME`.
+        Override `run_name` to specify the name of the run to create for this data. Default is None.
+        Override `run_id` to specify the id of the run to add this data to. Default is None.
+        Override `units_row` to specify which row contains unit information. Default is None.
+        Override `descriptions_row` to specify which row contains channel description information. Default is None.
+        Override `relative_start_time` if a relative time format is used. Default is None.
+        """
+        self._validate_file_type(path)
+
+        # Convert to 0 index
+        skip_rows: List[int] = []
+        if units_row is not None:
+            units_row -= 1
+            skip_rows.append(units_row)
+        if descriptions_row is not None:
+            descriptions_row -= 1
+            skip_rows.append(descriptions_row)
+
+        data_config = {}
+        df = pd.read_csv(path, skiprows=skip_rows)
+
+        units: List[str] = []
+        if units_row is not None:
+            df_units = pd.read_csv(path, nrows=units_row)
+            units = list(cast(List[str], df_units.iloc[units_row - 1].astype(str)))
+
+        descriptions: List[str] = []
+        if descriptions_row is not None:
+            df_descriptions = pd.read_csv(path, nrows=descriptions_row)
+            descriptions = list(
+                cast(List[str], df_descriptions.iloc[descriptions_row - 1].astype(str))
+            )
+
+        for i, header in enumerate(df.columns):
+            if i + 1 == time_column:
+                continue
+
+            raw_dtype = str(df[df.columns[i]].dtype)
+            if raw_dtype == "float64":
+                raw_dtype = "double"
+            # String columns are set to 'object'. Use infer_dtypes
+            # to verify this is a string column
+            elif raw_dtype == "object":
+                raw_dtype = pd.api.types.infer_dtype(df[df.columns[i]], skipna=False)
+
+            data_type = ChannelDataType.from_str(raw_dtype)
+            if data_type is None:
+                raise Exception(
+                    f"Unable to upload data type in column {i + 1} {header}: Type: {raw_dtype}."
+                )
+            data_config[i + 1] = {"name": header, "data_type": data_type}
+
+            if units:
+                data_config[i + 1]["units"] = units[i] if units[i] != "nan" else ""
+
+            if descriptions:
+                data_config[i + 1]["description"] = (
+                    descriptions[i] if descriptions[i] != "nan" else ""
+                )
+
+        config_info: Dict[str, Any] = {
+            "asset_name": asset_name,
+            "first_data_row": first_data_row,
+            "time_column": {
+                "format": time_format,
+                "column_number": time_column,
+            },
+            "data_columns": data_config,
+        }
+
+        if run_name is not None:
+            config_info["run_name"] = run_name
+
+        if run_id is not None:
+            config_info["run_id"] = run_id
+
+        if relative_start_time is not None:
+            config_info["time_column"]["relative_start_time"] = relative_start_time
+
+        csv_config = CsvConfig(config_info)
+
+        return self.upload(path, csv_config)
+
+    def _validate_file_type(self, path: Union[str, Path]) -> Optional[str]:
+        posix_path = Path(path) if isinstance(path, str) else path
+
+        if not posix_path.is_file():
+            raise Exception(f"Provided path, '{path}', does not point to a regular file.")
+
+        _, mimetype, content_encoding = self.__class__._mime_and_content_type_from_path(posix_path)
+
+        if not mimetype:
+            raise Exception(f"The MIME-type of '{posix_path}' could not be computed.")
+
+        valid_types = ["test/plain", "text/csv", "application/vnd.ms-excel"]
+        if mimetype not in valid_types:
+            raise Exception(
+                f"{path} is not a valid file type ({mimetype}). Must be {', '.join(valid_types)}."
+            )
+
+        return content_encoding
+
+    @staticmethod
+    def _mime_and_content_type_from_path(path: Path) -> Tuple[str, Optional[str], Optional[str]]:
+        file_name = path.name
+        mime, encoding = mimetypes.guess_type(path)
+        return file_name, mime, encoding