sift-stack-py 0.3.2__py3-none-any.whl

sift_py/data_import/_csv_test.py

@@ -0,0 +1,395 @@
+import json
+
+import pandas as pd
+import pytest
+from pytest_mock import MockFixture
+
+from sift_py.data_import.config import CsvConfig
+from sift_py.data_import.csv import CsvUploadService
+from sift_py.rest import SiftRestConfig
+
+
+class MockResponse:
+    status_code: int
+    text: str
+
+    def __init__(self, status_code: int, text: str):
+        self.status_code = status_code
+        self.text = text
+
+    def json(self) -> dict:
+        return json.loads(self.text)
+
+
+csv_config = CsvConfig(
+    {
+        "asset_name": "test_asset",
+        "first_data_row": 2,
+        "time_column": {
+            "format": "TIME_FORMAT_ABSOLUTE_DATETIME",
+            "column_number": 1,
+        },
+        "data_columns": {
+            2: {
+                "name": "channel_1",
+                "data_type": "CHANNEL_DATA_TYPE_DOUBLE",
+            }
+        },
+    }
+)
+
+rest_config: SiftRestConfig = {
+    "uri": "some_uri.com",
+    "apikey": "123123123",
+}
+
+
+def test_csv_upload_service_upload_validate_path(mocker: MockFixture):
+    mock_path_is_file = mocker.patch("sift_py.data_import.csv.Path.is_file")
+    mock_path_is_file.return_value = False
+
+    with pytest.raises(Exception, match="does not point to a regular file"):
+        svc = CsvUploadService(rest_config)
+        svc.upload(
+            path="some_csv.csv",
+            csv_config=csv_config,
+        )
+
+
+def test_csv_upload_service_upload_validate_mime_type(mocker: MockFixture):
+    mock_path_is_file = mocker.patch("sift_py.data_import.csv.Path.is_file")
+    mock_path_is_file.return_value = True
+
+    with pytest.raises(Exception, match="MIME"):
+        svc = CsvUploadService(rest_config)
+        svc.upload(
+            path="some_csv.asdfghjkl",
+            csv_config=csv_config,
+        )
+
+    with pytest.raises(Exception, match="Must be"):
+        svc = CsvUploadService(rest_config)
+        svc.upload(
+            path="some_file.pdf",
+            csv_config=csv_config,
+        )
+
+
+def test_csv_upload_service_invalid_config_response(mocker: MockFixture):
+    mock_path_is_file = mocker.patch("sift_py.data_import.csv.Path.is_file")
+    mock_path_is_file.return_value = True
+
+    mock_requests_post = mocker.patch("sift_py.data_import.csv.requests.post")
+    mock_requests_post.return_value = MockResponse(status_code=400, text="Invalid request")
+    with pytest.raises(Exception, match="Config file upload request failed"):
+        svc = CsvUploadService(rest_config)
+
+        svc.upload(
+            path="some_csv.csv",
+            csv_config=csv_config,
+        )
+
+
+def test_csv_upload_service_invalid_data_response(mocker: MockFixture):
+    mock_path_is_file = mocker.patch("sift_py.data_import.csv.Path.is_file")
+    mock_path_is_file.return_value = True
+
+    mocker.patch(
+        "sift_py.data_import.csv.open",
+        mocker.mock_open(),
+    )
+
+    mock_requests_post = mocker.patch("sift_py.data_import.csv.requests.post")
+    mock_requests_post.return_value = MockResponse(status_code=200, text="asdgasdg")
+
+    with pytest.raises(Exception, match="Invalid response"):
+        svc = CsvUploadService(rest_config)
+
+        svc.upload(
+            path="some_csv.csv",
+            csv_config=csv_config,
+        )
+
+    mock_requests_post = mocker.patch("sift_py.data_import.csv.requests.post")
+    mock_requests_post.return_value = MockResponse(status_code=200, text="{}")
+
+    with pytest.raises(Exception, match="Response missing required keys"):
+        svc = CsvUploadService(rest_config)
+
+        svc.upload(
+            path="some_csv.csv",
+            csv_config=csv_config,
+        )
+
+    mock_requests_post.side_effect = [
+        MockResponse(
+            status_code=200,
+            text=json.dumps({"uploadUrl": "some_url.com", "dataImportId": "123-123-123"}),
+        ),
+        MockResponse(status_code=400, text="Invalid request"),
+    ]
+
+    with pytest.raises(Exception, match="Data file upload request failed"):
+        svc = CsvUploadService(rest_config)
+
+        svc.upload(
+            path="some_csv.csv",
+            csv_config=csv_config,
+        )
+
+
+def test_csv_upload_service_success(mocker: MockFixture):
+    mock_path_is_file = mocker.patch("sift_py.data_import.csv.Path.is_file")
+    mock_path_is_file.return_value = True
+
+    mock_requests_post = mocker.patch("sift_py.data_import.csv.requests.post")
+    mock_requests_post.side_effect = [
+        MockResponse(
+            status_code=200,
+            text=json.dumps({"uploadUrl": "some_url.com", "dataImportId": "123-123-123"}),
+        ),
+        MockResponse(status_code=200, text=""),
+    ]
+
+    mocker.patch(
+        "sift_py.data_import.csv.open",
+        mocker.mock_open(),
+    )
+    svc = CsvUploadService(
+        {
+            "uri": "some_uri.com",
+            "apikey": "123123123",
+        },
+    )
+
+    svc.upload(
+        path="some_csv.csv",
+        csv_config=csv_config,
+    )
+
+
+def test_csv_upload_service_upload_validate_url(mocker: MockFixture):
+    with pytest.raises(Exception, match="Invalid URL scheme:"):
+        svc = CsvUploadService(rest_config)
+
+        svc.upload_from_url(
+            url="asdf://some_url.com/file.csv",
+            csv_config=csv_config,
+        )
+
+
+def test_csv_upload_service_upload_from_url_invalid_config(mocker: MockFixture):
+    mock_requests_post = mocker.patch("sift_py.data_import.csv.requests.post")
+    mock_requests_post.return_value = MockResponse(status_code=400, text="Invalid request")
+    with pytest.raises(Exception, match="URL upload request failed"):
+        svc = CsvUploadService(rest_config)
+
+        svc.upload_from_url(
+            url="http://some_url.com/file.csv",
+            csv_config=csv_config,
+        )
+
+
+def test_csv_upload_service_upload_from_url_success(mocker: MockFixture):
+    mock_requests_post = mocker.patch("sift_py.data_import.csv.requests.post")
+    mock_requests_post.return_value = MockResponse(
+        status_code=200,
+        text=json.dumps({"uploadUrl": "some_url.com", "dataImportId": "123-123-123"}),
+    )
+    svc = CsvUploadService(
+        {
+            "uri": "some_uri.com",
+            "apikey": "123123123",
+        },
+    )
+
+    svc.upload_from_url(
+        url="http://some_url.com/file.csv",
+        csv_config=csv_config,
+    )
+
+
+def test_simple_upload_invalid_csv(mocker: MockFixture):
+    mock_path_is_file = mocker.patch("sift_py.data_import.csv.Path.is_file")
+    mock_path_is_file.return_value = True
+
+    mock_read_csv = mocker.patch("sift_py.data_import.csv.pd.read_csv")
+    mock_read_csv.return_value = pd.DataFrame(
+        {
+            "time": [1, 2, 3],
+            "channel_1": [1, 1.0, True],
+        }
+    )
+    with pytest.raises(Exception, match="Unable to upload.*"):
+        svc = CsvUploadService(rest_config)
+        svc.simple_upload("test_asset", "sample.csv")
+
+    mock_read_csv = mocker.patch("sift_py.data_import.csv.pd.read_csv")
+    mock_read_csv.return_value = pd.DataFrame(
+        {
+            "time": [1, 2, 3],
+            "channel_1": [complex(1), complex(1), complex(1)],
+        }
+    )
+    with pytest.raises(Exception, match="Unable to upload.*"):
+        svc = CsvUploadService(rest_config)
+        svc.simple_upload("test_asset", "sample.csv")
+
+    mock_read_csv = mocker.patch("sift_py.data_import.csv.pd.read_csv")
+    mock_read_csv.return_value = pd.DataFrame(
+        {
+            "time": [1, 2, 3],
+            "channel_bool": [True, True, False],
+            "channel_int": [-1, 2, 0],
+            "channel_double": [1.0, 2.0, -3.3],
+            "channel_string": ["a", "b", "c"],
+        }
+    )
+    mock_requests_post = mocker.patch("sift_py.data_import.csv.requests.post")
+    mock_requests_post.side_effect = [
+        MockResponse(
+            status_code=200,
+            text=json.dumps({"uploadUrl": "some_url.com", "dataImportId": "123-123-123"}),
+        ),
+        MockResponse(status_code=200, text=""),
+    ]
+    mocker.patch(
+        "sift_py.data_import.csv.open",
+        mocker.mock_open(),
+    )
+    svc = CsvUploadService(rest_config)
+    svc.simple_upload("test_asset", "sample.csv")
+
+
+def test_simple_upload_metadata_csv(mocker: MockFixture):
+    mock_path_is_file = mocker.patch("sift_py.data_import.csv.Path.is_file")
+    mock_path_is_file.return_value = True
+
+    def mock_read_csv(*_, **kwargs):
+        if "skiprows" in kwargs:
+            return pd.DataFrame(
+                {
+                    "time": [1, 2, 3],
+                    "channel_int": [-1, 2, 1],
+                }
+            )
+        else:
+            return pd.DataFrame(
+                {
+                    "time": ["s", "a description", 1, 2, 3],
+                    "channel_int": ["degC", "another description", -1, 2, 1],
+                }
+            )
+
+    mocker.patch("sift_py.data_import.csv.pd.read_csv", mock_read_csv)
+
+    mock_requests_post = mocker.patch("sift_py.data_import.csv.requests.post")
+    mock_requests_post.side_effect = [
+        MockResponse(
+            status_code=200,
+            text=json.dumps({"uploadUrl": "some_url.com", "dataImportId": "123-123-123"}),
+        ),
+        MockResponse(status_code=200, text=""),
+    ]
+    mocker.patch(
+        "sift_py.data_import.csv.open",
+        mocker.mock_open(),
+    )
+    svc = CsvUploadService(rest_config)
+
+    svc.simple_upload("test_asset", "sample.csv", units_row=2, descriptions_row=3)
+
+    expected_csv_config = CsvConfig(
+        {
+            "asset_name": "test_asset",
+            "run_name": "",
+            "run_id": "",
+            "first_data_row": 2,
+            "time_column": {
+                "format": "TIME_FORMAT_ABSOLUTE_DATETIME",
+                "column_number": 1,
+            },
+            "data_columns": {
+                "2": {
+                    "name": "channel_int",
+                    "data_type": "CHANNEL_DATA_TYPE_INT_64",
+                    "component": "",
+                    "units": "degC",
+                    "description": "another description",
+                    "enum_types": [],
+                    "bit_field_elements": [],
+                }
+            },
+        }
+    )
+
+    mock_requests_post.assert_any_call(
+        url="https://some_uri.com/api/v1/data-imports:upload",
+        headers={
+            "Authorization": "Bearer 123123123",
+            "Content-Encoding": "application/octet-stream",
+        },
+        data=json.dumps({"csv_config": expected_csv_config.to_dict()}),
+    )
+
+
+def test_simple_upload_uint64_csv(mocker: MockFixture):
+    mock_path_is_file = mocker.patch("sift_py.data_import.csv.Path.is_file")
+    mock_path_is_file.return_value = True
+
+    mock_read_csv = mocker.patch("sift_py.data_import.csv.pd.read_csv")
+    mock_read_csv.return_value = pd.DataFrame(
+        {
+            "time": [1, 2, 3],
+            "channel_uint64": [1, 2, 2**63],
+        }
+    )
+
+    mock_requests_post = mocker.patch("sift_py.data_import.csv.requests.post")
+    mock_requests_post.side_effect = [
+        MockResponse(
+            status_code=200,
+            text=json.dumps({"uploadUrl": "some_url.com", "dataImportId": "123-123-123"}),
+        ),
+        MockResponse(status_code=200, text=""),
+    ]
+    mocker.patch(
+        "sift_py.data_import.csv.open",
+        mocker.mock_open(),
+    )
+    svc = CsvUploadService(rest_config)
+
+    svc.simple_upload("test_asset", "sample.csv")
+
+    expected_csv_config = CsvConfig(
+        {
+            "asset_name": "test_asset",
+            "run_name": "",
+            "run_id": "",
+            "first_data_row": 2,
+            "time_column": {
+                "format": "TIME_FORMAT_ABSOLUTE_DATETIME",
+                "column_number": 1,
+            },
+            "data_columns": {
+                "2": {
+                    "name": "channel_uint64",
+                    "data_type": "CHANNEL_DATA_TYPE_UINT_64",
+                    "component": "",
+                    "units": "",
+                    "description": "",
+                    "enum_types": [],
+                    "bit_field_elements": [],
+                }
+            },
+        }
+    )
+
+    mock_requests_post.assert_any_call(
+        url="https://some_uri.com/api/v1/data-imports:upload",
+        headers={
+            "Authorization": "Bearer 123123123",
+            "Content-Encoding": "application/octet-stream",
+        },
+        data=json.dumps({"csv_config": expected_csv_config.to_dict()}),
+    )

sift_py/data_import/_status_test.py

@@ -0,0 +1,176 @@
+import json
+from copy import deepcopy
+
+import pytest
+from pytest_mock import MockFixture
+
+from sift_py.data_import.status import DataImportService, DataImportStatusType
+from sift_py.rest import SiftRestConfig
+
+rest_config: SiftRestConfig = {
+    "uri": "some_uri.com",
+    "apikey": "123123123",
+}
+
+
+@pytest.fixture
+def data_import_data():
+    return {
+        "dataImport": {
+            "dataImportId": "random-data-import-id",
+            "createdDate": "2024-10-07T18:37:00.146649Z",
+            "modifiedDate": "2024-10-07T18:37:00.146649Z",
+            "sourceUrl": "",
+            "status": "",
+            "errorMessage": "",
+            "csvConfig": {},
+        }
+    }
+
+
+class MockResponse:
+    status_code: int
+    text: str
+
+    def __init__(self, status_code: int, text: str):
+        self.status_code = status_code
+        self.text = text
+
+    def json(self):
+        return json.loads(self.text)
+
+    def raise_for_status(self):
+        if self.status_code != 200:
+            raise Exception("Invalid status")
+
+
+def test_get_status(mocker: MockFixture, data_import_data: dict):
+    mock_requests_post = mocker.patch("sift_py.data_import.status.requests.get")
+    data_import_data["dataImport"]["status"] = "DATA_IMPORT_STATUS_SUCCEEDED"
+    mock_requests_post.return_value = MockResponse(
+        status_code=200, text=json.dumps(data_import_data)
+    )
+    service = DataImportService(rest_config, "123-123-123")
+    assert service.get_data_import().status == DataImportStatusType.SUCCEEDED
+
+    data_import_data["dataImport"]["status"] = "DATA_IMPORT_STATUS_PENDING"
+    mock_requests_post.return_value = MockResponse(
+        status_code=200, text=json.dumps(data_import_data)
+    )
+    service = DataImportService(rest_config, "123-123-123")
+    assert service.get_data_import().status == DataImportStatusType.PENDING
+
+    data_import_data["dataImport"]["status"] = "DATA_IMPORT_STATUS_IN_PROGRESS"
+    mock_requests_post.return_value = MockResponse(
+        status_code=200, text=json.dumps(data_import_data)
+    )
+    service = DataImportService(rest_config, "123-123-123")
+    assert service.get_data_import().status == DataImportStatusType.IN_PROGRESS
+
+    data_import_data["dataImport"]["status"] = "DATA_IMPORT_STATUS_FAILED"
+    mock_requests_post.return_value = MockResponse(
+        status_code=200, text=json.dumps(data_import_data)
+    )
+    service = DataImportService(rest_config, "123-123-123")
+    assert service.get_data_import().status == DataImportStatusType.FAILED
+
+    data_import_data["dataImport"]["status"] = "INVALID_STATUS"
+    with pytest.raises(Exception, match="Invalid data import status"):
+        mock_requests_post.return_value = MockResponse(
+            status_code=200, text=json.dumps(data_import_data)
+        )
+        service = DataImportService(rest_config, "123-123-123")
+        service.get_data_import()
+
+
+def test_wait_success(mocker: MockFixture, data_import_data: dict):
+    mock_time_sleep = mocker.patch("sift_py.data_import.status.time.sleep")
+    mock_requests_get = mocker.patch("sift_py.data_import.status.requests.get")
+
+    succeeded = deepcopy(data_import_data)
+    succeeded["dataImport"]["status"] = "DATA_IMPORT_STATUS_SUCCEEDED"
+
+    pending = deepcopy(data_import_data)
+    pending["dataImport"]["status"] = "DATA_IMPORT_STATUS_PENDING"
+
+    in_progress = deepcopy(data_import_data)
+    in_progress["dataImport"]["status"] = "DATA_IMPORT_STATUS_IN_PROGRESS"
+
+    mock_requests_get.side_effect = [
+        MockResponse(
+            status_code=200,
+            text=json.dumps(pending),
+        ),
+        MockResponse(
+            status_code=200,
+            text=json.dumps(in_progress),
+        ),
+        MockResponse(
+            status_code=200,
+            text=json.dumps(succeeded),
+        ),
+    ]
+
+    service = DataImportService(rest_config, "123-123-123")
+    assert service.wait_until_complete().status == DataImportStatusType.SUCCEEDED
+    mock_time_sleep.assert_any_call(1)
+    mock_time_sleep.assert_any_call(2)
+
+
+def test_wait_failure(mocker: MockFixture, data_import_data: dict):
+    mock_requests_get = mocker.patch("sift_py.data_import.status.requests.get")
+
+    failed = deepcopy(data_import_data)
+    failed["dataImport"]["status"] = "DATA_IMPORT_STATUS_FAILED"
+
+    pending = deepcopy(data_import_data)
+    pending["dataImport"]["status"] = "DATA_IMPORT_STATUS_PENDING"
+
+    in_progress = deepcopy(data_import_data)
+    in_progress["dataImport"]["status"] = "DATA_IMPORT_STATUS_IN_PROGRESS"
+
+    mock_requests_get.side_effect = [
+        MockResponse(
+            status_code=200,
+            text=json.dumps(pending),
+        ),
+        MockResponse(
+            status_code=200,
+            text=json.dumps(in_progress),
+        ),
+        MockResponse(
+            status_code=200,
+            text=json.dumps(failed),
+        ),
+    ]
+
+    service = DataImportService(rest_config, "123-123-123")
+    assert service.wait_until_complete().status == DataImportStatusType.FAILED
+
+
+def test_wait_max_polling_interval(mocker: MockFixture, data_import_data: dict):
+    mock_time_sleep = mocker.patch("sift_py.data_import.status.time.sleep")
+    mock_requests_get = mocker.patch("sift_py.data_import.status.requests.get")
+
+    succeeded = deepcopy(data_import_data)
+    succeeded["dataImport"]["status"] = "DATA_IMPORT_STATUS_SUCCEEDED"
+
+    in_progress = deepcopy(data_import_data)
+    in_progress["dataImport"]["status"] = "DATA_IMPORT_STATUS_IN_PROGRESS"
+
+    mock_requests_get.side_effect = [
+        MockResponse(
+            status_code=200,
+            text=json.dumps(in_progress),
+        )
+        for _ in range(60)
+    ] + [
+        MockResponse(
+            status_code=200,
+            text=json.dumps(succeeded),
+        )
+    ]
+
+    service = DataImportService(rest_config, "123-123-123")
+    assert service.wait_until_complete().status == DataImportStatusType.SUCCEEDED
+    mock_time_sleep.assert_called_with(60)