qmenta-core 4.1.dev709__tar.gz → 4.1.dev711__tar.gz

{qmenta_core-4.1.dev709 → qmenta_core-4.1.dev711}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: qmenta-core
-Version: 4.1.dev709
+Version: 4.1.dev711
 Summary: QMENTA core library to communicate with the QMENTA platform.
 License: Proprietary
 Author: QMENTA
@@ -16,9 +16,10 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: blinker (>=1.4,<2.0)
 Requires-Dist: importlib-metadata (>=6.8.0,<7.0.0)
+Requires-Dist: packaging (>=25.0,<26.0)
 Requires-Dist: python-dotenv (>=1.0.0,<2.0.0)
 Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
-Requires-Dist: qmenta-anon (>=2.0,<3.0)
+Requires-Dist: qmenta-anon (>=2.1.dev377,<3.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: xdg (>=6.0.0,<7.0.0)
 Project-URL: Documentation, https://docs.qmenta.com/core/

{qmenta_core-4.1.dev709 → qmenta_core-4.1.dev711}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "qmenta-core"
-version = "4.1.dev709"
+version = "4.1.dev711"
 description = "QMENTA core library to communicate with the QMENTA platform."
 license = "Proprietary"
 authors = ["QMENTA <dev@qmenta.com>"]
@@ -22,20 +22,21 @@ qmenta-auth = 'qmenta.core.auth:main'
 python = "^3.10"
 requests = "^2.31.0"
 pyyaml = "^6.0.1"
-qmenta-anon = "^2.0"
+qmenta-anon = "^2.1.dev377"
 importlib-metadata = "^6.8.0"
 xdg = "^6.0.0"
 python-dotenv = "^1.0.0"
 # We are not requiring the latest blinker 1.6.2 because that is not
 #   available in Google Colab, see EN-1810.
 blinker = "^1.4"
+packaging = "^25.0"
 
 [tool.poetry.group.dev.dependencies]
 flake8 = "^6.1.0"
 mypy = "^1.5.1"
 pytest = "^7.4.0"
 coverage = {version = "^7.3.0", extras = ["toml"]}
-
+pytest-cov = "^6.1.1"
 sphinx-rtd-theme = "^1.3.0"
 dom-toml = "^0.6.1"
 scriv = {version = "^1.3.1", extras = ["toml"]}

{qmenta_core-4.1.dev709 → qmenta_core-4.1.dev711}/src/qmenta/core/auth.py

@@ -35,11 +35,12 @@ class Needs2FAError(ActionFailedError):
 
 @unique
 class PlatformURL(Enum):
-    platform = 'platform.qmenta.com'
-    staging = 'staging.qmenta.com'
-    test = 'test.qmenta.com'
-    local_ip = "127.0.0.1:8080"
-    localhost = "localhost:8080"
+    platform = 'https://platform.qmenta.com'
+    staging = 'https://staging.qmenta.com'
+    test = 'https://test.qmenta.com'
+    test_insecure = 'http://test.qmenta.com'
+    local_ip = "http://127.0.0.1:8080"
+    localhost = "http://localhost:8080"
 
 
 class Auth:
@@ -287,7 +288,7 @@ def validate_url(url: str) -> None:
         )
 
     url_values = [p_url.value for p_url in PlatformURL]
-    if parsed_url.netloc not in url_values:
+    if parsed_url.scheme + '://' + parsed_url.netloc not in url_values:
         raise ValueError(
             "base_url must be one of '{}', ".format("', '".join(url_values))
             + f"not '{parsed_url.netloc}'."

{qmenta_core-4.1.dev709 → qmenta_core-4.1.dev711}/src/qmenta/core/platform.py

@@ -138,7 +138,7 @@ def parse_response(response: requests.Response) -> Any:
 def post(
     auth: Auth, endpoint: str, data: Dict[str, Any] = {},
     headers: Dict[str, Any] = {}, stream: bool = False,
-    timeout: float = 30.0
+    timeout: float = 60.0
 ) -> requests.Response:
     """
     Post the given data and headers to the specified platform's endpoint.

{qmenta_core-4.1.dev709 → qmenta_core-4.1.dev711}/src/qmenta/core/upload/single.py

@@ -95,6 +95,11 @@ class FileInfo:
         if self.is_result and not self.input_data_type:
             self.input_data_type = "offline_analysis:1.0"
 
+        if self.input_data_type and ":" not in self.input_data_type:
+            raise ValueError(
+                "If input_data_type is provided it must include its version"
+            )
+
         if self.split_data and self.session_id:
             # Don't split data when session_id is set
             self.split_data = False
@@ -148,6 +153,12 @@ class SingleUpload:
         after the upload finished. When set to False, the ``qm-*.zip`` files
         that were created by this SingleUpload will be deleted when the upload
         if finished. Default value: True.
+    chunk_size: int
+        The maximum size in bytes of each chunk of file to upload.
+        Default value: 2**24 == 16 MiB (optimized for GCS Bucket Storage)
+    max_retries: int
+        Maximum number of retries when uploading a file before raising
+        an error. Default value: 5
 
     Attributes
     ----------
@@ -186,12 +197,15 @@ class SingleUpload:
         anonymise: bool = True,
         upload_index: int = -1,
         keep_created_files: bool = True,
+        chunk_size: int = 2**24,  # 16 MiB
+        max_retries: int = 5
     ) -> None:
         self.auth = auth
         self.path = path
         self.file_info = file_info
         self.upload_index = upload_index
         self.keep_created_files = keep_created_files
+        self.max_retries = max_retries
         self._created_files_list: List[str] = []
 
         # file_size will be set in _check_type_and_size if the original file
@@ -206,7 +220,7 @@ class SingleUpload:
         self.upload_filename: str
 
         # Optimized for GCS Bucket Storage
-        self._chunk_size = 2**24  # 16 MiB
+        self._chunk_size = chunk_size
 
         if os.path.isfile(self.path):
             self.input_filename = self.path
@@ -472,6 +486,16 @@ class SingleUpload:
         UploadError
         platform.ConnectionError
         """
+        def iterate_over_file() -> Generator[Any, Any, Any]:
+            with open(self.upload_filename, "rb") as stream:
+                while True:
+                    contents = stream.read(self._chunk_size)
+                    if not contents:
+                        break
+                    else:
+                        yield contents
+                        self.bytes_uploaded += len(contents)
+
         assert self.status == UploadStatus.TO_UPLOAD
         self.status = UploadStatus.UPLOADING
         self.file_size = prepare.get_zipfile_size(self.upload_filename)
@@ -495,7 +519,6 @@ class SingleUpload:
                 "_pid": self.file_info.project_id,
             },
         )
-
         url_response_data = platform.parse_response(url_request_response)
         signed_url: str = url_response_data["url"]
         container_id: str = url_response_data["container_id"]
@@ -503,27 +526,14 @@ class SingleUpload:
         md5_hash: str = self._file_md5(self.upload_filename)
 
         # Stream file to signed-url without loading it to memory
-        max_retries: int = 5
         completed_upload: bool = False
         retries_count: int = 0
-        while not completed_upload and retries_count < max_retries:
-
-            def iterate_over_file() -> Generator[Any, Any, Any]:
-                with open(self.upload_filename, "rb") as stream:
-                    while True:
-                        contents = stream.read(self._chunk_size)
-                        if not contents:
-                            break
-                        else:
-                            yield contents
-                            self.bytes_uploaded += len(contents)
-
+        while not completed_upload and retries_count < self.max_retries:
             upload_response = post(
                 url=signed_url,
                 data=iterate_over_file(),
                 stream=True
             )
-
             # Verify upload
             try:
                 self._validate_upload_response(
@@ -531,7 +541,7 @@ class SingleUpload:
                 )
             except UploadError:
                 retries_count += 1
-                if retries_count < max_retries:
+                if retries_count < self.max_retries:
                     wait_time: int = 2 ** retries_count
                     time.sleep(wait_time)
                     print(
@@ -549,6 +559,11 @@ class SingleUpload:
                 )
             )
 
+        tool_type = None
+        tool_version = None
+        if self.file_info.input_data_type:
+            tool_type, tool_version = self.file_info.input_data_type.split(":")
+
         verification_response = platform.post(
             auth=self.auth,
             endpoint="file_manager/verify_upload_via_url",
@@ -556,6 +571,8 @@ class SingleUpload:
                 "container_id": container_id,
                 "path": fname,
                 "md5": md5_hash,
+                "tool_type": tool_type,
+                "tool_version": tool_version,
             },
         )
         platform.parse_response(verification_response)