pma-python 2.0.0.179__tar.gz → 2.0.0.181__tar.gz

{pma_python-2.0.0.179 → pma_python-2.0.0.181}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pma_python
-Version: 2.0.0.179
+Version: 2.0.0.181
 Summary: Universal viewing of digital microscopy, whole slide imaging and digital pathology data
 Home-page: https://github.com/pathomation/pma_python
 Author: Pathomation

{pma_python-2.0.0.179 → pma_python-2.0.0.181}/pma_python/core.py

@@ -18,7 +18,7 @@ import pandas as pd
 import requests
 from requests_toolbelt.multipart.encoder import MultipartEncoder, MultipartEncoderMonitor
 
-from .pma_core_client import PmaCoreClient, UploadHeaderModel, UploadFileModel
+from .pma_core_client import PmaCoreClient, UploadHeaderModel, UploadFileModel, UploadResponse
 from typing import Callable
 
 ProgressCallback = Callable[[int, int], None]
@@ -1532,119 +1532,6 @@ def _pma_upload_amazon_callback(bytes_read, total_size, previous, filename):
         return v
 
 
-def upload(local_source_slide, target_folder, target_pma_core_sessionID, callback=None, verify=True):
-    """
-        Uploads a slide to a PMA.core server. Requires a PMA.start installation
-        :param str local_source_slide: The local PMA.start relative file to upload
-        :param str target_folder: The root directory and path to upload to the PMA.core server
-        :param str target_pma_core_sessionID: A valid session id for a PMA.core server
-        :param function|boolean callback: If True a default progress will be printed.
-               If a function is passed it will be called for progress on each file upload.
-               The function has the following signature:
-                        `callback(bytes_read, bytes_length, filename)`
-    """
-    if not _pma_is_lite():
-        raise Exception(
-            "No PMA.start found on localhost. Are you sure it is running?")
-
-    if not target_folder:
-        raise ValueError("target_folder cannot be empty")
-
-    if (target_folder.startswith("/")):
-        target_folder = target_folder[1:]
-
-    files = get_files_for_slide(local_source_slide, _pma_pmacoreliteSessionID)
-    sessionID = _pma_session_id(target_pma_core_sessionID)
-    url = _pma_url(sessionID) + "transfer/Upload?sessionID=" + pma._pma_q(sessionID)
-
-    mainDirectory = ''
-    for i, f in enumerate(files):
-        md = os.path.dirname(f)
-        if i == 0 or len(md) < len(mainDirectory):
-            mainDirectory = md
-
-    uploadFiles = []
-    for i, filepath in enumerate(files):
-        s = os.path.getsize(filepath)
-        if s > 0:
-            uploadFiles.append({
-                "Path": filepath.replace(mainDirectory, '').strip("\\").strip('/'),
-                "Length": s,
-                "IsMain": i == len(files) - 1,
-                "FullPath": filepath
-            })
-
-    data = {"Path": target_folder, "Files": uploadFiles}
-
-    uploadHeaderResponse = requests.post(url, json=data, verify=verify)
-    if not uploadHeaderResponse.status_code == 200:
-        print(uploadHeaderResponse.json())
-        raise Exception(uploadHeaderResponse.json()["Message"])
-
-    uploadHeader = uploadHeaderResponse.json()
-
-    pmaCoreUploadUrl = _pma_url(sessionID) + "transfer/Upload/" + pma._pma_q(
-        uploadHeader["Id"]) + "?sessionID=" + pma._pma_q(sessionID) + "&path={0}"
-
-    isAmazonUpload = True
-    if not uploadHeader['Urls']:
-        isAmazonUpload = False
-        uploadHeader['Urls'] = [pmaCoreUploadUrl.format(
-            f["Path"]) for f in uploadFiles]
-
-    for i, f in enumerate(uploadFiles):
-        uploadUrl = uploadHeader['Urls'][i]
-
-        e = MultipartEncoder(
-            fields={"file": (os.path.basename(f["Path"]), open(f["FullPath"], 'rb'), 'application/octet-stream')})
-
-        _callback = None
-        if callback is True:
-            print("Uploading file: {0}".format(e.fields["file"][0]))
-            if not isAmazonUpload:
-                def _callback(x):
-                    return _pma_upload_callback(
-                        monitor, e.fields["file"][0])
-            else:
-                def _callback(bytes_read, total_size, previous):
-                    return _pma_upload_amazon_callback(
-                        bytes_read, total_size, previous, e.fields["file"][0])
-        elif callable(callback):
-            def _callback(x):
-                return callback(
-                    x.bytes_read, x.len, x.previous, e.fields["file"][0])
-
-        monitor = MultipartEncoderMonitor(e, _callback)
-
-        monitor.previous = 0
-
-        r = None
-        if not isAmazonUpload:
-            r = requests.post(uploadUrl, data=monitor, headers={
-                'Content-Type': monitor.content_type},
-                              verify=verify)
-        else:
-            headers = {'Content-Length': str(f["Length"])}
-            if uploadHeader['UploadType'] == 2:
-                headers = {
-                    'Content-Length': str(f["Length"]), 'x-ms-blob-type': 'BlockBlob'}
-
-            r = requests.put(uploadUrl, data=UploadChunksIterator(
-                open(f["FullPath"], 'rb'), f["Path"], f["Length"], _callback), headers=headers, verify=verify)
-
-        if r.status_code < 200 or r.status_code >= 300:
-            raise Exception("Error uploading file {0}: {1} \r\n{2}: {3}".format(
-                f["Path"], uploadUrl, r.status_code, r.text))
-
-        uploadFinalizeResponse = requests.get(_pma_url(sessionID) + "transfer/Upload/"
-                                              + pma._pma_q(uploadHeader["Id"]) + "?sessionID=" + pma._pma_q(sessionID),
-                                              verify=verify)
-        if uploadFinalizeResponse.status_code < 200 or uploadFinalizeResponse.status_code >= 300:
-            print(uploadFinalizeResponse.json())
-            raise Exception(uploadFinalizeResponse.json()[
-                                "Message"] + uploadFinalizeResponse.json()["ExceptionMessage"])
-
-
 class UploadChunksIterator:
     def __init__(self, file: io.BufferedReader, filename, total_size: int, callback, chunk_size: int = 16 * 1024):
         self.file = file
@@ -1663,11 +1550,10 @@ class UploadChunksIterator:
 
         if not data:
             raise StopIteration
-        self.bytes_read += self.chunk_size
-        if self.callback is not None:
-            v = self.callback(self.bytes_read, self.total_size, self.previous)
-            if v is not None:
-                self.previous = v
+        self.bytes_read += len(data)
+        if self.callback:
+            self.callback(self.bytes_read, self.total_size)
+
         return data
 
     def __len__(self):
@@ -2008,10 +1894,256 @@ def get_annotation_distance(slideRef, layerID, annotationID, sessionID=None, ver
     return r.text
 
 
-#********************************#
-#   === Large Files Uploader === #
-#********************************#
-async def upload_large_files(
+# **************************************#
+#           === Uploader ===            #
+# **************************************#
+FIVE_GB = 5 * 1024 * 1024 * 1024
+async def upload(
+        *,
+        pma_core_url: str,
+        session_id: str,
+        slide_path: str,
+        upload_directory: str,
+        progress_callback=None,
+):
+    """
+    Main upload entry point.
+
+    Automatically:
+    - calculates full slide size (file + folder + nested files)
+    - detects server upload type
+    - routes upload to legacy transfer or large file uploader
+    """
+
+    if not os.path.isfile(slide_path):
+        return False, f"File not found: {slide_path}"
+
+    # ================================
+    # CALCULATE TOTAL SLIDE SIZE
+    # ================================
+    total_size = 0
+
+    # main file size
+    total_size += os.stat(slide_path).st_size
+
+    # possible slide folder (multi-files slides)
+    slide_name_no_ext = os.path.splitext(os.path.basename(slide_path))[0]
+    slide_dir = os.path.join(os.path.dirname(slide_path), slide_name_no_ext)
+
+    if os.path.isdir(slide_dir):
+        for root, _, files in os.walk(slide_dir):
+            for f in files:
+                total_size += os.stat(os.path.join(root, f)).st_size
+
+    print("Total slide size:", round(total_size / (1024 ** 3), 2), "GB")
+
+    # ================================
+    # CHECK UPLOAD TYPE (PRE-FLIGHT)
+    # ================================
+
+    try:
+        client = PmaCoreClient(pma_core_url)
+
+        file_size = os.stat(slide_path).st_size
+        file_name = os.path.basename(slide_path)
+
+        header = UploadHeaderModel(
+            path=upload_directory,
+            files=[
+                UploadFileModel(
+                    isMain=True,
+                    length=file_size,
+                    path=file_name
+                )
+            ]
+        )
+
+        resp = await client.upload_header(header, session_id)
+
+        print("Detected UploadType:", resp.upload_type)
+
+        # ================================
+        # ROUTING LOGIC
+        # ================================
+
+        # UploadType 0 → always small uploader
+        if resp.upload_type == 0:
+            print("UploadType 0 → forcing SMALL uploader")
+
+            return upload_legacy(
+                session_id=session_id,
+                slide_path=slide_path,
+                upload_directory=upload_directory,
+                progress_callback=progress_callback
+            )
+
+        # Otherwise fallback to size logic
+        if total_size < FIVE_GB:
+            print("Using SMALL file uploader")
+
+            return upload_legacy(
+                session_id=session_id,
+                slide_path=slide_path,
+                upload_directory=upload_directory,
+                progress_callback=progress_callback
+            )
+
+        else:
+            print("Using LARGE file uploader")
+
+            return await upload_file_over_5gb(
+                pma_core_url=pma_core_url,
+                session_id=session_id,
+                slide_path=slide_path,
+                upload_directory=upload_directory,
+                progress_callback=progress_callback
+            )
+
+    except Exception as e:
+        return False, str(e)
+
+# **************************************#
+#   === Small Files < 5bg Uploader  === #
+# **************************************#
+def upload_legacy(slide_path, upload_directory, session_id, progress_callback=None, verify=True):
+    """
+    Synchronous upload implementation using direct HTTP requests.
+
+    Handles:
+    - filesystem uploads (any file size when UploadType == 0)
+    - non-chunked cloud uploads
+    - single-file and multi-file slides (MRXS, VSI, etc.)
+    - global progress aggregation across all uploaded files
+    """
+
+    if not upload_directory:
+        raise ValueError("target_folder cannot be empty")
+
+    if (upload_directory.startswith("/")):
+        upload_directory = upload_directory[1:]
+
+    files = get_files_for_slide(slide_path, _pma_pmacoreliteSessionID)
+    sessionID = _pma_session_id(session_id)
+    url = _pma_url(sessionID) + "transfer/Upload?sessionID=" + pma._pma_q(sessionID)
+
+    mainDirectory = ''
+    for i, f in enumerate(files):
+        md = os.path.dirname(f)
+        if i == 0 or len(md) < len(mainDirectory):
+            mainDirectory = md
+
+    uploadFiles = []
+    for i, filepath in enumerate(files):
+        s = os.path.getsize(filepath)
+        if s > 0:
+            uploadFiles.append({
+                "Path": filepath.replace(mainDirectory, '').strip("\\").strip('/'),
+                "Length": s,
+                "IsMain": i == len(files) - 1,
+                "FullPath": filepath
+            })
+            # ==============================
+            # GLOBAL PROGRESS (ALL FILES)
+            # ==============================
+            total_bytes = sum(x["Length"] for x in uploadFiles) if uploadFiles else 0
+            sent_total = 0
+
+            # If the user passed progress_callback=True → enable the default print
+            if progress_callback is True:
+                def progress_callback(sent: int, total: int):
+                    if total > 0:
+                        percent = sent / total * 100
+                        print(f"Upload progress: {percent:.2f}%")
+
+            def report_delta(delta: int):
+                nonlocal sent_total
+                if delta <= 0:
+                    return
+                sent_total += delta
+                if callable(progress_callback) and total_bytes > 0:
+                    if sent_total > total_bytes:
+                        sent_total = total_bytes
+                    progress_callback(sent_total, total_bytes)
+
+    data = {"Path": upload_directory, "Files": uploadFiles}
+
+    uploadHeaderResponse = requests.post(url, json=data, verify=verify)
+    if not uploadHeaderResponse.status_code == 200:
+        print(uploadHeaderResponse.json())
+        raise Exception(uploadHeaderResponse.json()["Message"])
+
+    uploadHeader = uploadHeaderResponse.json()
+
+    pmaCoreUploadUrl = _pma_url(sessionID) + "transfer/Upload/" + pma._pma_q(
+        uploadHeader["Id"]) + "?sessionID=" + pma._pma_q(sessionID) + "&path={0}"
+
+    isAmazonUpload = True
+    if not uploadHeader['Urls']:
+        isAmazonUpload = False
+        uploadHeader['Urls'] = [pmaCoreUploadUrl.format(
+            f["Path"]) for f in uploadFiles]
+
+    for i, f in enumerate(uploadFiles):
+        uploadUrl = uploadHeader['Urls'][i]
+
+        e = MultipartEncoder(
+            fields={"file": (os.path.basename(f["Path"]), open(f["FullPath"], 'rb'), 'application/octet-stream')})
+
+        # ==============================
+        # PER-FILE -> GLOBAL PROGRESS
+        # ==============================
+        last_sent = 0
+
+        def monitor_callback(monitor):
+            nonlocal last_sent
+            current = monitor.bytes_read
+            delta = current - last_sent
+            last_sent = current
+            report_delta(delta)
+
+        def iterator_callback(bytes_read, _total_size):
+            nonlocal last_sent
+            current = bytes_read
+            delta = current - last_sent
+            last_sent = current
+            report_delta(delta)
+
+        monitor = MultipartEncoderMonitor(e, monitor_callback)
+
+        monitor.previous = 0
+
+        r = None
+        if not isAmazonUpload:
+            r = requests.post(uploadUrl, data=monitor, headers={
+                'Content-Type': monitor.content_type},
+                              verify=verify)
+        else:
+            headers = {'Content-Length': str(f["Length"])}
+            if uploadHeader['UploadType'] == 2:
+                headers = {
+                    'Content-Length': str(f["Length"]), 'x-ms-blob-type': 'BlockBlob'}
+
+            r = requests.put(uploadUrl, data=UploadChunksIterator(
+                open(f["FullPath"], 'rb'), f["Path"], f["Length"], iterator_callback), headers=headers, verify=verify)
+
+        if r.status_code < 200 or r.status_code >= 300:
+            raise Exception("Error uploading file {0}: {1} \r\n{2}: {3}".format(
+                f["Path"], uploadUrl, r.status_code, r.text))
+
+        uploadFinalizeResponse = requests.get(_pma_url(sessionID) + "transfer/Upload/"
+                                              + pma._pma_q(uploadHeader["Id"]) + "?sessionID=" + pma._pma_q(sessionID),
+                                              verify=verify)
+        if uploadFinalizeResponse.status_code < 200 or uploadFinalizeResponse.status_code >= 300:
+            print(uploadFinalizeResponse.json())
+            raise Exception(uploadFinalizeResponse.json()[
+                                "Message"] + uploadFinalizeResponse.json()["ExceptionMessage"])
+    return True, None
+
+
+# **************************************#
+#   === Large Files > 5gb Uploader === #
+# **************************************#
+async def upload_file_over_5gb(
         *,
         pma_core_url: str,
         session_id: str,
@@ -2019,11 +2151,18 @@ async def upload_large_files(
         upload_directory: str,
         progress_callback=None,
 ):
+    """
+    Asynchronous large-file upload implementation using PmaCoreClient.
+
+    Handles:
+    - chunked cloud uploads
+    - multi-file slide stacks
+    - global progress tracking across all uploaded files
+    """
 
     # ===== GETTING ALL FILES =====
     all_files = []
 
-    # MAIN VSI
     all_files.append(slide_path)
 
     vsi_name_no_ext = os.path.splitext(os.path.basename(slide_path))[0]
@@ -2073,7 +2212,7 @@ async def upload_large_files(
     )
 
     if not ok:
-        return False, f"VSI upload failed: {err}"
+        return False, f"Upload failed: {err}"
 
     # ===== STACK =====
     if os.path.isdir(local_stack_root):
@@ -2105,6 +2244,7 @@ async def upload_large_files(
 
     return True, None
 
+
 async def large_files_upload_package(
         *,
         pma_core_url: str,
@@ -2113,6 +2253,15 @@ async def large_files_upload_package(
         upload_directory: str,
         progress_callback=None,
 ) -> tuple[bool, str | None]:
+    """
+    Low-level async upload routine for a single file.
+
+    Performs:
+    - upload header request
+    - multipart or direct upload
+    - upload completion polling
+    """
+
     try:
         client = PmaCoreClient(pma_core_url)
         print("trust_env:", client.session.trust_env)
@@ -2138,11 +2287,11 @@ async def large_files_upload_package(
         # ===== SEND HEADER =====
         resp = await client.upload_header(header, session_id)
 
-        print("UploadType:", resp.upload_type)
+        # print("UploadType:", resp.upload_type)
         print("MultipartFiles:", resp.multipart_files)
         print("Urls:", resp.urls)
 
-        # ===== MULTIPART =====
+        # ===== MULTIPART ONLY IF SERVER SENT PARTS =====
         if resp.multipart_files and len(resp.multipart_files) > 0:
 
             mp = resp.multipart_files[0]
@@ -2152,7 +2301,6 @@ async def large_files_upload_package(
             )
 
             with open(slide_path, "rb") as stream:
-
                 await client.upload_multipart_file_to_s3(
                     mp,
                     full_virtual_path,
@@ -2161,13 +2309,12 @@ async def large_files_upload_package(
                     progress_callback=progress_callback
                 )
 
-        # ===== SINGLE PUT =====
+        # ===== SINGLE PUT / AZURE / FILESYSTEM =====
         else:
 
             upload_url = resp.urls[0] if resp.urls else None
 
             with open(slide_path, "rb") as stream:
-
                 await client.upload_file(
                     resp.id,
                     resp.upload_type,
@@ -2193,7 +2340,14 @@ async def large_files_upload_package(
     except Exception as e:
         return False, str(e)
 
+
 # callback for large files upload
 def on_progress(bytes_sent, total_bytes):
+    """
+    Default progress callback.
+
+    Prints upload percentage based on transferred bytes.
+    """
+
     percent = bytes_sent / total_bytes * 100
     print(f"Upload progress: {percent:.2f}%")

{pma_python-2.0.0.179 → pma_python-2.0.0.181}/pma_python/pma_core_client.py

@@ -11,12 +11,10 @@ from urllib3.poolmanager import PoolManager
 
 
 UploadProgressCallback = Callable[[int, int], None]
-# args: bytes_sent, total_bytes
 
 class TLS12HttpAdapter(HTTPAdapter):
     def init_poolmanager(self, connections, maxsize, block=False, **pool_kwargs):
         ctx = ssl.create_default_context()
-        # Принудительно TLS 1.2 (часто лечит EOF на прокси/антивирусе)
         ctx.minimum_version = ssl.TLSVersion.TLSv1_2
         ctx.maximum_version = ssl.TLSVersion.TLSv1_2
 
@@ -176,12 +174,10 @@ class PmaCoreClient:
         if not self.server_url.endswith('/'):
             self.server_url += '/'
 
-        # --- FIX: не использовать proxy/SSL-intercept из окружения ---
         self.session = requests.Session()
-        self.session.trust_env = False  # <- ключевое: игнорировать HTTP(S)_PROXY и т.п.
+        self.session.trust_env = False
         self.session.mount("https://", TLS12HttpAdapter())
 
-
     def authenticate(self, username: str, password: str, caller: str) -> AuthenticateResponse:
         service_url = urljoin(self.server_url, 'api/json/Authenticate')
         params = {'username': username, 'password': password, 'caller': caller}
@@ -209,8 +205,13 @@ class PmaCoreClient:
                           total_bytes: int | None = None,
                           progress_callback: UploadProgressCallback | None = None):
 
-        if upload_type == "Azure":
-            await self.upload_file_to_azure(upload_url, stream)
+        if upload_type == "Azure" or upload_type == 2:
+            await self.upload_file_to_azure(
+                upload_url,
+                stream,
+                total_bytes=total_bytes,
+                progress_callback=progress_callback,
+            )
             return
 
         upload_to_presigned = bool(upload_url)
@@ -248,11 +249,9 @@ class PmaCoreClient:
 
             return
 
-        # upload через сервер (multipart form)
         url = f"{self.server_url}transfer/Upload/{upload_id}"
         params = {"sessionId": session_id, "path": relative_path}
 
-        # важно: wrapped передаём как file handle в files
         files = {"file": (relative_path.split("/")[-1], wrapped, "application/octet-stream")}
         response = await asyncio.to_thread(
             requests.post,
@@ -285,7 +284,7 @@ class PmaCoreClient:
         params = {'sessionId': session_id}
         payload = {
             "FilePath": relative_path,
-            "UploadId": multipart_info.upload_id,  # см. пункт 6 ниже
+            "UploadId": multipart_info.upload_id,
             "Parts": [{"PartNumber": p.PartNumber, "ETag": p.ETag} for p in e_tags],
         }
         headers = {'Content-Type': 'application/json'}
@@ -299,9 +298,21 @@ class PmaCoreClient:
         )
         response.raise_for_status()
 
-    async def upload_file_to_azure(self, upload_url: str, stream: IO):
+    async def upload_file_to_azure(
+            self,
+            upload_url: str,
+            stream: IO,
+            total_bytes: int | None = None,
+            progress_callback: UploadProgressCallback | None = None,
+    ):
         block_ids: List[str] = []
-        await self.upload_blocks_to_azure(block_ids, upload_url, stream)
+        await self.upload_blocks_to_azure(
+            block_ids,
+            upload_url,
+            stream,
+            total_bytes=total_bytes,
+            progress_callback=progress_callback,
+        )
         headers = {
             'x-ms-blob-content-type': 'application/octet-stream',
             'Connection': 'Keep-Alive',
@@ -321,11 +332,17 @@ class PmaCoreClient:
         response = requests.put(commit_url, data=body, headers=headers)
         response.raise_for_status()
 
-    async def upload_blocks_to_azure(self, block_ids: List[str], upload_url: str, stream: IO):
-        block_size = 4 * 1024 * 1024  # 4MB как в Java
+    async def upload_blocks_to_azure(
+            self,
+            block_ids: List[str],
+            upload_url: str,
+            stream: IO,
+            total_bytes: int | None = None,
+            progress_callback: UploadProgressCallback | None = None,
+    ):
+        block_size = 4 * 1024 * 1024
         block_id = 0
-
-        # если в upload_url есть пробелы/и т.п.
+        sent_total = 0
         safe_base = upload_url.replace(" ", "%20")
 
         while True:
@@ -333,21 +350,19 @@ class PmaCoreClient:
             if not chunk:
                 break
 
+            sent_total += len(chunk)
+            if progress_callback and total_bytes:
+                progress_callback(sent_total, total_bytes)
+
             plain = (f"block_{block_id:06d}").encode("utf-8")
             import base64
             base64_block_id = base64.b64encode(plain).decode("utf-8")
-
-            # ВАЖНО: blockid обязательно URL-encode
             encoded_block_id = quote(base64_block_id, safe="")
-
             block_upload_url = f"{safe_base}&comp=block&blockid={encoded_block_id}"
 
-            # Для block PUT не надо x-ms-blob-type (особенно если URL — SAS)
             headers = {
                 "Connection": "Keep-Alive",
                 "Keep-Alive": "3600",
-                # при необходимости можно добавить x-ms-version, если требует аккаунт:
-                # "x-ms-version": "2020-10-02",
             }
 
             response = await asyncio.to_thread(
@@ -358,7 +373,7 @@ class PmaCoreClient:
             )
             response.raise_for_status()
 
-            block_ids.append(base64_block_id)  # В список блоков кладём original base64 id (как в Java)
+            block_ids.append(base64_block_id)
             block_id += 1
 
     def _iter_file_range(self, stream: IO, start: int, length: int, buf_size: int = 8 * 1024 * 1024):

pma_python-2.0.0.181/pma_python/version.py

@@ -0,0 +1 @@
+__version__ = '2.0.0.181'

{pma_python-2.0.0.179 → pma_python-2.0.0.181}/pma_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pma_python
-Version: 2.0.0.179
+Version: 2.0.0.181
 Summary: Universal viewing of digital microscopy, whole slide imaging and digital pathology data
 Home-page: https://github.com/pathomation/pma_python
 Author: Pathomation

pma_python-2.0.0.179/pma_python/version.py

@@ -1 +0,0 @@
-__version__ = '2.0.0.179'