pma-python 2.0.0.184__tar.gz → 2.0.0.185__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pma_python-2.0.0.184 → pma_python-2.0.0.185}/PKG-INFO +1 -1
- {pma_python-2.0.0.184 → pma_python-2.0.0.185}/pma_python/core.py +298 -206
- {pma_python-2.0.0.184 → pma_python-2.0.0.185}/pma_python/pma_core_client.py +65 -34
- pma_python-2.0.0.185/pma_python/version.py +1 -0
- {pma_python-2.0.0.184 → pma_python-2.0.0.185}/pma_python.egg-info/PKG-INFO +1 -1
- pma_python-2.0.0.184/pma_python/version.py +0 -1
- {pma_python-2.0.0.184 → pma_python-2.0.0.185}/README.md +0 -0
- {pma_python-2.0.0.184 → pma_python-2.0.0.185}/pma_python/__init__.py +0 -0
- {pma_python-2.0.0.184 → pma_python-2.0.0.185}/pma_python/control.py +0 -0
- {pma_python-2.0.0.184 → pma_python-2.0.0.185}/pma_python/core_admin.py +0 -0
- {pma_python-2.0.0.184 → pma_python-2.0.0.185}/pma_python/pma.py +0 -0
- {pma_python-2.0.0.184 → pma_python-2.0.0.185}/pma_python/view.py +0 -0
- {pma_python-2.0.0.184 → pma_python-2.0.0.185}/pma_python.egg-info/SOURCES.txt +0 -0
- {pma_python-2.0.0.184 → pma_python-2.0.0.185}/pma_python.egg-info/dependency_links.txt +0 -0
- {pma_python-2.0.0.184 → pma_python-2.0.0.185}/pma_python.egg-info/not-zip-safe +0 -0
- {pma_python-2.0.0.184 → pma_python-2.0.0.185}/pma_python.egg-info/requires.txt +0 -0
- {pma_python-2.0.0.184 → pma_python-2.0.0.185}/pma_python.egg-info/top_level.txt +0 -0
- {pma_python-2.0.0.184 → pma_python-2.0.0.185}/setup.cfg +0 -0
- {pma_python-2.0.0.184 → pma_python-2.0.0.185}/setup.py +0 -0
|
@@ -14,6 +14,7 @@ import os
|
|
|
14
14
|
import datetime
|
|
15
15
|
import io
|
|
16
16
|
import re
|
|
17
|
+
import traceback
|
|
17
18
|
import pandas as pd
|
|
18
19
|
import requests
|
|
19
20
|
from requests_toolbelt.multipart.encoder import MultipartEncoder, MultipartEncoderMonitor
|
|
@@ -1901,30 +1902,256 @@ def get_annotation_distance(slideRef, layerID, annotationID, sessionID=None, ver
|
|
|
1901
1902
|
# **************************************#
|
|
1902
1903
|
# === Uploader === #
|
|
1903
1904
|
# **************************************#
|
|
1904
|
-
|
|
1905
|
-
|
|
1905
|
+
def _normalize_upload_directory(upload_directory: str) -> str:
|
|
1906
|
+
if upload_directory is None:
|
|
1907
|
+
raise ValueError("target_folder cannot be empty")
|
|
1908
|
+
normalized = upload_directory.strip()
|
|
1909
|
+
if normalized == "/":
|
|
1910
|
+
return ""
|
|
1911
|
+
normalized = normalized.lstrip("/")
|
|
1912
|
+
return normalized.rstrip("/")
|
|
1913
|
+
|
|
1914
|
+
|
|
1915
|
+
def _join_virtual_path(parent: str, child: str) -> str:
|
|
1916
|
+
parent = (parent or "").strip("/")
|
|
1917
|
+
child = (child or "").lstrip("/")
|
|
1918
|
+
return f"{parent}/{child}" if parent else child
|
|
1906
1919
|
|
|
1907
|
-
def _get_slide_package_size(slide_path: str) -> tuple[int, int]:
|
|
1908
|
-
"""Return total bytes and maximum single-file bytes for a slide package."""
|
|
1909
|
-
total_size = 0
|
|
1910
|
-
max_size = 0
|
|
1911
1920
|
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1921
|
+
def _normalize_upload_type(upload_type):
|
|
1922
|
+
if isinstance(upload_type, str):
|
|
1923
|
+
return upload_type.strip().lower()
|
|
1924
|
+
return upload_type
|
|
1915
1925
|
|
|
1916
|
-
|
|
1917
|
-
|
|
1926
|
+
|
|
1927
|
+
def _is_upload_type(upload_type, *candidates) -> bool:
|
|
1928
|
+
normalized = _normalize_upload_type(upload_type)
|
|
1929
|
+
for candidate in candidates:
|
|
1930
|
+
if isinstance(candidate, str):
|
|
1931
|
+
if normalized == candidate.lower():
|
|
1932
|
+
return True
|
|
1933
|
+
elif normalized == candidate:
|
|
1934
|
+
return True
|
|
1935
|
+
return False
|
|
1936
|
+
|
|
1937
|
+
|
|
1938
|
+
def _collect_slide_package_files(slide_path: str):
|
|
1939
|
+
if not os.path.isfile(slide_path):
|
|
1940
|
+
raise ValueError(f"File not found: {slide_path}")
|
|
1941
|
+
|
|
1942
|
+
main_name = os.path.basename(slide_path)
|
|
1943
|
+
child_folder = os.path.splitext(main_name)[0]
|
|
1944
|
+
stack_root = os.path.join(os.path.dirname(slide_path), child_folder)
|
|
1945
|
+
|
|
1946
|
+
package_files = [{
|
|
1947
|
+
"local_path": slide_path,
|
|
1948
|
+
"relative_path": main_name,
|
|
1949
|
+
"is_main": True,
|
|
1950
|
+
"size": os.stat(slide_path).st_size
|
|
1951
|
+
}]
|
|
1952
|
+
|
|
1953
|
+
if os.path.isdir(stack_root):
|
|
1954
|
+
for root, dirnames, filenames in os.walk(stack_root):
|
|
1955
|
+
dirnames.sort()
|
|
1956
|
+
filenames.sort()
|
|
1918
1957
|
for filename in filenames:
|
|
1919
|
-
|
|
1958
|
+
local_path = os.path.join(root, filename)
|
|
1959
|
+
rel_inside_stack = os.path.relpath(local_path, stack_root).replace("\\", "/")
|
|
1960
|
+
package_files.append({
|
|
1961
|
+
"local_path": local_path,
|
|
1962
|
+
"relative_path": f"{child_folder}/{rel_inside_stack}",
|
|
1963
|
+
"is_main": False,
|
|
1964
|
+
"size": os.stat(local_path).st_size
|
|
1965
|
+
})
|
|
1920
1966
|
|
|
1921
|
-
|
|
1922
|
-
size = os.stat(filepath).st_size
|
|
1923
|
-
total_size += size
|
|
1924
|
-
max_size = max(max_size, size)
|
|
1967
|
+
return package_files
|
|
1925
1968
|
|
|
1969
|
+
|
|
1970
|
+
def _build_upload_urls(server_url: str, session_id: str, upload_id: int, package_files) -> list[str]:
|
|
1971
|
+
base = server_url if server_url.endswith("/") else server_url + "/"
|
|
1972
|
+
encoded_session = pma._pma_q(session_id)
|
|
1973
|
+
urls = []
|
|
1974
|
+
for file in package_files:
|
|
1975
|
+
encoded_path = pma._pma_q(file["relative_path"])
|
|
1976
|
+
urls.append(f"{base}transfer/Upload/{upload_id}?sessionId={encoded_session}&path={encoded_path}")
|
|
1977
|
+
return urls
|
|
1978
|
+
|
|
1979
|
+
|
|
1980
|
+
def _get_slide_package_size(slide_path: str) -> tuple[int, int]:
|
|
1981
|
+
"""Return total bytes and maximum single-file bytes for a slide package."""
|
|
1982
|
+
files = _collect_slide_package_files(slide_path)
|
|
1983
|
+
total_size = sum(file["size"] for file in files)
|
|
1984
|
+
max_size = max(file["size"] for file in files) if files else 0
|
|
1926
1985
|
return total_size, max_size
|
|
1927
1986
|
|
|
1987
|
+
|
|
1988
|
+
async def _poll_image_info(
|
|
1989
|
+
slide_ref: str,
|
|
1990
|
+
session_id: str,
|
|
1991
|
+
verify=True,
|
|
1992
|
+
max_attempts: int = 5,
|
|
1993
|
+
interval_seconds: float = 1.0,
|
|
1994
|
+
) -> bool:
|
|
1995
|
+
for attempt in range(max_attempts):
|
|
1996
|
+
try:
|
|
1997
|
+
await asyncio.to_thread(
|
|
1998
|
+
get_slide_info,
|
|
1999
|
+
slide_ref,
|
|
2000
|
+
sessionID=session_id,
|
|
2001
|
+
verify=verify
|
|
2002
|
+
)
|
|
2003
|
+
return True
|
|
2004
|
+
except Exception:
|
|
2005
|
+
if attempt >= max_attempts - 1:
|
|
2006
|
+
return False
|
|
2007
|
+
await asyncio.sleep((attempt + 1) * interval_seconds)
|
|
2008
|
+
|
|
2009
|
+
return False
|
|
2010
|
+
|
|
2011
|
+
|
|
2012
|
+
async def _upload_package_files(
|
|
2013
|
+
*,
|
|
2014
|
+
client: PmaCoreClient,
|
|
2015
|
+
session_id: str,
|
|
2016
|
+
upload_directory: str,
|
|
2017
|
+
package_files,
|
|
2018
|
+
progress_callback=None,
|
|
2019
|
+
verify=True,
|
|
2020
|
+
) -> tuple[bool, str | None]:
|
|
2021
|
+
try:
|
|
2022
|
+
header = UploadHeaderModel(
|
|
2023
|
+
path=upload_directory,
|
|
2024
|
+
files=[
|
|
2025
|
+
UploadFileModel(
|
|
2026
|
+
isMain=file["is_main"],
|
|
2027
|
+
length=file["size"],
|
|
2028
|
+
path=file["relative_path"]
|
|
2029
|
+
)
|
|
2030
|
+
for file in package_files
|
|
2031
|
+
]
|
|
2032
|
+
)
|
|
2033
|
+
|
|
2034
|
+
resp = await client.upload_header(header, session_id)
|
|
2035
|
+
|
|
2036
|
+
is_azure = _is_upload_type(resp.upload_type, 2, "azure")
|
|
2037
|
+
is_amazon = _is_upload_type(resp.upload_type, 1, "amazons3")
|
|
2038
|
+
|
|
2039
|
+
def _normalize_relative_path(path: str) -> str:
|
|
2040
|
+
return (path or "").replace("\\", "/").lstrip("/")
|
|
2041
|
+
|
|
2042
|
+
multipart_by_path = {
|
|
2043
|
+
_normalize_relative_path(mf.file_path): mf
|
|
2044
|
+
for mf in (resp.multipart_files or [])
|
|
2045
|
+
}
|
|
2046
|
+
|
|
2047
|
+
# Keep aggregate progress in bytes across all files (same semantics as the rest of this SDK).
|
|
2048
|
+
total_bytes = sum(file["size"] for file in package_files)
|
|
2049
|
+
uploaded_size = [0] * len(package_files)
|
|
2050
|
+
|
|
2051
|
+
def update_progress(index: int, loaded: int):
|
|
2052
|
+
capped = min(max(loaded, 0), package_files[index]["size"])
|
|
2053
|
+
uploaded_size[index] = capped
|
|
2054
|
+
if callable(progress_callback) and total_bytes > 0:
|
|
2055
|
+
progress_callback(sum(uploaded_size), total_bytes)
|
|
2056
|
+
|
|
2057
|
+
if callable(progress_callback):
|
|
2058
|
+
progress_callback(0, total_bytes)
|
|
2059
|
+
|
|
2060
|
+
has_direct_urls = bool(resp.urls)
|
|
2061
|
+
direct_urls = list(resp.urls or [])
|
|
2062
|
+
fallback_urls = _build_upload_urls(client.server_url, session_id, resp.id, package_files)
|
|
2063
|
+
|
|
2064
|
+
for index, file in enumerate(package_files):
|
|
2065
|
+
direct_url = direct_urls[index] if index < len(direct_urls) else None
|
|
2066
|
+
relative_path = _normalize_relative_path(file["relative_path"])
|
|
2067
|
+
multipart_file = multipart_by_path.get(relative_path)
|
|
2068
|
+
|
|
2069
|
+
def per_file_progress(sent: int, _total: int, idx=index):
|
|
2070
|
+
update_progress(idx, sent)
|
|
2071
|
+
|
|
2072
|
+
with open(file["local_path"], "rb") as stream:
|
|
2073
|
+
if is_amazon and multipart_file is not None:
|
|
2074
|
+
remote_file_path = _join_virtual_path(upload_directory, multipart_file.file_path)
|
|
2075
|
+
await client.upload_multipart_file_to_s3(
|
|
2076
|
+
multipart_file,
|
|
2077
|
+
remote_file_path,
|
|
2078
|
+
stream,
|
|
2079
|
+
session_id,
|
|
2080
|
+
progress_callback=per_file_progress
|
|
2081
|
+
)
|
|
2082
|
+
elif is_azure:
|
|
2083
|
+
if not direct_url:
|
|
2084
|
+
return False, f"No upload URL provided for file {file['relative_path']}"
|
|
2085
|
+
await client.upload_file(
|
|
2086
|
+
resp.id,
|
|
2087
|
+
resp.upload_type,
|
|
2088
|
+
direct_url,
|
|
2089
|
+
file["relative_path"],
|
|
2090
|
+
stream,
|
|
2091
|
+
session_id,
|
|
2092
|
+
total_bytes=file["size"],
|
|
2093
|
+
progress_callback=per_file_progress,
|
|
2094
|
+
method="PUT",
|
|
2095
|
+
verify=verify
|
|
2096
|
+
)
|
|
2097
|
+
else:
|
|
2098
|
+
upload_url = direct_url or (fallback_urls[index] if index < len(fallback_urls) else None)
|
|
2099
|
+
method = "PUT" if bool(direct_url) else "POST"
|
|
2100
|
+
if not upload_url:
|
|
2101
|
+
return False, f"No upload URL provided for file {file['relative_path']}"
|
|
2102
|
+
await client.upload_file(
|
|
2103
|
+
resp.id,
|
|
2104
|
+
resp.upload_type,
|
|
2105
|
+
upload_url,
|
|
2106
|
+
file["relative_path"],
|
|
2107
|
+
stream,
|
|
2108
|
+
session_id,
|
|
2109
|
+
total_bytes=file["size"],
|
|
2110
|
+
progress_callback=per_file_progress,
|
|
2111
|
+
method=method,
|
|
2112
|
+
verify=verify
|
|
2113
|
+
)
|
|
2114
|
+
|
|
2115
|
+
update_progress(index, file["size"])
|
|
2116
|
+
|
|
2117
|
+
finalize_url = f"{client.server_url}transfer/Upload/{resp.id}"
|
|
2118
|
+
finalize_response = await asyncio.to_thread(
|
|
2119
|
+
requests.get,
|
|
2120
|
+
finalize_url,
|
|
2121
|
+
params={"sessionId": session_id},
|
|
2122
|
+
verify=verify
|
|
2123
|
+
)
|
|
2124
|
+
|
|
2125
|
+
if finalize_response.status_code < 200 or finalize_response.status_code >= 300:
|
|
2126
|
+
try:
|
|
2127
|
+
payload = finalize_response.json()
|
|
2128
|
+
error_message = payload.get("Message") or payload.get("ExceptionMessage")
|
|
2129
|
+
except Exception:
|
|
2130
|
+
error_message = finalize_response.text
|
|
2131
|
+
return False, f"Upload finalize failed: {error_message}"
|
|
2132
|
+
|
|
2133
|
+
main_slide_path = _join_virtual_path(upload_directory, package_files[0]["relative_path"])
|
|
2134
|
+
image_ready = await _poll_image_info(
|
|
2135
|
+
main_slide_path,
|
|
2136
|
+
session_id,
|
|
2137
|
+
verify=verify,
|
|
2138
|
+
max_attempts=5,
|
|
2139
|
+
interval_seconds=1.0
|
|
2140
|
+
)
|
|
2141
|
+
|
|
2142
|
+
if not image_ready:
|
|
2143
|
+
return False, "Slide upload completed, but unable to fetch image information."
|
|
2144
|
+
|
|
2145
|
+
if callable(progress_callback):
|
|
2146
|
+
progress_callback(total_bytes, total_bytes)
|
|
2147
|
+
|
|
2148
|
+
return True, None
|
|
2149
|
+
except Exception as e:
|
|
2150
|
+
if os.getenv("PMA_UPLOAD_DEBUG_TRACE", "").lower() in ("1", "true", "yes"):
|
|
2151
|
+
return False, f"{e}\n{traceback.format_exc()}"
|
|
2152
|
+
return False, str(e)
|
|
2153
|
+
|
|
2154
|
+
|
|
1928
2155
|
async def upload(
|
|
1929
2156
|
*,
|
|
1930
2157
|
pma_core_url: str,
|
|
@@ -1939,7 +2166,7 @@ async def upload(
|
|
|
1939
2166
|
Automatically:
|
|
1940
2167
|
- calculates full slide size (file + folder + nested files)
|
|
1941
2168
|
- detects server upload type
|
|
1942
|
-
- routes
|
|
2169
|
+
- routes according to server upload strategy
|
|
1943
2170
|
"""
|
|
1944
2171
|
|
|
1945
2172
|
if not os.path.isfile(slide_path):
|
|
@@ -1959,18 +2186,18 @@ async def upload(
|
|
|
1959
2186
|
|
|
1960
2187
|
try:
|
|
1961
2188
|
client = PmaCoreClient(pma_core_url)
|
|
2189
|
+
package_files = _collect_slide_package_files(slide_path)
|
|
1962
2190
|
|
|
1963
|
-
|
|
1964
|
-
file_name = os.path.basename(slide_path)
|
|
1965
|
-
|
|
2191
|
+
normalized_upload_directory = _normalize_upload_directory(upload_directory)
|
|
1966
2192
|
header = UploadHeaderModel(
|
|
1967
|
-
path=
|
|
2193
|
+
path=normalized_upload_directory,
|
|
1968
2194
|
files=[
|
|
1969
2195
|
UploadFileModel(
|
|
1970
|
-
isMain=
|
|
1971
|
-
length=
|
|
1972
|
-
path=
|
|
2196
|
+
isMain=file["is_main"],
|
|
2197
|
+
length=file["size"],
|
|
2198
|
+
path=file["relative_path"]
|
|
1973
2199
|
)
|
|
2200
|
+
for file in package_files
|
|
1974
2201
|
]
|
|
1975
2202
|
)
|
|
1976
2203
|
|
|
@@ -1983,7 +2210,7 @@ async def upload(
|
|
|
1983
2210
|
# ================================
|
|
1984
2211
|
|
|
1985
2212
|
# UploadType 0 → always small uploader
|
|
1986
|
-
if resp.upload_type
|
|
2213
|
+
if _is_upload_type(resp.upload_type, 0, "filesystem"):
|
|
1987
2214
|
print("UploadType 0 → forcing SMALL uploader")
|
|
1988
2215
|
|
|
1989
2216
|
return upload_legacy(
|
|
@@ -1993,33 +2220,21 @@ async def upload(
|
|
|
1993
2220
|
progress_callback=progress_callback
|
|
1994
2221
|
)
|
|
1995
2222
|
|
|
1996
|
-
#
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
)
|
|
2006
|
-
|
|
2007
|
-
else:
|
|
2008
|
-
print("Using LARGE file uploader")
|
|
2009
|
-
|
|
2010
|
-
return await upload_file_over_5gb(
|
|
2011
|
-
pma_core_url=pma_core_url,
|
|
2012
|
-
session_id=session_id,
|
|
2013
|
-
slide_path=slide_path,
|
|
2014
|
-
upload_directory=upload_directory,
|
|
2015
|
-
progress_callback=progress_callback
|
|
2016
|
-
)
|
|
2223
|
+
# For cloud uploads, follow server strategy (S3 multipart, Azure blocks, or direct URLs).
|
|
2224
|
+
print("Using SERVER-DRIVEN uploader")
|
|
2225
|
+
return await upload_file_on_cloud(
|
|
2226
|
+
pma_core_url=pma_core_url,
|
|
2227
|
+
session_id=session_id,
|
|
2228
|
+
slide_path=slide_path,
|
|
2229
|
+
upload_directory=upload_directory,
|
|
2230
|
+
progress_callback=progress_callback
|
|
2231
|
+
)
|
|
2017
2232
|
|
|
2018
2233
|
except Exception as e:
|
|
2019
2234
|
return False, str(e)
|
|
2020
2235
|
|
|
2021
2236
|
# **************************************#
|
|
2022
|
-
# ===
|
|
2237
|
+
# === Filesystem Uploader === #
|
|
2023
2238
|
# **************************************#
|
|
2024
2239
|
def upload_legacy(slide_path, upload_directory, session_id, progress_callback=None, verify=True):
|
|
2025
2240
|
"""
|
|
@@ -2157,15 +2372,16 @@ def upload_legacy(slide_path, upload_directory, session_id, progress_callback=No
|
|
|
2157
2372
|
|
|
2158
2373
|
|
|
2159
2374
|
# **************************************#
|
|
2160
|
-
# ===
|
|
2375
|
+
# === Cloud Uploader === #
|
|
2161
2376
|
# **************************************#
|
|
2162
|
-
async def
|
|
2377
|
+
async def upload_file_on_cloud(
|
|
2163
2378
|
*,
|
|
2164
2379
|
pma_core_url: str,
|
|
2165
2380
|
session_id: str,
|
|
2166
2381
|
slide_path: str,
|
|
2167
2382
|
upload_directory: str,
|
|
2168
2383
|
progress_callback=None,
|
|
2384
|
+
verify=True,
|
|
2169
2385
|
):
|
|
2170
2386
|
"""
|
|
2171
2387
|
Asynchronous large-file upload implementation using PmaCoreClient.
|
|
@@ -2176,90 +2392,22 @@ async def upload_file_over_5gb(
|
|
|
2176
2392
|
- global progress tracking across all uploaded files
|
|
2177
2393
|
"""
|
|
2178
2394
|
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
all_files.append(slide_path)
|
|
2183
|
-
|
|
2184
|
-
vsi_name_no_ext = os.path.splitext(os.path.basename(slide_path))[0]
|
|
2185
|
-
local_stack_root = os.path.join(os.path.dirname(slide_path), vsi_name_no_ext)
|
|
2186
|
-
|
|
2187
|
-
# stack files
|
|
2188
|
-
if os.path.isdir(local_stack_root):
|
|
2189
|
-
for root, _, files in os.walk(local_stack_root):
|
|
2190
|
-
for f in files:
|
|
2191
|
-
all_files.append(os.path.join(root, f))
|
|
2192
|
-
|
|
2193
|
-
# ===== general size =====
|
|
2194
|
-
total_bytes = sum(os.stat(f).st_size for f in all_files)
|
|
2195
|
-
|
|
2196
|
-
# ===== GLOBAL TRACKER =====
|
|
2197
|
-
class GlobalProgressTracker:
|
|
2198
|
-
def __init__(self, total_bytes, callback):
|
|
2199
|
-
self.total_bytes = total_bytes
|
|
2200
|
-
self.callback = callback
|
|
2201
|
-
self.sent_total = 0
|
|
2202
|
-
|
|
2203
|
-
def add(self, delta):
|
|
2204
|
-
self.sent_total += delta
|
|
2205
|
-
if self.callback and self.total_bytes > 0:
|
|
2206
|
-
self.callback(self.sent_total, self.total_bytes)
|
|
2207
|
-
|
|
2208
|
-
tracker = GlobalProgressTracker(total_bytes, progress_callback)
|
|
2209
|
-
|
|
2210
|
-
# ===== FILE CALLBACK WRAPPER =====
|
|
2211
|
-
def make_file_cb():
|
|
2212
|
-
last_sent = 0
|
|
2213
|
-
|
|
2214
|
-
def _cb(sent, total):
|
|
2215
|
-
nonlocal last_sent
|
|
2216
|
-
delta = sent - last_sent
|
|
2217
|
-
last_sent = sent
|
|
2218
|
-
tracker.add(delta)
|
|
2395
|
+
if not os.path.isfile(slide_path):
|
|
2396
|
+
return False, f"File not found: {slide_path}"
|
|
2219
2397
|
|
|
2220
|
-
|
|
2398
|
+
normalized_upload_directory = _normalize_upload_directory(upload_directory)
|
|
2399
|
+
package_files = _collect_slide_package_files(slide_path)
|
|
2221
2400
|
|
|
2222
|
-
|
|
2223
|
-
|
|
2401
|
+
client = PmaCoreClient(pma_core_url)
|
|
2402
|
+
return await _upload_package_files(
|
|
2403
|
+
client=client,
|
|
2224
2404
|
session_id=session_id,
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
progress_callback=
|
|
2405
|
+
upload_directory=normalized_upload_directory,
|
|
2406
|
+
package_files=package_files,
|
|
2407
|
+
progress_callback=progress_callback,
|
|
2408
|
+
verify=verify,
|
|
2228
2409
|
)
|
|
2229
2410
|
|
|
2230
|
-
if not ok:
|
|
2231
|
-
return False, f"Upload failed: {err}"
|
|
2232
|
-
|
|
2233
|
-
# ===== STACK =====
|
|
2234
|
-
if os.path.isdir(local_stack_root):
|
|
2235
|
-
|
|
2236
|
-
for root, _, files in os.walk(local_stack_root):
|
|
2237
|
-
|
|
2238
|
-
for file_name in files:
|
|
2239
|
-
|
|
2240
|
-
local_file = os.path.join(root, file_name)
|
|
2241
|
-
|
|
2242
|
-
rel_inside_stack = os.path.relpath(
|
|
2243
|
-
local_file,
|
|
2244
|
-
local_stack_root
|
|
2245
|
-
).replace("\\", "/")
|
|
2246
|
-
|
|
2247
|
-
remote_dir = f"{upload_directory}/{vsi_name_no_ext}/{os.path.dirname(rel_inside_stack)}"
|
|
2248
|
-
remote_dir = remote_dir.rstrip("/")
|
|
2249
|
-
|
|
2250
|
-
ok, err = await large_files_upload_package(
|
|
2251
|
-
pma_core_url=pma_core_url,
|
|
2252
|
-
session_id=session_id,
|
|
2253
|
-
slide_path=local_file,
|
|
2254
|
-
upload_directory=remote_dir,
|
|
2255
|
-
progress_callback=make_file_cb()
|
|
2256
|
-
)
|
|
2257
|
-
|
|
2258
|
-
if not ok:
|
|
2259
|
-
return False, f"Stack upload failed: {local_file} -> {err}"
|
|
2260
|
-
|
|
2261
|
-
return True, None
|
|
2262
|
-
|
|
2263
2411
|
|
|
2264
2412
|
async def large_files_upload_package(
|
|
2265
2413
|
*,
|
|
@@ -2268,93 +2416,37 @@ async def large_files_upload_package(
|
|
|
2268
2416
|
slide_path: str,
|
|
2269
2417
|
upload_directory: str,
|
|
2270
2418
|
progress_callback=None,
|
|
2419
|
+
verify=True,
|
|
2271
2420
|
) -> tuple[bool, str | None]:
|
|
2272
2421
|
"""
|
|
2273
|
-
Low-level async upload
|
|
2422
|
+
Low-level async upload wrapper for a single file.
|
|
2274
2423
|
|
|
2275
2424
|
Performs:
|
|
2276
2425
|
- upload header request
|
|
2277
2426
|
- multipart or direct upload
|
|
2278
|
-
- upload
|
|
2427
|
+
- upload finalization + image info polling
|
|
2279
2428
|
"""
|
|
2280
2429
|
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
print("trust_env:", client.session.trust_env)
|
|
2284
|
-
|
|
2285
|
-
if not os.path.isfile(slide_path):
|
|
2286
|
-
return False, f"File not found: {slide_path}"
|
|
2287
|
-
|
|
2288
|
-
file_size = os.stat(slide_path).st_size
|
|
2289
|
-
file_name = os.path.basename(slide_path)
|
|
2290
|
-
|
|
2291
|
-
# ===== HEADER =====
|
|
2292
|
-
header = UploadHeaderModel(
|
|
2293
|
-
path=upload_directory,
|
|
2294
|
-
files=[
|
|
2295
|
-
UploadFileModel(
|
|
2296
|
-
isMain=True,
|
|
2297
|
-
length=file_size,
|
|
2298
|
-
path=file_name
|
|
2299
|
-
)
|
|
2300
|
-
]
|
|
2301
|
-
)
|
|
2302
|
-
|
|
2303
|
-
# ===== SEND HEADER =====
|
|
2304
|
-
resp = await client.upload_header(header, session_id)
|
|
2305
|
-
|
|
2306
|
-
# print("UploadType:", resp.upload_type)
|
|
2307
|
-
print("MultipartFiles:", resp.multipart_files)
|
|
2308
|
-
print("Urls:", resp.urls)
|
|
2309
|
-
|
|
2310
|
-
# ===== MULTIPART ONLY IF SERVER SENT PARTS =====
|
|
2311
|
-
if resp.multipart_files and len(resp.multipart_files) > 0:
|
|
2312
|
-
|
|
2313
|
-
mp = resp.multipart_files[0]
|
|
2314
|
-
|
|
2315
|
-
full_virtual_path = (
|
|
2316
|
-
header.path.rstrip("/") + "/" + mp.file_path.lstrip("/")
|
|
2317
|
-
)
|
|
2318
|
-
|
|
2319
|
-
with open(slide_path, "rb") as stream:
|
|
2320
|
-
await client.upload_multipart_file_to_s3(
|
|
2321
|
-
mp,
|
|
2322
|
-
full_virtual_path,
|
|
2323
|
-
stream,
|
|
2324
|
-
session_id,
|
|
2325
|
-
progress_callback=progress_callback
|
|
2326
|
-
)
|
|
2327
|
-
|
|
2328
|
-
# ===== SINGLE PUT / AZURE / FILESYSTEM =====
|
|
2329
|
-
else:
|
|
2330
|
-
|
|
2331
|
-
upload_url = resp.urls[0] if resp.urls else None
|
|
2332
|
-
|
|
2333
|
-
with open(slide_path, "rb") as stream:
|
|
2334
|
-
await client.upload_file(
|
|
2335
|
-
resp.id,
|
|
2336
|
-
resp.upload_type,
|
|
2337
|
-
upload_url,
|
|
2338
|
-
file_name,
|
|
2339
|
-
stream,
|
|
2340
|
-
session_id,
|
|
2341
|
-
total_bytes=file_size,
|
|
2342
|
-
progress_callback=progress_callback
|
|
2343
|
-
)
|
|
2344
|
-
|
|
2345
|
-
# ===== STATUS =====
|
|
2346
|
-
for _ in range(12):
|
|
2347
|
-
st = await client.get_upload_status(resp.id, session_id)
|
|
2348
|
-
|
|
2349
|
-
if '"Complete": true' in st or '"Complete" : true' in st:
|
|
2350
|
-
return True, None
|
|
2351
|
-
|
|
2352
|
-
await asyncio.sleep(10)
|
|
2353
|
-
|
|
2354
|
-
return False, "Upload not completed"
|
|
2430
|
+
if not os.path.isfile(slide_path):
|
|
2431
|
+
return False, f"File not found: {slide_path}"
|
|
2355
2432
|
|
|
2356
|
-
|
|
2357
|
-
|
|
2433
|
+
normalized_upload_directory = _normalize_upload_directory(upload_directory)
|
|
2434
|
+
single_file = [{
|
|
2435
|
+
"local_path": slide_path,
|
|
2436
|
+
"relative_path": os.path.basename(slide_path),
|
|
2437
|
+
"is_main": True,
|
|
2438
|
+
"size": os.stat(slide_path).st_size
|
|
2439
|
+
}]
|
|
2440
|
+
|
|
2441
|
+
client = PmaCoreClient(pma_core_url)
|
|
2442
|
+
return await _upload_package_files(
|
|
2443
|
+
client=client,
|
|
2444
|
+
session_id=session_id,
|
|
2445
|
+
upload_directory=normalized_upload_directory,
|
|
2446
|
+
package_files=single_file,
|
|
2447
|
+
progress_callback=progress_callback,
|
|
2448
|
+
verify=verify,
|
|
2449
|
+
)
|
|
2358
2450
|
|
|
2359
2451
|
|
|
2360
2452
|
# callback for large files upload
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import asyncio
|
|
2
4
|
|
|
3
5
|
import requests
|
|
@@ -17,6 +19,9 @@ class TLS12HttpAdapter(HTTPAdapter):
|
|
|
17
19
|
ctx = ssl.create_default_context()
|
|
18
20
|
ctx.minimum_version = ssl.TLSVersion.TLSv1_2
|
|
19
21
|
ctx.maximum_version = ssl.TLSVersion.TLSv1_2
|
|
22
|
+
# Requests may switch cert requirements to CERT_NONE when verify=False.
|
|
23
|
+
# Keep check_hostname disabled on the context to avoid SSL runtime errors.
|
|
24
|
+
ctx.check_hostname = False
|
|
20
25
|
|
|
21
26
|
self.poolmanager = PoolManager(
|
|
22
27
|
num_pools=connections,
|
|
@@ -26,6 +31,13 @@ class TLS12HttpAdapter(HTTPAdapter):
|
|
|
26
31
|
**pool_kwargs
|
|
27
32
|
)
|
|
28
33
|
|
|
34
|
+
def cert_verify(self, conn, url, verify, cert):
|
|
35
|
+
super().cert_verify(conn, url, verify, cert)
|
|
36
|
+
# urllib3 expects assert_hostname to be False/None/string, not bool True.
|
|
37
|
+
# False disables hostname checks (verify=False); None means default behavior.
|
|
38
|
+
if hasattr(conn, "assert_hostname"):
|
|
39
|
+
conn.assert_hostname = False if not verify else None
|
|
40
|
+
|
|
29
41
|
class _ProgressStream:
|
|
30
42
|
def __init__(self, raw, total_bytes: int, callback: UploadProgressCallback | None):
|
|
31
43
|
self.raw = raw
|
|
@@ -48,6 +60,12 @@ class _ProgressStream:
|
|
|
48
60
|
def tell(self):
|
|
49
61
|
return self.raw.tell()
|
|
50
62
|
|
|
63
|
+
def __len__(self):
|
|
64
|
+
return self.total_bytes
|
|
65
|
+
|
|
66
|
+
def close(self):
|
|
67
|
+
return self.raw.close()
|
|
68
|
+
|
|
51
69
|
|
|
52
70
|
# Model classes
|
|
53
71
|
class UploadFileModel:
|
|
@@ -200,12 +218,17 @@ class PmaCoreClient:
|
|
|
200
218
|
response.raise_for_status()
|
|
201
219
|
return UploadResponse(**response.json())
|
|
202
220
|
|
|
203
|
-
async def upload_file(self, upload_id: int, upload_type: Optional[str], upload_url: str,
|
|
221
|
+
async def upload_file(self, upload_id: int, upload_type: Optional[str], upload_url: Optional[str],
|
|
204
222
|
relative_path: str, stream: IO, session_id: str,
|
|
205
223
|
total_bytes: int | None = None,
|
|
206
|
-
progress_callback: UploadProgressCallback | None = None
|
|
207
|
-
|
|
208
|
-
|
|
224
|
+
progress_callback: UploadProgressCallback | None = None,
|
|
225
|
+
method: str = "PUT",
|
|
226
|
+
verify: bool = True):
|
|
227
|
+
|
|
228
|
+
normalized_upload_type = str(upload_type).lower() if upload_type is not None else ""
|
|
229
|
+
if normalized_upload_type in ("azure", "2"):
|
|
230
|
+
if not upload_url:
|
|
231
|
+
raise ValueError("Azure upload requires an upload URL")
|
|
209
232
|
await self.upload_file_to_azure(
|
|
210
233
|
upload_url,
|
|
211
234
|
stream,
|
|
@@ -221,32 +244,29 @@ class PmaCoreClient:
|
|
|
221
244
|
wrapped = _ProgressStream(stream, total_bytes, progress_callback)
|
|
222
245
|
|
|
223
246
|
if upload_to_presigned:
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
)
|
|
246
|
-
|
|
247
|
-
if response.status >= 300:
|
|
248
|
-
raise Exception(f"S3 upload failed: {response.status}")
|
|
247
|
+
resolved_method = (method or "PUT").upper()
|
|
248
|
+
headers = {}
|
|
249
|
+
if total_bytes is not None:
|
|
250
|
+
headers["Content-Length"] = str(total_bytes)
|
|
251
|
+
|
|
252
|
+
if resolved_method == "POST":
|
|
253
|
+
files = {"file": (relative_path.split("/")[-1], wrapped, "application/octet-stream")}
|
|
254
|
+
response = await asyncio.to_thread(
|
|
255
|
+
self.session.post,
|
|
256
|
+
upload_url,
|
|
257
|
+
files=files,
|
|
258
|
+
verify=verify
|
|
259
|
+
)
|
|
260
|
+
else:
|
|
261
|
+
response = await asyncio.to_thread(
|
|
262
|
+
self.session.put,
|
|
263
|
+
upload_url,
|
|
264
|
+
data=wrapped,
|
|
265
|
+
headers=headers,
|
|
266
|
+
verify=verify
|
|
267
|
+
)
|
|
249
268
|
|
|
269
|
+
response.raise_for_status()
|
|
250
270
|
return
|
|
251
271
|
|
|
252
272
|
url = f"{self.server_url}transfer/Upload/{upload_id}"
|
|
@@ -254,10 +274,11 @@ class PmaCoreClient:
|
|
|
254
274
|
|
|
255
275
|
files = {"file": (relative_path.split("/")[-1], wrapped, "application/octet-stream")}
|
|
256
276
|
response = await asyncio.to_thread(
|
|
257
|
-
|
|
277
|
+
self.session.post,
|
|
258
278
|
url,
|
|
259
279
|
params=params,
|
|
260
|
-
files=files
|
|
280
|
+
files=files,
|
|
281
|
+
verify=verify
|
|
261
282
|
)
|
|
262
283
|
response.raise_for_status()
|
|
263
284
|
|
|
@@ -340,7 +361,7 @@ class PmaCoreClient:
|
|
|
340
361
|
total_bytes: int | None = None,
|
|
341
362
|
progress_callback: UploadProgressCallback | None = None,
|
|
342
363
|
):
|
|
343
|
-
block_size =
|
|
364
|
+
block_size = 1024 * 1024 * 1024
|
|
344
365
|
block_id = 0
|
|
345
366
|
sent_total = 0
|
|
346
367
|
safe_base = upload_url.replace(" ", "%20")
|
|
@@ -439,8 +460,18 @@ class PmaCoreClient:
|
|
|
439
460
|
|
|
440
461
|
response.raise_for_status()
|
|
441
462
|
|
|
442
|
-
|
|
443
|
-
|
|
463
|
+
raw_etag = response.headers.get("ETag")
|
|
464
|
+
if raw_etag is None:
|
|
465
|
+
raise ValueError(
|
|
466
|
+
f"Multipart part upload succeeded but no ETag header was returned for part {part.PartNumber}."
|
|
467
|
+
)
|
|
468
|
+
|
|
469
|
+
# Some proxies/S3-compatible layers may surface non-string header values.
|
|
470
|
+
etag = str(raw_etag).strip().strip('"')
|
|
471
|
+
if not etag:
|
|
472
|
+
raise ValueError(
|
|
473
|
+
f"Multipart part upload succeeded but ETag value was empty for part {part.PartNumber}."
|
|
474
|
+
)
|
|
444
475
|
|
|
445
476
|
part_etags.append(
|
|
446
477
|
PartETagModel(
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = '2.0.0.185'
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = '2.0.0.184'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|