pma-python 2.0.0.180__tar.gz → 2.0.0.182__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pma_python-2.0.0.180 → pma_python-2.0.0.182}/PKG-INFO +1 -1
- {pma_python-2.0.0.180 → pma_python-2.0.0.182}/pma_python/core.py +290 -132
- {pma_python-2.0.0.180 → pma_python-2.0.0.182}/pma_python/pma_core_client.py +38 -23
- pma_python-2.0.0.182/pma_python/version.py +1 -0
- {pma_python-2.0.0.180 → pma_python-2.0.0.182}/pma_python.egg-info/PKG-INFO +1 -1
- pma_python-2.0.0.180/pma_python/version.py +0 -1
- {pma_python-2.0.0.180 → pma_python-2.0.0.182}/README.md +0 -0
- {pma_python-2.0.0.180 → pma_python-2.0.0.182}/long_desc.txt +0 -0
- {pma_python-2.0.0.180 → pma_python-2.0.0.182}/pma_python/__init__.py +0 -0
- {pma_python-2.0.0.180 → pma_python-2.0.0.182}/pma_python/control.py +0 -0
- {pma_python-2.0.0.180 → pma_python-2.0.0.182}/pma_python/core_admin.py +0 -0
- {pma_python-2.0.0.180 → pma_python-2.0.0.182}/pma_python/pma.py +0 -0
- {pma_python-2.0.0.180 → pma_python-2.0.0.182}/pma_python/view.py +0 -0
- {pma_python-2.0.0.180 → pma_python-2.0.0.182}/pma_python.egg-info/SOURCES.txt +0 -0
- {pma_python-2.0.0.180 → pma_python-2.0.0.182}/pma_python.egg-info/dependency_links.txt +0 -0
- {pma_python-2.0.0.180 → pma_python-2.0.0.182}/pma_python.egg-info/not-zip-safe +0 -0
- {pma_python-2.0.0.180 → pma_python-2.0.0.182}/pma_python.egg-info/requires.txt +0 -0
- {pma_python-2.0.0.180 → pma_python-2.0.0.182}/pma_python.egg-info/top_level.txt +0 -0
- {pma_python-2.0.0.180 → pma_python-2.0.0.182}/setup.cfg +0 -0
- {pma_python-2.0.0.180 → pma_python-2.0.0.182}/setup.py +0 -0
|
@@ -18,7 +18,7 @@ import pandas as pd
|
|
|
18
18
|
import requests
|
|
19
19
|
from requests_toolbelt.multipart.encoder import MultipartEncoder, MultipartEncoderMonitor
|
|
20
20
|
|
|
21
|
-
from .pma_core_client import PmaCoreClient, UploadHeaderModel, UploadFileModel
|
|
21
|
+
from .pma_core_client import PmaCoreClient, UploadHeaderModel, UploadFileModel, UploadResponse
|
|
22
22
|
from typing import Callable
|
|
23
23
|
|
|
24
24
|
ProgressCallback = Callable[[int, int], None]
|
|
@@ -599,6 +599,7 @@ def analyse_corresponding_slides(sessionPathDict, recursive=False, includeFinger
|
|
|
599
599
|
"""
|
|
600
600
|
Return a pandas DataFrame that indicates which slides exist on which PMA.core instances
|
|
601
601
|
:param dict sessionPathDict: a dictionary that looks e.g. like
|
|
602
|
+
|
|
602
603
|
{DevSessionID: rootDirAndPath1, ProdSessionID: rootDirAndPath2 }
|
|
603
604
|
:param bool recursive: indicates whether the method should look in sub-directories or not
|
|
604
605
|
"""
|
|
@@ -825,10 +826,12 @@ def get_zoomlevels_dict(slideRef, sessionID=None, min_number_of_tiles=0):
|
|
|
825
826
|
"""
|
|
826
827
|
Obtain a dictionary with the number of tiles per zoomlevel.
|
|
827
828
|
Information is returned as (x, y, n) tuples per zoomlevel, with
|
|
829
|
+
|
|
828
830
|
x = number of horizontal tiles,
|
|
829
831
|
y = number of vertical tiles,
|
|
830
832
|
n = total number of tiles at specified zoomlevel (x * y)
|
|
831
|
-
|
|
833
|
+
|
|
834
|
+
Use min_number_of_tiles argument to specify that you're only interested in zoomlevels that include at least a given number of tiles
|
|
832
835
|
"""
|
|
833
836
|
zoomlevels = list(range(0, get_max_zoomlevel(slideRef, sessionID) + 1))
|
|
834
837
|
dimensions = [
|
|
@@ -1532,119 +1535,6 @@ def _pma_upload_amazon_callback(bytes_read, total_size, previous, filename):
|
|
|
1532
1535
|
return v
|
|
1533
1536
|
|
|
1534
1537
|
|
|
1535
|
-
def upload(local_source_slide, target_folder, target_pma_core_sessionID, callback=None, verify=True):
|
|
1536
|
-
"""
|
|
1537
|
-
Uploads a slide to a PMA.core server. Requires a PMA.start installation
|
|
1538
|
-
:param str local_source_slide: The local PMA.start relative file to upload
|
|
1539
|
-
:param str target_folder: The root directory and path to upload to the PMA.core server
|
|
1540
|
-
:param str target_pma_core_sessionID: A valid session id for a PMA.core server
|
|
1541
|
-
:param function|boolean callback: If True a default progress will be printed.
|
|
1542
|
-
If a function is passed it will be called for progress on each file upload.
|
|
1543
|
-
The function has the following signature:
|
|
1544
|
-
`callback(bytes_read, bytes_length, filename)`
|
|
1545
|
-
"""
|
|
1546
|
-
if not _pma_is_lite():
|
|
1547
|
-
raise Exception(
|
|
1548
|
-
"No PMA.start found on localhost. Are you sure it is running?")
|
|
1549
|
-
|
|
1550
|
-
if not target_folder:
|
|
1551
|
-
raise ValueError("target_folder cannot be empty")
|
|
1552
|
-
|
|
1553
|
-
if (target_folder.startswith("/")):
|
|
1554
|
-
target_folder = target_folder[1:]
|
|
1555
|
-
|
|
1556
|
-
files = get_files_for_slide(local_source_slide, _pma_pmacoreliteSessionID)
|
|
1557
|
-
sessionID = _pma_session_id(target_pma_core_sessionID)
|
|
1558
|
-
url = _pma_url(sessionID) + "transfer/Upload?sessionID=" + pma._pma_q(sessionID)
|
|
1559
|
-
|
|
1560
|
-
mainDirectory = ''
|
|
1561
|
-
for i, f in enumerate(files):
|
|
1562
|
-
md = os.path.dirname(f)
|
|
1563
|
-
if i == 0 or len(md) < len(mainDirectory):
|
|
1564
|
-
mainDirectory = md
|
|
1565
|
-
|
|
1566
|
-
uploadFiles = []
|
|
1567
|
-
for i, filepath in enumerate(files):
|
|
1568
|
-
s = os.path.getsize(filepath)
|
|
1569
|
-
if s > 0:
|
|
1570
|
-
uploadFiles.append({
|
|
1571
|
-
"Path": filepath.replace(mainDirectory, '').strip("\\").strip('/'),
|
|
1572
|
-
"Length": s,
|
|
1573
|
-
"IsMain": i == len(files) - 1,
|
|
1574
|
-
"FullPath": filepath
|
|
1575
|
-
})
|
|
1576
|
-
|
|
1577
|
-
data = {"Path": target_folder, "Files": uploadFiles}
|
|
1578
|
-
|
|
1579
|
-
uploadHeaderResponse = requests.post(url, json=data, verify=verify)
|
|
1580
|
-
if not uploadHeaderResponse.status_code == 200:
|
|
1581
|
-
print(uploadHeaderResponse.json())
|
|
1582
|
-
raise Exception(uploadHeaderResponse.json()["Message"])
|
|
1583
|
-
|
|
1584
|
-
uploadHeader = uploadHeaderResponse.json()
|
|
1585
|
-
|
|
1586
|
-
pmaCoreUploadUrl = _pma_url(sessionID) + "transfer/Upload/" + pma._pma_q(
|
|
1587
|
-
uploadHeader["Id"]) + "?sessionID=" + pma._pma_q(sessionID) + "&path={0}"
|
|
1588
|
-
|
|
1589
|
-
isAmazonUpload = True
|
|
1590
|
-
if not uploadHeader['Urls']:
|
|
1591
|
-
isAmazonUpload = False
|
|
1592
|
-
uploadHeader['Urls'] = [pmaCoreUploadUrl.format(
|
|
1593
|
-
f["Path"]) for f in uploadFiles]
|
|
1594
|
-
|
|
1595
|
-
for i, f in enumerate(uploadFiles):
|
|
1596
|
-
uploadUrl = uploadHeader['Urls'][i]
|
|
1597
|
-
|
|
1598
|
-
e = MultipartEncoder(
|
|
1599
|
-
fields={"file": (os.path.basename(f["Path"]), open(f["FullPath"], 'rb'), 'application/octet-stream')})
|
|
1600
|
-
|
|
1601
|
-
_callback = None
|
|
1602
|
-
if callback is True:
|
|
1603
|
-
print("Uploading file: {0}".format(e.fields["file"][0]))
|
|
1604
|
-
if not isAmazonUpload:
|
|
1605
|
-
def _callback(x):
|
|
1606
|
-
return _pma_upload_callback(
|
|
1607
|
-
monitor, e.fields["file"][0])
|
|
1608
|
-
else:
|
|
1609
|
-
def _callback(bytes_read, total_size, previous):
|
|
1610
|
-
return _pma_upload_amazon_callback(
|
|
1611
|
-
bytes_read, total_size, previous, e.fields["file"][0])
|
|
1612
|
-
elif callable(callback):
|
|
1613
|
-
def _callback(x):
|
|
1614
|
-
return callback(
|
|
1615
|
-
x.bytes_read, x.len, x.previous, e.fields["file"][0])
|
|
1616
|
-
|
|
1617
|
-
monitor = MultipartEncoderMonitor(e, _callback)
|
|
1618
|
-
|
|
1619
|
-
monitor.previous = 0
|
|
1620
|
-
|
|
1621
|
-
r = None
|
|
1622
|
-
if not isAmazonUpload:
|
|
1623
|
-
r = requests.post(uploadUrl, data=monitor, headers={
|
|
1624
|
-
'Content-Type': monitor.content_type},
|
|
1625
|
-
verify=verify)
|
|
1626
|
-
else:
|
|
1627
|
-
headers = {'Content-Length': str(f["Length"])}
|
|
1628
|
-
if uploadHeader['UploadType'] == 2:
|
|
1629
|
-
headers = {
|
|
1630
|
-
'Content-Length': str(f["Length"]), 'x-ms-blob-type': 'BlockBlob'}
|
|
1631
|
-
|
|
1632
|
-
r = requests.put(uploadUrl, data=UploadChunksIterator(
|
|
1633
|
-
open(f["FullPath"], 'rb'), f["Path"], f["Length"], _callback), headers=headers, verify=verify)
|
|
1634
|
-
|
|
1635
|
-
if r.status_code < 200 or r.status_code >= 300:
|
|
1636
|
-
raise Exception("Error uploading file {0}: {1} \r\n{2}: {3}".format(
|
|
1637
|
-
f["Path"], uploadUrl, r.status_code, r.text))
|
|
1638
|
-
|
|
1639
|
-
uploadFinalizeResponse = requests.get(_pma_url(sessionID) + "transfer/Upload/"
|
|
1640
|
-
+ pma._pma_q(uploadHeader["Id"]) + "?sessionID=" + pma._pma_q(sessionID),
|
|
1641
|
-
verify=verify)
|
|
1642
|
-
if uploadFinalizeResponse.status_code < 200 or uploadFinalizeResponse.status_code >= 300:
|
|
1643
|
-
print(uploadFinalizeResponse.json())
|
|
1644
|
-
raise Exception(uploadFinalizeResponse.json()[
|
|
1645
|
-
"Message"] + uploadFinalizeResponse.json()["ExceptionMessage"])
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
1538
|
class UploadChunksIterator:
|
|
1649
1539
|
def __init__(self, file: io.BufferedReader, filename, total_size: int, callback, chunk_size: int = 16 * 1024):
|
|
1650
1540
|
self.file = file
|
|
@@ -1663,11 +1553,10 @@ class UploadChunksIterator:
|
|
|
1663
1553
|
|
|
1664
1554
|
if not data:
|
|
1665
1555
|
raise StopIteration
|
|
1666
|
-
self.bytes_read +=
|
|
1667
|
-
if self.callback
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
self.previous = v
|
|
1556
|
+
self.bytes_read += len(data)
|
|
1557
|
+
if self.callback:
|
|
1558
|
+
self.callback(self.bytes_read, self.total_size)
|
|
1559
|
+
|
|
1671
1560
|
return data
|
|
1672
1561
|
|
|
1673
1562
|
def __len__(self):
|
|
@@ -1842,7 +1731,8 @@ def add_annotations(slideRef, classification, notes, anns, color="#000000", laye
|
|
|
1842
1731
|
:return: An integer representing the annotation id
|
|
1843
1732
|
:rtype: int
|
|
1844
1733
|
|
|
1845
|
-
Example
|
|
1734
|
+
Example::
|
|
1735
|
+
|
|
1846
1736
|
anns = []
|
|
1847
1737
|
annotation = {
|
|
1848
1738
|
"geometry": string,
|
|
@@ -2008,10 +1898,11 @@ def get_annotation_distance(slideRef, layerID, annotationID, sessionID=None, ver
|
|
|
2008
1898
|
return r.text
|
|
2009
1899
|
|
|
2010
1900
|
|
|
2011
|
-
|
|
2012
|
-
#
|
|
2013
|
-
|
|
2014
|
-
|
|
1901
|
+
# **************************************#
|
|
1902
|
+
# === Uploader === #
|
|
1903
|
+
# **************************************#
|
|
1904
|
+
FIVE_GB = 5 * 1024 * 1024 * 1024
|
|
1905
|
+
async def upload(
|
|
2015
1906
|
*,
|
|
2016
1907
|
pma_core_url: str,
|
|
2017
1908
|
session_id: str,
|
|
@@ -2019,11 +1910,263 @@ async def upload_large_files(
|
|
|
2019
1910
|
upload_directory: str,
|
|
2020
1911
|
progress_callback=None,
|
|
2021
1912
|
):
|
|
1913
|
+
"""
|
|
1914
|
+
Main upload entry point.
|
|
1915
|
+
|
|
1916
|
+
Automatically:
|
|
1917
|
+
- calculates full slide size (file + folder + nested files)
|
|
1918
|
+
- detects server upload type
|
|
1919
|
+
- routes upload to legacy transfer or large file uploader
|
|
1920
|
+
"""
|
|
1921
|
+
|
|
1922
|
+
if not os.path.isfile(slide_path):
|
|
1923
|
+
return False, f"File not found: {slide_path}"
|
|
1924
|
+
|
|
1925
|
+
# ================================
|
|
1926
|
+
# CALCULATE TOTAL SLIDE SIZE
|
|
1927
|
+
# ================================
|
|
1928
|
+
total_size = 0
|
|
1929
|
+
|
|
1930
|
+
# main file size
|
|
1931
|
+
total_size += os.stat(slide_path).st_size
|
|
1932
|
+
|
|
1933
|
+
# possible slide folder (multi-files slides)
|
|
1934
|
+
slide_name_no_ext = os.path.splitext(os.path.basename(slide_path))[0]
|
|
1935
|
+
slide_dir = os.path.join(os.path.dirname(slide_path), slide_name_no_ext)
|
|
1936
|
+
|
|
1937
|
+
if os.path.isdir(slide_dir):
|
|
1938
|
+
for root, _, files in os.walk(slide_dir):
|
|
1939
|
+
for f in files:
|
|
1940
|
+
total_size += os.stat(os.path.join(root, f)).st_size
|
|
1941
|
+
|
|
1942
|
+
print("Total slide size:", round(total_size / (1024 ** 3), 2), "GB")
|
|
1943
|
+
|
|
1944
|
+
# ================================
|
|
1945
|
+
# CHECK UPLOAD TYPE (PRE-FLIGHT)
|
|
1946
|
+
# ================================
|
|
1947
|
+
|
|
1948
|
+
try:
|
|
1949
|
+
client = PmaCoreClient(pma_core_url)
|
|
1950
|
+
|
|
1951
|
+
file_size = os.stat(slide_path).st_size
|
|
1952
|
+
file_name = os.path.basename(slide_path)
|
|
1953
|
+
|
|
1954
|
+
header = UploadHeaderModel(
|
|
1955
|
+
path=upload_directory,
|
|
1956
|
+
files=[
|
|
1957
|
+
UploadFileModel(
|
|
1958
|
+
isMain=True,
|
|
1959
|
+
length=file_size,
|
|
1960
|
+
path=file_name
|
|
1961
|
+
)
|
|
1962
|
+
]
|
|
1963
|
+
)
|
|
1964
|
+
|
|
1965
|
+
resp = await client.upload_header(header, session_id)
|
|
1966
|
+
|
|
1967
|
+
print("Detected UploadType:", resp.upload_type)
|
|
1968
|
+
|
|
1969
|
+
# ================================
|
|
1970
|
+
# ROUTING LOGIC
|
|
1971
|
+
# ================================
|
|
1972
|
+
|
|
1973
|
+
# UploadType 0 → always small uploader
|
|
1974
|
+
if resp.upload_type == 0:
|
|
1975
|
+
print("UploadType 0 → forcing SMALL uploader")
|
|
1976
|
+
|
|
1977
|
+
return upload_legacy(
|
|
1978
|
+
session_id=session_id,
|
|
1979
|
+
slide_path=slide_path,
|
|
1980
|
+
upload_directory=upload_directory,
|
|
1981
|
+
progress_callback=progress_callback
|
|
1982
|
+
)
|
|
1983
|
+
|
|
1984
|
+
# Otherwise fallback to size logic
|
|
1985
|
+
if total_size < FIVE_GB:
|
|
1986
|
+
print("Using SMALL file uploader")
|
|
1987
|
+
|
|
1988
|
+
return upload_legacy(
|
|
1989
|
+
session_id=session_id,
|
|
1990
|
+
slide_path=slide_path,
|
|
1991
|
+
upload_directory=upload_directory,
|
|
1992
|
+
progress_callback=progress_callback
|
|
1993
|
+
)
|
|
1994
|
+
|
|
1995
|
+
else:
|
|
1996
|
+
print("Using LARGE file uploader")
|
|
1997
|
+
|
|
1998
|
+
return await upload_file_over_5gb(
|
|
1999
|
+
pma_core_url=pma_core_url,
|
|
2000
|
+
session_id=session_id,
|
|
2001
|
+
slide_path=slide_path,
|
|
2002
|
+
upload_directory=upload_directory,
|
|
2003
|
+
progress_callback=progress_callback
|
|
2004
|
+
)
|
|
2005
|
+
|
|
2006
|
+
except Exception as e:
|
|
2007
|
+
return False, str(e)
|
|
2008
|
+
|
|
2009
|
+
# **************************************#
|
|
2010
|
+
# === Small Files < 5bg Uploader === #
|
|
2011
|
+
# **************************************#
|
|
2012
|
+
def upload_legacy(slide_path, upload_directory, session_id, progress_callback=None, verify=True):
|
|
2013
|
+
"""
|
|
2014
|
+
Synchronous upload implementation using direct HTTP requests.
|
|
2015
|
+
|
|
2016
|
+
Handles:
|
|
2017
|
+
- filesystem uploads (any file size when UploadType == 0)
|
|
2018
|
+
- non-chunked cloud uploads
|
|
2019
|
+
- single-file and multi-file slides (MRXS, VSI, etc.)
|
|
2020
|
+
- global progress aggregation across all uploaded files
|
|
2021
|
+
"""
|
|
2022
|
+
|
|
2023
|
+
if not upload_directory:
|
|
2024
|
+
raise ValueError("target_folder cannot be empty")
|
|
2025
|
+
|
|
2026
|
+
if (upload_directory.startswith("/")):
|
|
2027
|
+
upload_directory = upload_directory[1:]
|
|
2028
|
+
|
|
2029
|
+
files = get_files_for_slide(slide_path, _pma_pmacoreliteSessionID)
|
|
2030
|
+
sessionID = _pma_session_id(session_id)
|
|
2031
|
+
url = _pma_url(sessionID) + "transfer/Upload?sessionID=" + pma._pma_q(sessionID)
|
|
2032
|
+
|
|
2033
|
+
mainDirectory = ''
|
|
2034
|
+
for i, f in enumerate(files):
|
|
2035
|
+
md = os.path.dirname(f)
|
|
2036
|
+
if i == 0 or len(md) < len(mainDirectory):
|
|
2037
|
+
mainDirectory = md
|
|
2038
|
+
|
|
2039
|
+
uploadFiles = []
|
|
2040
|
+
for i, filepath in enumerate(files):
|
|
2041
|
+
s = os.path.getsize(filepath)
|
|
2042
|
+
if s > 0:
|
|
2043
|
+
uploadFiles.append({
|
|
2044
|
+
"Path": filepath.replace(mainDirectory, '').strip("\\").strip('/'),
|
|
2045
|
+
"Length": s,
|
|
2046
|
+
"IsMain": i == len(files) - 1,
|
|
2047
|
+
"FullPath": filepath
|
|
2048
|
+
})
|
|
2049
|
+
# ==============================
|
|
2050
|
+
# GLOBAL PROGRESS (ALL FILES)
|
|
2051
|
+
# ==============================
|
|
2052
|
+
total_bytes = sum(x["Length"] for x in uploadFiles) if uploadFiles else 0
|
|
2053
|
+
sent_total = 0
|
|
2054
|
+
|
|
2055
|
+
# If the user passed progress_callback=True → enable the default print
|
|
2056
|
+
if progress_callback is True:
|
|
2057
|
+
def progress_callback(sent: int, total: int):
|
|
2058
|
+
if total > 0:
|
|
2059
|
+
percent = sent / total * 100
|
|
2060
|
+
print(f"Upload progress: {percent:.2f}%")
|
|
2061
|
+
|
|
2062
|
+
def report_delta(delta: int):
|
|
2063
|
+
nonlocal sent_total
|
|
2064
|
+
if delta <= 0:
|
|
2065
|
+
return
|
|
2066
|
+
sent_total += delta
|
|
2067
|
+
if callable(progress_callback) and total_bytes > 0:
|
|
2068
|
+
if sent_total > total_bytes:
|
|
2069
|
+
sent_total = total_bytes
|
|
2070
|
+
progress_callback(sent_total, total_bytes)
|
|
2071
|
+
|
|
2072
|
+
data = {"Path": upload_directory, "Files": uploadFiles}
|
|
2073
|
+
|
|
2074
|
+
uploadHeaderResponse = requests.post(url, json=data, verify=verify)
|
|
2075
|
+
if not uploadHeaderResponse.status_code == 200:
|
|
2076
|
+
print(uploadHeaderResponse.json())
|
|
2077
|
+
raise Exception(uploadHeaderResponse.json()["Message"])
|
|
2078
|
+
|
|
2079
|
+
uploadHeader = uploadHeaderResponse.json()
|
|
2080
|
+
|
|
2081
|
+
pmaCoreUploadUrl = _pma_url(sessionID) + "transfer/Upload/" + pma._pma_q(
|
|
2082
|
+
uploadHeader["Id"]) + "?sessionID=" + pma._pma_q(sessionID) + "&path={0}"
|
|
2083
|
+
|
|
2084
|
+
isAmazonUpload = True
|
|
2085
|
+
if not uploadHeader['Urls']:
|
|
2086
|
+
isAmazonUpload = False
|
|
2087
|
+
uploadHeader['Urls'] = [pmaCoreUploadUrl.format(
|
|
2088
|
+
f["Path"]) for f in uploadFiles]
|
|
2089
|
+
|
|
2090
|
+
for i, f in enumerate(uploadFiles):
|
|
2091
|
+
uploadUrl = uploadHeader['Urls'][i]
|
|
2092
|
+
|
|
2093
|
+
e = MultipartEncoder(
|
|
2094
|
+
fields={"file": (os.path.basename(f["Path"]), open(f["FullPath"], 'rb'), 'application/octet-stream')})
|
|
2095
|
+
|
|
2096
|
+
# ==============================
|
|
2097
|
+
# PER-FILE -> GLOBAL PROGRESS
|
|
2098
|
+
# ==============================
|
|
2099
|
+
last_sent = 0
|
|
2100
|
+
|
|
2101
|
+
def monitor_callback(monitor):
|
|
2102
|
+
nonlocal last_sent
|
|
2103
|
+
current = monitor.bytes_read
|
|
2104
|
+
delta = current - last_sent
|
|
2105
|
+
last_sent = current
|
|
2106
|
+
report_delta(delta)
|
|
2107
|
+
|
|
2108
|
+
def iterator_callback(bytes_read, _total_size):
|
|
2109
|
+
nonlocal last_sent
|
|
2110
|
+
current = bytes_read
|
|
2111
|
+
delta = current - last_sent
|
|
2112
|
+
last_sent = current
|
|
2113
|
+
report_delta(delta)
|
|
2114
|
+
|
|
2115
|
+
monitor = MultipartEncoderMonitor(e, monitor_callback)
|
|
2116
|
+
|
|
2117
|
+
monitor.previous = 0
|
|
2118
|
+
|
|
2119
|
+
r = None
|
|
2120
|
+
if not isAmazonUpload:
|
|
2121
|
+
r = requests.post(uploadUrl, data=monitor, headers={
|
|
2122
|
+
'Content-Type': monitor.content_type},
|
|
2123
|
+
verify=verify)
|
|
2124
|
+
else:
|
|
2125
|
+
headers = {'Content-Length': str(f["Length"])}
|
|
2126
|
+
if uploadHeader['UploadType'] == 2:
|
|
2127
|
+
headers = {
|
|
2128
|
+
'Content-Length': str(f["Length"]), 'x-ms-blob-type': 'BlockBlob'}
|
|
2129
|
+
|
|
2130
|
+
r = requests.put(uploadUrl, data=UploadChunksIterator(
|
|
2131
|
+
open(f["FullPath"], 'rb'), f["Path"], f["Length"], iterator_callback), headers=headers, verify=verify)
|
|
2132
|
+
|
|
2133
|
+
if r.status_code < 200 or r.status_code >= 300:
|
|
2134
|
+
raise Exception("Error uploading file {0}: {1} \r\n{2}: {3}".format(
|
|
2135
|
+
f["Path"], uploadUrl, r.status_code, r.text))
|
|
2136
|
+
|
|
2137
|
+
uploadFinalizeResponse = requests.get(_pma_url(sessionID) + "transfer/Upload/"
|
|
2138
|
+
+ pma._pma_q(uploadHeader["Id"]) + "?sessionID=" + pma._pma_q(sessionID),
|
|
2139
|
+
verify=verify)
|
|
2140
|
+
if uploadFinalizeResponse.status_code < 200 or uploadFinalizeResponse.status_code >= 300:
|
|
2141
|
+
print(uploadFinalizeResponse.json())
|
|
2142
|
+
raise Exception(uploadFinalizeResponse.json()[
|
|
2143
|
+
"Message"] + uploadFinalizeResponse.json()["ExceptionMessage"])
|
|
2144
|
+
return True, None
|
|
2145
|
+
|
|
2146
|
+
|
|
2147
|
+
# **************************************#
|
|
2148
|
+
# === Large Files > 5gb Uploader === #
|
|
2149
|
+
# **************************************#
|
|
2150
|
+
async def upload_file_over_5gb(
|
|
2151
|
+
*,
|
|
2152
|
+
pma_core_url: str,
|
|
2153
|
+
session_id: str,
|
|
2154
|
+
slide_path: str,
|
|
2155
|
+
upload_directory: str,
|
|
2156
|
+
progress_callback=None,
|
|
2157
|
+
):
|
|
2158
|
+
"""
|
|
2159
|
+
Asynchronous large-file upload implementation using PmaCoreClient.
|
|
2160
|
+
|
|
2161
|
+
Handles:
|
|
2162
|
+
- chunked cloud uploads
|
|
2163
|
+
- multi-file slide stacks
|
|
2164
|
+
- global progress tracking across all uploaded files
|
|
2165
|
+
"""
|
|
2022
2166
|
|
|
2023
2167
|
# ===== GETTING ALL FILES =====
|
|
2024
2168
|
all_files = []
|
|
2025
2169
|
|
|
2026
|
-
# MAIN VSI
|
|
2027
2170
|
all_files.append(slide_path)
|
|
2028
2171
|
|
|
2029
2172
|
vsi_name_no_ext = os.path.splitext(os.path.basename(slide_path))[0]
|
|
@@ -2073,7 +2216,7 @@ async def upload_large_files(
|
|
|
2073
2216
|
)
|
|
2074
2217
|
|
|
2075
2218
|
if not ok:
|
|
2076
|
-
return False, f"
|
|
2219
|
+
return False, f"Upload failed: {err}"
|
|
2077
2220
|
|
|
2078
2221
|
# ===== STACK =====
|
|
2079
2222
|
if os.path.isdir(local_stack_root):
|
|
@@ -2105,6 +2248,7 @@ async def upload_large_files(
|
|
|
2105
2248
|
|
|
2106
2249
|
return True, None
|
|
2107
2250
|
|
|
2251
|
+
|
|
2108
2252
|
async def large_files_upload_package(
|
|
2109
2253
|
*,
|
|
2110
2254
|
pma_core_url: str,
|
|
@@ -2113,6 +2257,15 @@ async def large_files_upload_package(
|
|
|
2113
2257
|
upload_directory: str,
|
|
2114
2258
|
progress_callback=None,
|
|
2115
2259
|
) -> tuple[bool, str | None]:
|
|
2260
|
+
"""
|
|
2261
|
+
Low-level async upload routine for a single file.
|
|
2262
|
+
|
|
2263
|
+
Performs:
|
|
2264
|
+
- upload header request
|
|
2265
|
+
- multipart or direct upload
|
|
2266
|
+
- upload completion polling
|
|
2267
|
+
"""
|
|
2268
|
+
|
|
2116
2269
|
try:
|
|
2117
2270
|
client = PmaCoreClient(pma_core_url)
|
|
2118
2271
|
print("trust_env:", client.session.trust_env)
|
|
@@ -2138,11 +2291,11 @@ async def large_files_upload_package(
|
|
|
2138
2291
|
# ===== SEND HEADER =====
|
|
2139
2292
|
resp = await client.upload_header(header, session_id)
|
|
2140
2293
|
|
|
2141
|
-
print("UploadType:", resp.upload_type)
|
|
2294
|
+
# print("UploadType:", resp.upload_type)
|
|
2142
2295
|
print("MultipartFiles:", resp.multipart_files)
|
|
2143
2296
|
print("Urls:", resp.urls)
|
|
2144
2297
|
|
|
2145
|
-
# ===== MULTIPART =====
|
|
2298
|
+
# ===== MULTIPART ONLY IF SERVER SENT PARTS =====
|
|
2146
2299
|
if resp.multipart_files and len(resp.multipart_files) > 0:
|
|
2147
2300
|
|
|
2148
2301
|
mp = resp.multipart_files[0]
|
|
@@ -2152,7 +2305,6 @@ async def large_files_upload_package(
|
|
|
2152
2305
|
)
|
|
2153
2306
|
|
|
2154
2307
|
with open(slide_path, "rb") as stream:
|
|
2155
|
-
|
|
2156
2308
|
await client.upload_multipart_file_to_s3(
|
|
2157
2309
|
mp,
|
|
2158
2310
|
full_virtual_path,
|
|
@@ -2161,13 +2313,12 @@ async def large_files_upload_package(
|
|
|
2161
2313
|
progress_callback=progress_callback
|
|
2162
2314
|
)
|
|
2163
2315
|
|
|
2164
|
-
# ===== SINGLE PUT =====
|
|
2316
|
+
# ===== SINGLE PUT / AZURE / FILESYSTEM =====
|
|
2165
2317
|
else:
|
|
2166
2318
|
|
|
2167
2319
|
upload_url = resp.urls[0] if resp.urls else None
|
|
2168
2320
|
|
|
2169
2321
|
with open(slide_path, "rb") as stream:
|
|
2170
|
-
|
|
2171
2322
|
await client.upload_file(
|
|
2172
2323
|
resp.id,
|
|
2173
2324
|
resp.upload_type,
|
|
@@ -2193,7 +2344,14 @@ async def large_files_upload_package(
|
|
|
2193
2344
|
except Exception as e:
|
|
2194
2345
|
return False, str(e)
|
|
2195
2346
|
|
|
2347
|
+
|
|
2196
2348
|
# callback for large files upload
|
|
2197
2349
|
def on_progress(bytes_sent, total_bytes):
|
|
2350
|
+
"""
|
|
2351
|
+
Default progress callback.
|
|
2352
|
+
|
|
2353
|
+
Prints upload percentage based on transferred bytes.
|
|
2354
|
+
"""
|
|
2355
|
+
|
|
2198
2356
|
percent = bytes_sent / total_bytes * 100
|
|
2199
2357
|
print(f"Upload progress: {percent:.2f}%")
|
|
@@ -11,12 +11,10 @@ from urllib3.poolmanager import PoolManager
|
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
UploadProgressCallback = Callable[[int, int], None]
|
|
14
|
-
# args: bytes_sent, total_bytes
|
|
15
14
|
|
|
16
15
|
class TLS12HttpAdapter(HTTPAdapter):
|
|
17
16
|
def init_poolmanager(self, connections, maxsize, block=False, **pool_kwargs):
|
|
18
17
|
ctx = ssl.create_default_context()
|
|
19
|
-
# Принудительно TLS 1.2 (часто лечит EOF на прокси/антивирусе)
|
|
20
18
|
ctx.minimum_version = ssl.TLSVersion.TLSv1_2
|
|
21
19
|
ctx.maximum_version = ssl.TLSVersion.TLSv1_2
|
|
22
20
|
|
|
@@ -176,12 +174,10 @@ class PmaCoreClient:
|
|
|
176
174
|
if not self.server_url.endswith('/'):
|
|
177
175
|
self.server_url += '/'
|
|
178
176
|
|
|
179
|
-
# --- FIX: не использовать proxy/SSL-intercept из окружения ---
|
|
180
177
|
self.session = requests.Session()
|
|
181
|
-
self.session.trust_env = False
|
|
178
|
+
self.session.trust_env = False
|
|
182
179
|
self.session.mount("https://", TLS12HttpAdapter())
|
|
183
180
|
|
|
184
|
-
|
|
185
181
|
def authenticate(self, username: str, password: str, caller: str) -> AuthenticateResponse:
|
|
186
182
|
service_url = urljoin(self.server_url, 'api/json/Authenticate')
|
|
187
183
|
params = {'username': username, 'password': password, 'caller': caller}
|
|
@@ -209,8 +205,13 @@ class PmaCoreClient:
|
|
|
209
205
|
total_bytes: int | None = None,
|
|
210
206
|
progress_callback: UploadProgressCallback | None = None):
|
|
211
207
|
|
|
212
|
-
if upload_type == "Azure":
|
|
213
|
-
await self.upload_file_to_azure(
|
|
208
|
+
if upload_type == "Azure" or upload_type == 2:
|
|
209
|
+
await self.upload_file_to_azure(
|
|
210
|
+
upload_url,
|
|
211
|
+
stream,
|
|
212
|
+
total_bytes=total_bytes,
|
|
213
|
+
progress_callback=progress_callback,
|
|
214
|
+
)
|
|
214
215
|
return
|
|
215
216
|
|
|
216
217
|
upload_to_presigned = bool(upload_url)
|
|
@@ -248,11 +249,9 @@ class PmaCoreClient:
|
|
|
248
249
|
|
|
249
250
|
return
|
|
250
251
|
|
|
251
|
-
# upload через сервер (multipart form)
|
|
252
252
|
url = f"{self.server_url}transfer/Upload/{upload_id}"
|
|
253
253
|
params = {"sessionId": session_id, "path": relative_path}
|
|
254
254
|
|
|
255
|
-
# важно: wrapped передаём как file handle в files
|
|
256
255
|
files = {"file": (relative_path.split("/")[-1], wrapped, "application/octet-stream")}
|
|
257
256
|
response = await asyncio.to_thread(
|
|
258
257
|
requests.post,
|
|
@@ -285,7 +284,7 @@ class PmaCoreClient:
|
|
|
285
284
|
params = {'sessionId': session_id}
|
|
286
285
|
payload = {
|
|
287
286
|
"FilePath": relative_path,
|
|
288
|
-
"UploadId": multipart_info.upload_id,
|
|
287
|
+
"UploadId": multipart_info.upload_id,
|
|
289
288
|
"Parts": [{"PartNumber": p.PartNumber, "ETag": p.ETag} for p in e_tags],
|
|
290
289
|
}
|
|
291
290
|
headers = {'Content-Type': 'application/json'}
|
|
@@ -299,9 +298,21 @@ class PmaCoreClient:
|
|
|
299
298
|
)
|
|
300
299
|
response.raise_for_status()
|
|
301
300
|
|
|
302
|
-
async def upload_file_to_azure(
|
|
301
|
+
async def upload_file_to_azure(
|
|
302
|
+
self,
|
|
303
|
+
upload_url: str,
|
|
304
|
+
stream: IO,
|
|
305
|
+
total_bytes: int | None = None,
|
|
306
|
+
progress_callback: UploadProgressCallback | None = None,
|
|
307
|
+
):
|
|
303
308
|
block_ids: List[str] = []
|
|
304
|
-
await self.upload_blocks_to_azure(
|
|
309
|
+
await self.upload_blocks_to_azure(
|
|
310
|
+
block_ids,
|
|
311
|
+
upload_url,
|
|
312
|
+
stream,
|
|
313
|
+
total_bytes=total_bytes,
|
|
314
|
+
progress_callback=progress_callback,
|
|
315
|
+
)
|
|
305
316
|
headers = {
|
|
306
317
|
'x-ms-blob-content-type': 'application/octet-stream',
|
|
307
318
|
'Connection': 'Keep-Alive',
|
|
@@ -321,11 +332,17 @@ class PmaCoreClient:
|
|
|
321
332
|
response = requests.put(commit_url, data=body, headers=headers)
|
|
322
333
|
response.raise_for_status()
|
|
323
334
|
|
|
324
|
-
async def upload_blocks_to_azure(
|
|
325
|
-
|
|
335
|
+
async def upload_blocks_to_azure(
|
|
336
|
+
self,
|
|
337
|
+
block_ids: List[str],
|
|
338
|
+
upload_url: str,
|
|
339
|
+
stream: IO,
|
|
340
|
+
total_bytes: int | None = None,
|
|
341
|
+
progress_callback: UploadProgressCallback | None = None,
|
|
342
|
+
):
|
|
343
|
+
block_size = 4 * 1024 * 1024
|
|
326
344
|
block_id = 0
|
|
327
|
-
|
|
328
|
-
# если в upload_url есть пробелы/и т.п.
|
|
345
|
+
sent_total = 0
|
|
329
346
|
safe_base = upload_url.replace(" ", "%20")
|
|
330
347
|
|
|
331
348
|
while True:
|
|
@@ -333,21 +350,19 @@ class PmaCoreClient:
|
|
|
333
350
|
if not chunk:
|
|
334
351
|
break
|
|
335
352
|
|
|
353
|
+
sent_total += len(chunk)
|
|
354
|
+
if progress_callback and total_bytes:
|
|
355
|
+
progress_callback(sent_total, total_bytes)
|
|
356
|
+
|
|
336
357
|
plain = (f"block_{block_id:06d}").encode("utf-8")
|
|
337
358
|
import base64
|
|
338
359
|
base64_block_id = base64.b64encode(plain).decode("utf-8")
|
|
339
|
-
|
|
340
|
-
# ВАЖНО: blockid обязательно URL-encode
|
|
341
360
|
encoded_block_id = quote(base64_block_id, safe="")
|
|
342
|
-
|
|
343
361
|
block_upload_url = f"{safe_base}&comp=block&blockid={encoded_block_id}"
|
|
344
362
|
|
|
345
|
-
# Для block PUT не надо x-ms-blob-type (особенно если URL — SAS)
|
|
346
363
|
headers = {
|
|
347
364
|
"Connection": "Keep-Alive",
|
|
348
365
|
"Keep-Alive": "3600",
|
|
349
|
-
# при необходимости можно добавить x-ms-version, если требует аккаунт:
|
|
350
|
-
# "x-ms-version": "2020-10-02",
|
|
351
366
|
}
|
|
352
367
|
|
|
353
368
|
response = await asyncio.to_thread(
|
|
@@ -358,7 +373,7 @@ class PmaCoreClient:
|
|
|
358
373
|
)
|
|
359
374
|
response.raise_for_status()
|
|
360
375
|
|
|
361
|
-
block_ids.append(base64_block_id)
|
|
376
|
+
block_ids.append(base64_block_id)
|
|
362
377
|
block_id += 1
|
|
363
378
|
|
|
364
379
|
def _iter_file_range(self, stream: IO, start: int, length: int, buf_size: int = 8 * 1024 * 1024):
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = '2.0.0.182'
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = '2.0.0.180'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|