supervisely 6.73.320__py3-none-any.whl → 6.73.321__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- supervisely/_utils.py +33 -0
- supervisely/api/api.py +17 -13
- supervisely/api/file_api.py +158 -25
- supervisely/io/fs.py +81 -4
- supervisely/nn/training/train_app.py +48 -36
- {supervisely-6.73.320.dist-info → supervisely-6.73.321.dist-info}/METADATA +1 -1
- {supervisely-6.73.320.dist-info → supervisely-6.73.321.dist-info}/RECORD +11 -11
- {supervisely-6.73.320.dist-info → supervisely-6.73.321.dist-info}/LICENSE +0 -0
- {supervisely-6.73.320.dist-info → supervisely-6.73.321.dist-info}/WHEEL +0 -0
- {supervisely-6.73.320.dist-info → supervisely-6.73.321.dist-info}/entry_points.txt +0 -0
- {supervisely-6.73.320.dist-info → supervisely-6.73.321.dist-info}/top_level.txt +0 -0
supervisely/_utils.py
CHANGED
|
@@ -471,6 +471,39 @@ def get_or_create_event_loop() -> asyncio.AbstractEventLoop:
|
|
|
471
471
|
return loop
|
|
472
472
|
|
|
473
473
|
|
|
474
|
+
def sync_call(coro):
|
|
475
|
+
"""
|
|
476
|
+
This function is used to run asynchronous functions in synchronous context.
|
|
477
|
+
|
|
478
|
+
:param coro: Asynchronous function.
|
|
479
|
+
:type coro: Coroutine
|
|
480
|
+
:return: Result of the asynchronous function.
|
|
481
|
+
:rtype: Any
|
|
482
|
+
|
|
483
|
+
:Usage example:
|
|
484
|
+
|
|
485
|
+
.. code-block:: python
|
|
486
|
+
|
|
487
|
+
from supervisely.utils import sync_call
|
|
488
|
+
|
|
489
|
+
async def async_function():
|
|
490
|
+
await asyncio.sleep(1)
|
|
491
|
+
return "Hello, World!"
|
|
492
|
+
coro = async_function()
|
|
493
|
+
result = sync_call(coro)
|
|
494
|
+
print(result)
|
|
495
|
+
# Output: Hello, World!
|
|
496
|
+
"""
|
|
497
|
+
|
|
498
|
+
loop = get_or_create_event_loop()
|
|
499
|
+
|
|
500
|
+
if loop.is_running():
|
|
501
|
+
future = asyncio.run_coroutine_threadsafe(coro, loop=loop)
|
|
502
|
+
return future.result()
|
|
503
|
+
else:
|
|
504
|
+
return loop.run_until_complete(coro)
|
|
505
|
+
|
|
506
|
+
|
|
474
507
|
def get_filename_from_headers(url):
|
|
475
508
|
try:
|
|
476
509
|
response = requests.head(url, allow_redirects=True)
|
supervisely/api/api.py
CHANGED
|
@@ -1443,6 +1443,7 @@ class Api:
|
|
|
1443
1443
|
chunk_size: int = 8192,
|
|
1444
1444
|
use_public_api: Optional[bool] = True,
|
|
1445
1445
|
timeout: httpx._types.TimeoutTypes = 60,
|
|
1446
|
+
**kwargs,
|
|
1446
1447
|
) -> AsyncGenerator:
|
|
1447
1448
|
"""
|
|
1448
1449
|
Performs asynchronous streaming GET or POST request to server with given parameters.
|
|
@@ -1486,18 +1487,19 @@ class Api:
|
|
|
1486
1487
|
else:
|
|
1487
1488
|
headers = {**self.headers, **headers}
|
|
1488
1489
|
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
elif isinstance(data, Dict):
|
|
1494
|
-
json_body = {**data, **self.additional_fields}
|
|
1495
|
-
content = None
|
|
1496
|
-
params = None
|
|
1490
|
+
params = kwargs.get("params", None)
|
|
1491
|
+
if "content" in kwargs or "json_body" in kwargs:
|
|
1492
|
+
content = kwargs.get("content", None)
|
|
1493
|
+
json_body = kwargs.get("json_body", None)
|
|
1497
1494
|
else:
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1495
|
+
if isinstance(data, (bytes, Generator)):
|
|
1496
|
+
content = data
|
|
1497
|
+
json_body = None
|
|
1498
|
+
elif isinstance(data, Dict):
|
|
1499
|
+
json_body = {**data, **self.additional_fields}
|
|
1500
|
+
content = None
|
|
1501
|
+
else:
|
|
1502
|
+
raise ValueError("Data should be either bytes or dict")
|
|
1501
1503
|
|
|
1502
1504
|
if range_start is not None or range_end is not None:
|
|
1503
1505
|
headers["Range"] = f"bytes={range_start or ''}-{range_end or ''}"
|
|
@@ -1512,17 +1514,19 @@ class Api:
|
|
|
1512
1514
|
url,
|
|
1513
1515
|
content=content,
|
|
1514
1516
|
json=json_body,
|
|
1515
|
-
params=params,
|
|
1516
1517
|
headers=headers,
|
|
1517
1518
|
timeout=timeout,
|
|
1519
|
+
params=params,
|
|
1518
1520
|
)
|
|
1519
1521
|
elif method_type == "GET":
|
|
1520
1522
|
response = self.async_httpx_client.stream(
|
|
1521
1523
|
method_type,
|
|
1522
1524
|
url,
|
|
1523
|
-
|
|
1525
|
+
content=content,
|
|
1526
|
+
json=json_body,
|
|
1524
1527
|
headers=headers,
|
|
1525
1528
|
timeout=timeout,
|
|
1529
|
+
params=params,
|
|
1526
1530
|
)
|
|
1527
1531
|
else:
|
|
1528
1532
|
raise NotImplementedError(
|
supervisely/api/file_api.py
CHANGED
|
@@ -33,7 +33,9 @@ from supervisely.io.fs import (
|
|
|
33
33
|
get_file_name,
|
|
34
34
|
get_file_name_with_ext,
|
|
35
35
|
get_file_size,
|
|
36
|
+
get_or_create_event_loop,
|
|
36
37
|
list_files_recursively,
|
|
38
|
+
list_files_recursively_async,
|
|
37
39
|
silent_remove,
|
|
38
40
|
)
|
|
39
41
|
from supervisely.io.fs_cache import FileCache
|
|
@@ -2041,7 +2043,7 @@ class FileApi(ModuleApiBase):
|
|
|
2041
2043
|
# check_hash: bool = True, #TODO add with resumaple api
|
|
2042
2044
|
progress_cb: Optional[Union[tqdm, Callable]] = None,
|
|
2043
2045
|
progress_cb_type: Literal["number", "size"] = "size",
|
|
2044
|
-
) ->
|
|
2046
|
+
) -> None:
|
|
2045
2047
|
"""
|
|
2046
2048
|
Upload file from local path to Team Files asynchronously.
|
|
2047
2049
|
|
|
@@ -2057,8 +2059,8 @@ class FileApi(ModuleApiBase):
|
|
|
2057
2059
|
:type progress_cb: tqdm or callable, optional
|
|
2058
2060
|
:param progress_cb_type: Type of progress callback. Can be "number" or "size". Default is "size".
|
|
2059
2061
|
:type progress_cb_type: Literal["number", "size"], optional
|
|
2060
|
-
:return:
|
|
2061
|
-
:rtype: :class:`
|
|
2062
|
+
:return: None
|
|
2063
|
+
:rtype: :class:`NoneType`
|
|
2062
2064
|
:Usage example:
|
|
2063
2065
|
|
|
2064
2066
|
.. code-block:: python
|
|
@@ -2087,17 +2089,30 @@ class FileApi(ModuleApiBase):
|
|
|
2087
2089
|
}
|
|
2088
2090
|
if semaphore is None:
|
|
2089
2091
|
semaphore = self._api.get_default_semaphore()
|
|
2092
|
+
logger.debug(f"Uploading with async to: {dst}. Semaphore: {semaphore}")
|
|
2090
2093
|
async with semaphore:
|
|
2091
2094
|
async with aiofiles.open(src, "rb") as fd:
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2095
|
+
|
|
2096
|
+
async def file_chunk_generator():
|
|
2097
|
+
while True:
|
|
2098
|
+
chunk = await fd.read(8 * 1024 * 1024)
|
|
2099
|
+
if not chunk:
|
|
2100
|
+
break
|
|
2101
|
+
if progress_cb is not None and progress_cb_type == "size":
|
|
2102
|
+
progress_cb(len(chunk))
|
|
2103
|
+
yield chunk
|
|
2104
|
+
|
|
2105
|
+
async for chunk, _ in self._api.stream_async(
|
|
2106
|
+
method=api_method,
|
|
2107
|
+
method_type="POST",
|
|
2108
|
+
data=file_chunk_generator(), # added as required, but not used inside
|
|
2109
|
+
headers=headers,
|
|
2110
|
+
content=file_chunk_generator(), # used instead of data inside stream_async
|
|
2111
|
+
params=json_body,
|
|
2112
|
+
):
|
|
2113
|
+
pass
|
|
2098
2114
|
if progress_cb is not None and progress_cb_type == "number":
|
|
2099
2115
|
progress_cb(1)
|
|
2100
|
-
return response
|
|
2101
2116
|
|
|
2102
2117
|
async def upload_bulk_async(
|
|
2103
2118
|
self,
|
|
@@ -2109,6 +2124,7 @@ class FileApi(ModuleApiBase):
|
|
|
2109
2124
|
# check_hash: bool = True, #TODO add with resumaple api
|
|
2110
2125
|
progress_cb: Optional[Union[tqdm, Callable]] = None,
|
|
2111
2126
|
progress_cb_type: Literal["number", "size"] = "size",
|
|
2127
|
+
enable_fallback: Optional[bool] = True,
|
|
2112
2128
|
) -> None:
|
|
2113
2129
|
"""
|
|
2114
2130
|
Upload multiple files from local paths to Team Files asynchronously.
|
|
@@ -2125,6 +2141,8 @@ class FileApi(ModuleApiBase):
|
|
|
2125
2141
|
:type progress_cb: tqdm or callable, optional
|
|
2126
2142
|
:param progress_cb_type: Type of progress callback. Can be "number" or "size". Default is "size".
|
|
2127
2143
|
:type progress_cb_type: Literal["number", "size"], optional
|
|
2144
|
+
:param enable_fallback: If True, the method will fallback to synchronous upload if an error occurs.
|
|
2145
|
+
:type enable_fallback: bool, optional
|
|
2128
2146
|
:return: None
|
|
2129
2147
|
:rtype: :class:`NoneType`
|
|
2130
2148
|
:Usage example:
|
|
@@ -2153,19 +2171,134 @@ class FileApi(ModuleApiBase):
|
|
|
2153
2171
|
api.file.upload_bulk_async(8, paths_to_files, paths_to_save)
|
|
2154
2172
|
)
|
|
2155
2173
|
"""
|
|
2156
|
-
|
|
2157
|
-
semaphore
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2174
|
+
try:
|
|
2175
|
+
if semaphore is None:
|
|
2176
|
+
semaphore = self._api.get_default_semaphore()
|
|
2177
|
+
tasks = []
|
|
2178
|
+
for src, dst in zip(src_paths, dst_paths):
|
|
2179
|
+
task = asyncio.create_task(
|
|
2180
|
+
self.upload_async(
|
|
2181
|
+
team_id=team_id,
|
|
2182
|
+
src=src,
|
|
2183
|
+
dst=dst,
|
|
2184
|
+
semaphore=semaphore,
|
|
2185
|
+
# chunk_size=chunk_size, #TODO add with resumaple api
|
|
2186
|
+
# check_hash=check_hash, #TODO add with resumaple api
|
|
2187
|
+
progress_cb=progress_cb,
|
|
2188
|
+
progress_cb_type=progress_cb_type,
|
|
2189
|
+
)
|
|
2190
|
+
)
|
|
2191
|
+
tasks.append(task)
|
|
2192
|
+
for task in tasks:
|
|
2193
|
+
await task
|
|
2194
|
+
except Exception as e:
|
|
2195
|
+
if enable_fallback:
|
|
2196
|
+
logger.warning(
|
|
2197
|
+
f"Upload files bulk asynchronously failed. Fallback to synchronous upload.",
|
|
2198
|
+
exc_info=True,
|
|
2199
|
+
)
|
|
2200
|
+
if progress_cb is not None and progress_cb_type == "number":
|
|
2201
|
+
logger.warning(
|
|
2202
|
+
"Progress callback type 'number' is not supported for synchronous upload. "
|
|
2203
|
+
"Progress callback will be disabled."
|
|
2204
|
+
)
|
|
2205
|
+
progress_cb = None
|
|
2206
|
+
self.upload_bulk(
|
|
2207
|
+
team_id=team_id,
|
|
2208
|
+
src_paths=src_paths,
|
|
2209
|
+
dst_paths=dst_paths,
|
|
2210
|
+
progress_cb=progress_cb,
|
|
2211
|
+
)
|
|
2212
|
+
else:
|
|
2213
|
+
raise e
|
|
2214
|
+
|
|
2215
|
+
async def upload_directory_async(
|
|
2216
|
+
self,
|
|
2217
|
+
team_id: int,
|
|
2218
|
+
local_dir: str,
|
|
2219
|
+
remote_dir: str,
|
|
2220
|
+
change_name_if_conflict: Optional[bool] = True,
|
|
2221
|
+
progress_size_cb: Optional[Union[tqdm, Callable]] = None,
|
|
2222
|
+
replace_if_conflict: Optional[bool] = False,
|
|
2223
|
+
enable_fallback: Optional[bool] = True,
|
|
2224
|
+
) -> str:
|
|
2225
|
+
"""
|
|
2226
|
+
Upload Directory to Team Files from local path.
|
|
2227
|
+
Files are uploaded asynchronously.
|
|
2228
|
+
|
|
2229
|
+
:param team_id: Team ID in Supervisely.
|
|
2230
|
+
:type team_id: int
|
|
2231
|
+
:param local_dir: Path to local Directory.
|
|
2232
|
+
:type local_dir: str
|
|
2233
|
+
:param remote_dir: Path to Directory in Team Files.
|
|
2234
|
+
:type remote_dir: str
|
|
2235
|
+
:param change_name_if_conflict: Checks if given name already exists and adds suffix to the end of the name.
|
|
2236
|
+
:type change_name_if_conflict: bool, optional
|
|
2237
|
+
:param progress_size_cb: Function for tracking download progress.
|
|
2238
|
+
:type progress_size_cb: Progress, optional
|
|
2239
|
+
:param replace_if_conflict: If True, replace existing dir.
|
|
2240
|
+
:type replace_if_conflict: bool, optional
|
|
2241
|
+
:param enable_fallback: If True, the method will fallback to synchronous upload if an error occurs.
|
|
2242
|
+
:type enable_fallback: bool, optional
|
|
2243
|
+
:return: Path to Directory in Team Files
|
|
2244
|
+
:rtype: :class:`str`
|
|
2245
|
+
:Usage example:
|
|
2246
|
+
|
|
2247
|
+
.. code-block:: python
|
|
2248
|
+
|
|
2249
|
+
import supervisely as sly
|
|
2250
|
+
|
|
2251
|
+
os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
|
|
2252
|
+
os.environ['API_TOKEN'] = 'Your Supervisely API Token'
|
|
2253
|
+
api = sly.Api.from_env()
|
|
2254
|
+
|
|
2255
|
+
path_to_dir = "/My_App_Test/ds1"
|
|
2256
|
+
local_path = "/home/admin/Downloads/My_local_test"
|
|
2257
|
+
|
|
2258
|
+
api.file.upload_directory(9, local_path, path_to_dir)
|
|
2259
|
+
"""
|
|
2260
|
+
try:
|
|
2261
|
+
if not remote_dir.startswith("/"):
|
|
2262
|
+
remote_dir = "/" + remote_dir
|
|
2263
|
+
|
|
2264
|
+
if self.dir_exists(team_id, remote_dir):
|
|
2265
|
+
if change_name_if_conflict is True:
|
|
2266
|
+
res_remote_dir = self.get_free_dir_name(team_id, remote_dir)
|
|
2267
|
+
elif replace_if_conflict is True:
|
|
2268
|
+
res_remote_dir = remote_dir
|
|
2269
|
+
else:
|
|
2270
|
+
raise FileExistsError(
|
|
2271
|
+
f"Directory {remote_dir} already exists in your team (id={team_id})"
|
|
2272
|
+
)
|
|
2273
|
+
else:
|
|
2274
|
+
res_remote_dir = remote_dir
|
|
2275
|
+
|
|
2276
|
+
local_files = await list_files_recursively_async(local_dir)
|
|
2277
|
+
dir_prefix = local_dir.rstrip("/") + "/"
|
|
2278
|
+
remote_files = [
|
|
2279
|
+
res_remote_dir.rstrip("/") + "/" + file[len(dir_prefix) :] for file in local_files
|
|
2280
|
+
]
|
|
2281
|
+
|
|
2282
|
+
await self.upload_bulk_async(
|
|
2283
|
+
team_id=team_id,
|
|
2284
|
+
src_paths=local_files,
|
|
2285
|
+
dst_paths=remote_files,
|
|
2286
|
+
progress_cb=progress_size_cb,
|
|
2169
2287
|
)
|
|
2170
|
-
|
|
2171
|
-
|
|
2288
|
+
except Exception as e:
|
|
2289
|
+
if enable_fallback:
|
|
2290
|
+
logger.warning(
|
|
2291
|
+
f"Upload directory asynchronously failed. Fallback to synchronous upload.",
|
|
2292
|
+
exc_info=True,
|
|
2293
|
+
)
|
|
2294
|
+
res_remote_dir = self.upload_directory(
|
|
2295
|
+
team_id=team_id,
|
|
2296
|
+
local_dir=local_dir,
|
|
2297
|
+
remote_dir=res_remote_dir,
|
|
2298
|
+
change_name_if_conflict=change_name_if_conflict,
|
|
2299
|
+
progress_size_cb=progress_size_cb,
|
|
2300
|
+
replace_if_conflict=replace_if_conflict,
|
|
2301
|
+
)
|
|
2302
|
+
else:
|
|
2303
|
+
raise e
|
|
2304
|
+
return res_remote_dir
|
supervisely/io/fs.py
CHANGED
|
@@ -205,15 +205,19 @@ def list_files_recursively(
|
|
|
205
205
|
for filename in file_names:
|
|
206
206
|
yield os.path.join(dir_name, filename)
|
|
207
207
|
|
|
208
|
-
valid_extensions =
|
|
208
|
+
valid_extensions = (
|
|
209
|
+
valid_extensions
|
|
210
|
+
if ignore_valid_extensions_case is False
|
|
211
|
+
else [ext.lower() for ext in valid_extensions]
|
|
212
|
+
)
|
|
209
213
|
files = []
|
|
210
214
|
for file_path in file_path_generator():
|
|
211
215
|
file_ext = get_file_ext(file_path)
|
|
212
216
|
if ignore_valid_extensions_case:
|
|
213
217
|
file_ext.lower()
|
|
214
|
-
if (
|
|
215
|
-
|
|
216
|
-
)
|
|
218
|
+
if (valid_extensions is None or file_ext in valid_extensions) and (
|
|
219
|
+
filter_fn is None or filter_fn(file_path)
|
|
220
|
+
):
|
|
217
221
|
files.append(file_path)
|
|
218
222
|
return files
|
|
219
223
|
|
|
@@ -1558,3 +1562,76 @@ async def touch_async(path: str) -> None:
|
|
|
1558
1562
|
async with aiofiles.open(path, "a"):
|
|
1559
1563
|
loop = get_or_create_event_loop()
|
|
1560
1564
|
await loop.run_in_executor(None, os.utime, path, None)
|
|
1565
|
+
|
|
1566
|
+
|
|
1567
|
+
async def list_files_recursively_async(
|
|
1568
|
+
dir_path: str,
|
|
1569
|
+
valid_extensions: Optional[List[str]] = None,
|
|
1570
|
+
filter_fn: Optional[Callable[[str], bool]] = None,
|
|
1571
|
+
ignore_valid_extensions_case: bool = False,
|
|
1572
|
+
) -> List[str]:
|
|
1573
|
+
"""
|
|
1574
|
+
Recursively list files in the directory asynchronously.
|
|
1575
|
+
Returns list with all file paths.
|
|
1576
|
+
Can be filtered by valid extensions and filter function.
|
|
1577
|
+
|
|
1578
|
+
:param dir_path: Target directory path.
|
|
1579
|
+
:type dir_path: str
|
|
1580
|
+
:param valid_extensions: List of valid extensions. Default is None.
|
|
1581
|
+
:type valid_extensions: Optional[List[str]]
|
|
1582
|
+
:param filter_fn: Filter function. Default is None.
|
|
1583
|
+
:type filter_fn: Optional[Callable[[str], bool]]
|
|
1584
|
+
:param ignore_valid_extensions_case: Ignore case when checking valid extensions. Default is False.
|
|
1585
|
+
:type ignore_valid_extensions_case: bool
|
|
1586
|
+
:returns: List of file paths
|
|
1587
|
+
:rtype: List[str]
|
|
1588
|
+
|
|
1589
|
+
:Usage example:
|
|
1590
|
+
|
|
1591
|
+
.. code-block:: python
|
|
1592
|
+
|
|
1593
|
+
import supervisely as sly
|
|
1594
|
+
|
|
1595
|
+
dir_path = '/home/admin/work/projects/examples'
|
|
1596
|
+
loop = sly.utils.get_or_create_event_loop()
|
|
1597
|
+
coro = sly.fs.list_files_recursively_async(dir_path)
|
|
1598
|
+
if loop.is_running():
|
|
1599
|
+
future = asyncio.run_coroutine_threadsafe(coro, loop)
|
|
1600
|
+
files = future.result()
|
|
1601
|
+
else:
|
|
1602
|
+
files = loop.run_until_complete(coro)
|
|
1603
|
+
"""
|
|
1604
|
+
|
|
1605
|
+
def sync_file_list():
|
|
1606
|
+
if valid_extensions and ignore_valid_extensions_case:
|
|
1607
|
+
valid_extensions_set = set(map(str.lower, valid_extensions))
|
|
1608
|
+
else:
|
|
1609
|
+
valid_extensions_set = set(valid_extensions) if valid_extensions else None
|
|
1610
|
+
|
|
1611
|
+
files = []
|
|
1612
|
+
for dir_name, _, file_names in os.walk(dir_path):
|
|
1613
|
+
full_paths = [os.path.join(dir_name, filename) for filename in file_names]
|
|
1614
|
+
|
|
1615
|
+
if valid_extensions_set:
|
|
1616
|
+
full_paths = [
|
|
1617
|
+
fp
|
|
1618
|
+
for fp in full_paths
|
|
1619
|
+
if (
|
|
1620
|
+
ext := (
|
|
1621
|
+
os.path.splitext(fp)[1].lower()
|
|
1622
|
+
if ignore_valid_extensions_case
|
|
1623
|
+
else os.path.splitext(fp)[1]
|
|
1624
|
+
)
|
|
1625
|
+
)
|
|
1626
|
+
in valid_extensions_set
|
|
1627
|
+
]
|
|
1628
|
+
|
|
1629
|
+
if filter_fn:
|
|
1630
|
+
full_paths = [fp for fp in full_paths if filter_fn(fp)]
|
|
1631
|
+
|
|
1632
|
+
files.extend(full_paths)
|
|
1633
|
+
|
|
1634
|
+
return files
|
|
1635
|
+
|
|
1636
|
+
loop = get_or_create_event_loop()
|
|
1637
|
+
return await loop.run_in_executor(None, sync_file_list)
|
|
@@ -39,7 +39,7 @@ from supervisely import (
|
|
|
39
39
|
is_production,
|
|
40
40
|
logger,
|
|
41
41
|
)
|
|
42
|
-
from supervisely._utils import abs_url, get_filename_from_headers
|
|
42
|
+
from supervisely._utils import abs_url, get_filename_from_headers, sync_call
|
|
43
43
|
from supervisely.api.file_api import FileInfo
|
|
44
44
|
from supervisely.app import get_synced_data_dir
|
|
45
45
|
from supervisely.app.widgets import Progress
|
|
@@ -60,6 +60,7 @@ from supervisely.nn.utils import ModelSource
|
|
|
60
60
|
from supervisely.output import set_directory
|
|
61
61
|
from supervisely.project.download import (
|
|
62
62
|
copy_from_cache,
|
|
63
|
+
download_async_or_sync,
|
|
63
64
|
download_to_cache,
|
|
64
65
|
get_cache_size,
|
|
65
66
|
is_cached,
|
|
@@ -803,8 +804,9 @@ class TrainApp:
|
|
|
803
804
|
:type total_images: int
|
|
804
805
|
"""
|
|
805
806
|
with self.progress_bar_main(message="Downloading input data", total=total_images) as pbar:
|
|
807
|
+
logger.debug("Downloading project data without cache")
|
|
806
808
|
self.progress_bar_main.show()
|
|
807
|
-
|
|
809
|
+
download_async_or_sync(
|
|
808
810
|
api=self._api,
|
|
809
811
|
project_id=self.project_id,
|
|
810
812
|
dest_dir=self.project_dir,
|
|
@@ -834,6 +836,7 @@ class TrainApp:
|
|
|
834
836
|
|
|
835
837
|
logger.info(self._get_cache_log_message(cached, to_download))
|
|
836
838
|
with self.progress_bar_main(message="Downloading input data", total=total_images) as pbar:
|
|
839
|
+
logger.debug("Downloading project data with cache")
|
|
837
840
|
self.progress_bar_main.show()
|
|
838
841
|
download_to_cache(
|
|
839
842
|
api=self._api,
|
|
@@ -1603,10 +1606,12 @@ class TrainApp:
|
|
|
1603
1606
|
logger.info(f"Demo directory '{local_demo_dir}' does not exist")
|
|
1604
1607
|
return
|
|
1605
1608
|
|
|
1606
|
-
logger.debug(f"Uploading demo files to Supervisely")
|
|
1607
1609
|
remote_demo_dir = join(remote_dir, "demo")
|
|
1608
1610
|
local_files = sly_fs.list_files_recursively(local_demo_dir)
|
|
1609
1611
|
total_size = sum([sly_fs.get_file_size(file_path) for file_path in local_files])
|
|
1612
|
+
logger.debug(
|
|
1613
|
+
f"Uploading demo files of total size {total_size} bytes to Supervisely Team Files directory '{remote_demo_dir}'"
|
|
1614
|
+
)
|
|
1610
1615
|
with self.progress_bar_main(
|
|
1611
1616
|
message="Uploading demo files to Team Files",
|
|
1612
1617
|
total=total_size,
|
|
@@ -1614,11 +1619,13 @@ class TrainApp:
|
|
|
1614
1619
|
unit_scale=True,
|
|
1615
1620
|
) as upload_artifacts_pbar:
|
|
1616
1621
|
self.progress_bar_main.show()
|
|
1617
|
-
remote_dir =
|
|
1618
|
-
self.
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
+
remote_dir = sync_call(
|
|
1623
|
+
self._api.file.upload_directory_async(
|
|
1624
|
+
team_id=self.team_id,
|
|
1625
|
+
local_dir=local_demo_dir,
|
|
1626
|
+
remote_dir=remote_demo_dir,
|
|
1627
|
+
progress_size_cb=upload_artifacts_pbar.update,
|
|
1628
|
+
)
|
|
1622
1629
|
)
|
|
1623
1630
|
self.progress_bar_main.hide()
|
|
1624
1631
|
|
|
@@ -1690,11 +1697,12 @@ class TrainApp:
|
|
|
1690
1697
|
Path: /experiments/{project_id}_{project_name}/{task_id}_{framework_name}/
|
|
1691
1698
|
Example path: /experiments/43192_Apples/68271_rt-detr/
|
|
1692
1699
|
"""
|
|
1693
|
-
logger.info(f"Uploading directory: '{self.output_dir}' to Supervisely")
|
|
1694
1700
|
task_id = self.task_id
|
|
1695
1701
|
|
|
1696
1702
|
remote_artifacts_dir = f"/{self._experiments_dir_name}/{self.project_id}_{self.project_name}/{task_id}_{self.framework_name}/"
|
|
1697
|
-
|
|
1703
|
+
logger.info(
|
|
1704
|
+
f"Uploading artifacts directory: '{self.output_dir}' to Supervisely Team Files directory '{remote_artifacts_dir}'"
|
|
1705
|
+
)
|
|
1698
1706
|
# Clean debug directory if exists
|
|
1699
1707
|
if task_id == "debug-session":
|
|
1700
1708
|
if self._api.file.dir_exists(self.team_id, f"{remote_artifacts_dir}/", True):
|
|
@@ -1725,11 +1733,13 @@ class TrainApp:
|
|
|
1725
1733
|
unit_scale=True,
|
|
1726
1734
|
) as upload_artifacts_pbar:
|
|
1727
1735
|
self.progress_bar_main.show()
|
|
1728
|
-
remote_dir =
|
|
1729
|
-
self.
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1736
|
+
remote_dir = sync_call(
|
|
1737
|
+
self._api.file.upload_directory_async(
|
|
1738
|
+
team_id=self.team_id,
|
|
1739
|
+
local_dir=self.output_dir,
|
|
1740
|
+
remote_dir=remote_artifacts_dir,
|
|
1741
|
+
progress_size_cb=upload_artifacts_pbar.update,
|
|
1742
|
+
)
|
|
1733
1743
|
)
|
|
1734
1744
|
self.progress_bar_main.hide()
|
|
1735
1745
|
|
|
@@ -2088,6 +2098,7 @@ class TrainApp:
|
|
|
2088
2098
|
self.project_info.id, version_id=project_version_id
|
|
2089
2099
|
)
|
|
2090
2100
|
|
|
2101
|
+
file_info = None
|
|
2091
2102
|
if self.model_source == ModelSource.CUSTOM:
|
|
2092
2103
|
file_info = self._api.file.get_info_by_path(
|
|
2093
2104
|
self.team_id,
|
|
@@ -2496,32 +2507,33 @@ class TrainApp:
|
|
|
2496
2507
|
def _upload_export_weights(
|
|
2497
2508
|
self, export_weights: Dict[str, str], remote_dir: str
|
|
2498
2509
|
) -> Dict[str, str]:
|
|
2510
|
+
"""Uploads export weights (any other specified formats) to Supervisely Team Files.
|
|
2511
|
+
The default export is handled by the `_upload_artifacts` method."""
|
|
2512
|
+
file_dest_paths = []
|
|
2513
|
+
size = 0
|
|
2514
|
+
for path in export_weights.values():
|
|
2515
|
+
file_name = sly_fs.get_file_name_with_ext(path)
|
|
2516
|
+
file_dest_paths.append(join(remote_dir, self._export_dir_name, file_name))
|
|
2517
|
+
size += sly_fs.get_file_size(path)
|
|
2499
2518
|
with self.progress_bar_main(
|
|
2500
|
-
message="Uploading export weights",
|
|
2501
|
-
total=
|
|
2519
|
+
message=f"Uploading {len(export_weights)} export weights",
|
|
2520
|
+
total=size,
|
|
2521
|
+
unit="B",
|
|
2522
|
+
unit_scale=True,
|
|
2502
2523
|
) as export_upload_main_pbar:
|
|
2524
|
+
logger.debug(f"Uploading {len(export_weights)} export weights of size {size} bytes")
|
|
2525
|
+
logger.debug(f"Destination paths: {file_dest_paths}")
|
|
2503
2526
|
self.progress_bar_main.show()
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
|
|
2512
|
-
) as export_upload_secondary_pbar:
|
|
2513
|
-
self.progress_bar_secondary.show()
|
|
2514
|
-
destination_path = join(remote_dir, self._export_dir_name, file_name)
|
|
2515
|
-
self._api.file.upload(
|
|
2516
|
-
self.team_id,
|
|
2517
|
-
path,
|
|
2518
|
-
destination_path,
|
|
2519
|
-
export_upload_secondary_pbar,
|
|
2520
|
-
)
|
|
2521
|
-
export_upload_main_pbar.update(1)
|
|
2527
|
+
sync_call(
|
|
2528
|
+
self._api.file.upload_bulk_async(
|
|
2529
|
+
team_id=self.team_id,
|
|
2530
|
+
src_paths=export_weights.values(),
|
|
2531
|
+
dst_paths=file_dest_paths,
|
|
2532
|
+
progress_cb=export_upload_main_pbar.update,
|
|
2533
|
+
)
|
|
2534
|
+
)
|
|
2522
2535
|
|
|
2523
2536
|
self.progress_bar_main.hide()
|
|
2524
|
-
self.progress_bar_secondary.hide()
|
|
2525
2537
|
|
|
2526
2538
|
remote_export_weights = {
|
|
2527
2539
|
runtime: join(self._export_dir_name, sly_fs.get_file_name_with_ext(path))
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
supervisely/README.md,sha256=XM-DiMC6To3I9RjQZ0c61905EFRR_jnCUx2q3uNR-X8,3331
|
|
2
2
|
supervisely/__init__.py,sha256=mtgVKiRSlnRU7yKG0Re130mBL10wCzsNfOfi-w-Kj4c,10833
|
|
3
|
-
supervisely/_utils.py,sha256=
|
|
3
|
+
supervisely/_utils.py,sha256=hYzGRVAh-cB2RmqixHbaJQZHy4byNip4KZm2Gdt8P7k,16849
|
|
4
4
|
supervisely/function_wrapper.py,sha256=R5YajTQ0GnRp2vtjwfC9hINkzQc0JiyGsu8TER373xY,1912
|
|
5
5
|
supervisely/sly_logger.py,sha256=z92Vu5hmC0GgTIJO1n6kPDayRW9__8ix8hL6poDZj-Y,6274
|
|
6
6
|
supervisely/tiny_timer.py,sha256=hkpe_7FE6bsKL79blSs7WBaktuPavEVu67IpEPrfmjE,183
|
|
@@ -22,10 +22,10 @@ supervisely/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
|
22
22
|
supervisely/api/advanced_api.py,sha256=Nd5cCnHFWc3PSUrCtENxTGtDjS37_lCHXsgXvUI3Ti8,2054
|
|
23
23
|
supervisely/api/agent_api.py,sha256=ShWAIlXcWXcyI9fqVuP5GZVCigCMJmjnvdGUfLspD6Y,8890
|
|
24
24
|
supervisely/api/annotation_api.py,sha256=kuk4qwojTJxYr2iqAKbW-QhWw_DFc4TsjA2Wc2MEaqw,68449
|
|
25
|
-
supervisely/api/api.py,sha256=
|
|
25
|
+
supervisely/api/api.py,sha256=6TczKT1t0MWlbArSW31RmeyWP04pqngfUO_NrG5FETE,66287
|
|
26
26
|
supervisely/api/app_api.py,sha256=RsbVej8WxWVn9cNo5s3Fqd1symsCdsfOaKVBKEUapRY,71927
|
|
27
27
|
supervisely/api/dataset_api.py,sha256=GH7prDRJKyJlTv_7_Y-RkTwJN7ED4EkXNqqmi3iIdI4,41352
|
|
28
|
-
supervisely/api/file_api.py,sha256=
|
|
28
|
+
supervisely/api/file_api.py,sha256=xVM4fFeIc52aKnxduCIU7L6Rgd7Rh36rzTJ8hVT8hw4,88925
|
|
29
29
|
supervisely/api/github_api.py,sha256=NIexNjEer9H5rf5sw2LEZd7C1WR-tK4t6IZzsgeAAwQ,623
|
|
30
30
|
supervisely/api/image_annotation_tool_api.py,sha256=YcUo78jRDBJYvIjrd-Y6FJAasLta54nnxhyaGyanovA,5237
|
|
31
31
|
supervisely/api/image_api.py,sha256=WIML_6N1qgOWBm3acexmGSWz4hAaSxlYmUtbytROaP8,192375
|
|
@@ -708,7 +708,7 @@ supervisely/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
|
708
708
|
supervisely/io/docker_utils.py,sha256=hb_HXGM8IYB0PF-nD7NxMwaHgzaxIFxofsUzQ_RCUZI,7935
|
|
709
709
|
supervisely/io/env.py,sha256=QQDDE79sz4inRHfcXoAKr3IzqkXQ0qleJ0YYojmHoCk,17712
|
|
710
710
|
supervisely/io/exception_handlers.py,sha256=_nAgMFeE94bCxEvWakR82hMtdOJUyn7Gc7OymMxI9WI,36484
|
|
711
|
-
supervisely/io/fs.py,sha256
|
|
711
|
+
supervisely/io/fs.py,sha256=DvLDzZUEpo7_ieSbt1gw8BYmoSsNUBcGKpXVs0Wqckk,55296
|
|
712
712
|
supervisely/io/fs_cache.py,sha256=985gvBGzveLcDudgz10E4EWVjP9jxdU1Pa0GFfCBoCA,6520
|
|
713
713
|
supervisely/io/github_utils.py,sha256=jGmvQJ5bjtACuSFABzrxL0jJdh14SezovrHp8T-9y8g,1779
|
|
714
714
|
supervisely/io/json.py,sha256=VvyqXZl22nb6_DJK3TUOPetd5xq9xwRFKumWqsGs7iI,8679
|
|
@@ -973,7 +973,7 @@ supervisely/nn/tracker/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
|
|
|
973
973
|
supervisely/nn/tracker/utils/gmc.py,sha256=3JX8979H3NA-YHNaRQyj9Z-xb9qtyMittPEjGw8y2Jo,11557
|
|
974
974
|
supervisely/nn/tracker/utils/kalman_filter.py,sha256=eSFmCjM0mikHCAFvj-KCVzw-0Jxpoc3Cfc2NWEjJC1Q,17268
|
|
975
975
|
supervisely/nn/training/__init__.py,sha256=gY4PCykJ-42MWKsqb9kl-skemKa8yB6t_fb5kzqR66U,111
|
|
976
|
-
supervisely/nn/training/train_app.py,sha256=
|
|
976
|
+
supervisely/nn/training/train_app.py,sha256=yVsMdMlV6OHCRMJ63-hPmTfdsC3Z1-0ohphsMtDMpPw,104944
|
|
977
977
|
supervisely/nn/training/gui/__init__.py,sha256=Nqnn8clbgv-5l0PgxcTOldg8mkMKrFn4TvPL-rYUUGg,1
|
|
978
978
|
supervisely/nn/training/gui/classes_selector.py,sha256=8UgzA4aogOAr1s42smwEcDbgaBj_i0JLhjwlZ9bFdIA,3772
|
|
979
979
|
supervisely/nn/training/gui/gui.py,sha256=CnT_QhihrxdSHKybpI0pXhPLwCaXEana_qdn0DhXByg,25558
|
|
@@ -1075,9 +1075,9 @@ supervisely/worker_proto/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
|
|
|
1075
1075
|
supervisely/worker_proto/worker_api_pb2.py,sha256=VQfi5JRBHs2pFCK1snec3JECgGnua3Xjqw_-b3aFxuM,59142
|
|
1076
1076
|
supervisely/worker_proto/worker_api_pb2_grpc.py,sha256=3BwQXOaP9qpdi0Dt9EKG--Lm8KGN0C5AgmUfRv77_Jk,28940
|
|
1077
1077
|
supervisely_lib/__init__.py,sha256=7-3QnN8Zf0wj8NCr2oJmqoQWMKKPKTECvjH9pd2S5vY,159
|
|
1078
|
-
supervisely-6.73.
|
|
1079
|
-
supervisely-6.73.
|
|
1080
|
-
supervisely-6.73.
|
|
1081
|
-
supervisely-6.73.
|
|
1082
|
-
supervisely-6.73.
|
|
1083
|
-
supervisely-6.73.
|
|
1078
|
+
supervisely-6.73.321.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
1079
|
+
supervisely-6.73.321.dist-info/METADATA,sha256=yVJfg3OU_JHg5N-hBOHneb0i5S2tBLYZsVQ9sdn67Co,33596
|
|
1080
|
+
supervisely-6.73.321.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
|
|
1081
|
+
supervisely-6.73.321.dist-info/entry_points.txt,sha256=U96-5Hxrp2ApRjnCoUiUhWMqijqh8zLR03sEhWtAcms,102
|
|
1082
|
+
supervisely-6.73.321.dist-info/top_level.txt,sha256=kcFVwb7SXtfqZifrZaSE3owHExX4gcNYe7Q2uoby084,28
|
|
1083
|
+
supervisely-6.73.321.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|