supervisely 6.73.319__py3-none-any.whl → 6.73.321__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- supervisely/_utils.py +33 -0
- supervisely/api/api.py +17 -13
- supervisely/api/file_api.py +158 -25
- supervisely/api/task_api.py +9 -12
- supervisely/app/fastapi/subapp.py +24 -4
- supervisely/io/env.py +16 -0
- supervisely/io/fs.py +81 -4
- supervisely/nn/training/train_app.py +62 -42
- supervisely/sly_logger.py +49 -45
- {supervisely-6.73.319.dist-info → supervisely-6.73.321.dist-info}/METADATA +1 -1
- {supervisely-6.73.319.dist-info → supervisely-6.73.321.dist-info}/RECORD +15 -15
- {supervisely-6.73.319.dist-info → supervisely-6.73.321.dist-info}/LICENSE +0 -0
- {supervisely-6.73.319.dist-info → supervisely-6.73.321.dist-info}/WHEEL +0 -0
- {supervisely-6.73.319.dist-info → supervisely-6.73.321.dist-info}/entry_points.txt +0 -0
- {supervisely-6.73.319.dist-info → supervisely-6.73.321.dist-info}/top_level.txt +0 -0
supervisely/_utils.py
CHANGED
|
@@ -471,6 +471,39 @@ def get_or_create_event_loop() -> asyncio.AbstractEventLoop:
|
|
|
471
471
|
return loop
|
|
472
472
|
|
|
473
473
|
|
|
474
|
+
def sync_call(coro):
|
|
475
|
+
"""
|
|
476
|
+
This function is used to run asynchronous functions in synchronous context.
|
|
477
|
+
|
|
478
|
+
:param coro: Asynchronous function.
|
|
479
|
+
:type coro: Coroutine
|
|
480
|
+
:return: Result of the asynchronous function.
|
|
481
|
+
:rtype: Any
|
|
482
|
+
|
|
483
|
+
:Usage example:
|
|
484
|
+
|
|
485
|
+
.. code-block:: python
|
|
486
|
+
|
|
487
|
+
from supervisely.utils import sync_call
|
|
488
|
+
|
|
489
|
+
async def async_function():
|
|
490
|
+
await asyncio.sleep(1)
|
|
491
|
+
return "Hello, World!"
|
|
492
|
+
coro = async_function()
|
|
493
|
+
result = sync_call(coro)
|
|
494
|
+
print(result)
|
|
495
|
+
# Output: Hello, World!
|
|
496
|
+
"""
|
|
497
|
+
|
|
498
|
+
loop = get_or_create_event_loop()
|
|
499
|
+
|
|
500
|
+
if loop.is_running():
|
|
501
|
+
future = asyncio.run_coroutine_threadsafe(coro, loop=loop)
|
|
502
|
+
return future.result()
|
|
503
|
+
else:
|
|
504
|
+
return loop.run_until_complete(coro)
|
|
505
|
+
|
|
506
|
+
|
|
474
507
|
def get_filename_from_headers(url):
|
|
475
508
|
try:
|
|
476
509
|
response = requests.head(url, allow_redirects=True)
|
supervisely/api/api.py
CHANGED
|
@@ -1443,6 +1443,7 @@ class Api:
|
|
|
1443
1443
|
chunk_size: int = 8192,
|
|
1444
1444
|
use_public_api: Optional[bool] = True,
|
|
1445
1445
|
timeout: httpx._types.TimeoutTypes = 60,
|
|
1446
|
+
**kwargs,
|
|
1446
1447
|
) -> AsyncGenerator:
|
|
1447
1448
|
"""
|
|
1448
1449
|
Performs asynchronous streaming GET or POST request to server with given parameters.
|
|
@@ -1486,18 +1487,19 @@ class Api:
|
|
|
1486
1487
|
else:
|
|
1487
1488
|
headers = {**self.headers, **headers}
|
|
1488
1489
|
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
elif isinstance(data, Dict):
|
|
1494
|
-
json_body = {**data, **self.additional_fields}
|
|
1495
|
-
content = None
|
|
1496
|
-
params = None
|
|
1490
|
+
params = kwargs.get("params", None)
|
|
1491
|
+
if "content" in kwargs or "json_body" in kwargs:
|
|
1492
|
+
content = kwargs.get("content", None)
|
|
1493
|
+
json_body = kwargs.get("json_body", None)
|
|
1497
1494
|
else:
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1495
|
+
if isinstance(data, (bytes, Generator)):
|
|
1496
|
+
content = data
|
|
1497
|
+
json_body = None
|
|
1498
|
+
elif isinstance(data, Dict):
|
|
1499
|
+
json_body = {**data, **self.additional_fields}
|
|
1500
|
+
content = None
|
|
1501
|
+
else:
|
|
1502
|
+
raise ValueError("Data should be either bytes or dict")
|
|
1501
1503
|
|
|
1502
1504
|
if range_start is not None or range_end is not None:
|
|
1503
1505
|
headers["Range"] = f"bytes={range_start or ''}-{range_end or ''}"
|
|
@@ -1512,17 +1514,19 @@ class Api:
|
|
|
1512
1514
|
url,
|
|
1513
1515
|
content=content,
|
|
1514
1516
|
json=json_body,
|
|
1515
|
-
params=params,
|
|
1516
1517
|
headers=headers,
|
|
1517
1518
|
timeout=timeout,
|
|
1519
|
+
params=params,
|
|
1518
1520
|
)
|
|
1519
1521
|
elif method_type == "GET":
|
|
1520
1522
|
response = self.async_httpx_client.stream(
|
|
1521
1523
|
method_type,
|
|
1522
1524
|
url,
|
|
1523
|
-
|
|
1525
|
+
content=content,
|
|
1526
|
+
json=json_body,
|
|
1524
1527
|
headers=headers,
|
|
1525
1528
|
timeout=timeout,
|
|
1529
|
+
params=params,
|
|
1526
1530
|
)
|
|
1527
1531
|
else:
|
|
1528
1532
|
raise NotImplementedError(
|
supervisely/api/file_api.py
CHANGED
|
@@ -33,7 +33,9 @@ from supervisely.io.fs import (
|
|
|
33
33
|
get_file_name,
|
|
34
34
|
get_file_name_with_ext,
|
|
35
35
|
get_file_size,
|
|
36
|
+
get_or_create_event_loop,
|
|
36
37
|
list_files_recursively,
|
|
38
|
+
list_files_recursively_async,
|
|
37
39
|
silent_remove,
|
|
38
40
|
)
|
|
39
41
|
from supervisely.io.fs_cache import FileCache
|
|
@@ -2041,7 +2043,7 @@ class FileApi(ModuleApiBase):
|
|
|
2041
2043
|
# check_hash: bool = True, #TODO add with resumaple api
|
|
2042
2044
|
progress_cb: Optional[Union[tqdm, Callable]] = None,
|
|
2043
2045
|
progress_cb_type: Literal["number", "size"] = "size",
|
|
2044
|
-
) ->
|
|
2046
|
+
) -> None:
|
|
2045
2047
|
"""
|
|
2046
2048
|
Upload file from local path to Team Files asynchronously.
|
|
2047
2049
|
|
|
@@ -2057,8 +2059,8 @@ class FileApi(ModuleApiBase):
|
|
|
2057
2059
|
:type progress_cb: tqdm or callable, optional
|
|
2058
2060
|
:param progress_cb_type: Type of progress callback. Can be "number" or "size". Default is "size".
|
|
2059
2061
|
:type progress_cb_type: Literal["number", "size"], optional
|
|
2060
|
-
:return:
|
|
2061
|
-
:rtype: :class:`
|
|
2062
|
+
:return: None
|
|
2063
|
+
:rtype: :class:`NoneType`
|
|
2062
2064
|
:Usage example:
|
|
2063
2065
|
|
|
2064
2066
|
.. code-block:: python
|
|
@@ -2087,17 +2089,30 @@ class FileApi(ModuleApiBase):
|
|
|
2087
2089
|
}
|
|
2088
2090
|
if semaphore is None:
|
|
2089
2091
|
semaphore = self._api.get_default_semaphore()
|
|
2092
|
+
logger.debug(f"Uploading with async to: {dst}. Semaphore: {semaphore}")
|
|
2090
2093
|
async with semaphore:
|
|
2091
2094
|
async with aiofiles.open(src, "rb") as fd:
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2095
|
+
|
|
2096
|
+
async def file_chunk_generator():
|
|
2097
|
+
while True:
|
|
2098
|
+
chunk = await fd.read(8 * 1024 * 1024)
|
|
2099
|
+
if not chunk:
|
|
2100
|
+
break
|
|
2101
|
+
if progress_cb is not None and progress_cb_type == "size":
|
|
2102
|
+
progress_cb(len(chunk))
|
|
2103
|
+
yield chunk
|
|
2104
|
+
|
|
2105
|
+
async for chunk, _ in self._api.stream_async(
|
|
2106
|
+
method=api_method,
|
|
2107
|
+
method_type="POST",
|
|
2108
|
+
data=file_chunk_generator(), # added as required, but not used inside
|
|
2109
|
+
headers=headers,
|
|
2110
|
+
content=file_chunk_generator(), # used instead of data inside stream_async
|
|
2111
|
+
params=json_body,
|
|
2112
|
+
):
|
|
2113
|
+
pass
|
|
2098
2114
|
if progress_cb is not None and progress_cb_type == "number":
|
|
2099
2115
|
progress_cb(1)
|
|
2100
|
-
return response
|
|
2101
2116
|
|
|
2102
2117
|
async def upload_bulk_async(
|
|
2103
2118
|
self,
|
|
@@ -2109,6 +2124,7 @@ class FileApi(ModuleApiBase):
|
|
|
2109
2124
|
# check_hash: bool = True, #TODO add with resumaple api
|
|
2110
2125
|
progress_cb: Optional[Union[tqdm, Callable]] = None,
|
|
2111
2126
|
progress_cb_type: Literal["number", "size"] = "size",
|
|
2127
|
+
enable_fallback: Optional[bool] = True,
|
|
2112
2128
|
) -> None:
|
|
2113
2129
|
"""
|
|
2114
2130
|
Upload multiple files from local paths to Team Files asynchronously.
|
|
@@ -2125,6 +2141,8 @@ class FileApi(ModuleApiBase):
|
|
|
2125
2141
|
:type progress_cb: tqdm or callable, optional
|
|
2126
2142
|
:param progress_cb_type: Type of progress callback. Can be "number" or "size". Default is "size".
|
|
2127
2143
|
:type progress_cb_type: Literal["number", "size"], optional
|
|
2144
|
+
:param enable_fallback: If True, the method will fallback to synchronous upload if an error occurs.
|
|
2145
|
+
:type enable_fallback: bool, optional
|
|
2128
2146
|
:return: None
|
|
2129
2147
|
:rtype: :class:`NoneType`
|
|
2130
2148
|
:Usage example:
|
|
@@ -2153,19 +2171,134 @@ class FileApi(ModuleApiBase):
|
|
|
2153
2171
|
api.file.upload_bulk_async(8, paths_to_files, paths_to_save)
|
|
2154
2172
|
)
|
|
2155
2173
|
"""
|
|
2156
|
-
|
|
2157
|
-
semaphore
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2174
|
+
try:
|
|
2175
|
+
if semaphore is None:
|
|
2176
|
+
semaphore = self._api.get_default_semaphore()
|
|
2177
|
+
tasks = []
|
|
2178
|
+
for src, dst in zip(src_paths, dst_paths):
|
|
2179
|
+
task = asyncio.create_task(
|
|
2180
|
+
self.upload_async(
|
|
2181
|
+
team_id=team_id,
|
|
2182
|
+
src=src,
|
|
2183
|
+
dst=dst,
|
|
2184
|
+
semaphore=semaphore,
|
|
2185
|
+
# chunk_size=chunk_size, #TODO add with resumaple api
|
|
2186
|
+
# check_hash=check_hash, #TODO add with resumaple api
|
|
2187
|
+
progress_cb=progress_cb,
|
|
2188
|
+
progress_cb_type=progress_cb_type,
|
|
2189
|
+
)
|
|
2190
|
+
)
|
|
2191
|
+
tasks.append(task)
|
|
2192
|
+
for task in tasks:
|
|
2193
|
+
await task
|
|
2194
|
+
except Exception as e:
|
|
2195
|
+
if enable_fallback:
|
|
2196
|
+
logger.warning(
|
|
2197
|
+
f"Upload files bulk asynchronously failed. Fallback to synchronous upload.",
|
|
2198
|
+
exc_info=True,
|
|
2199
|
+
)
|
|
2200
|
+
if progress_cb is not None and progress_cb_type == "number":
|
|
2201
|
+
logger.warning(
|
|
2202
|
+
"Progress callback type 'number' is not supported for synchronous upload. "
|
|
2203
|
+
"Progress callback will be disabled."
|
|
2204
|
+
)
|
|
2205
|
+
progress_cb = None
|
|
2206
|
+
self.upload_bulk(
|
|
2207
|
+
team_id=team_id,
|
|
2208
|
+
src_paths=src_paths,
|
|
2209
|
+
dst_paths=dst_paths,
|
|
2210
|
+
progress_cb=progress_cb,
|
|
2211
|
+
)
|
|
2212
|
+
else:
|
|
2213
|
+
raise e
|
|
2214
|
+
|
|
2215
|
+
async def upload_directory_async(
|
|
2216
|
+
self,
|
|
2217
|
+
team_id: int,
|
|
2218
|
+
local_dir: str,
|
|
2219
|
+
remote_dir: str,
|
|
2220
|
+
change_name_if_conflict: Optional[bool] = True,
|
|
2221
|
+
progress_size_cb: Optional[Union[tqdm, Callable]] = None,
|
|
2222
|
+
replace_if_conflict: Optional[bool] = False,
|
|
2223
|
+
enable_fallback: Optional[bool] = True,
|
|
2224
|
+
) -> str:
|
|
2225
|
+
"""
|
|
2226
|
+
Upload Directory to Team Files from local path.
|
|
2227
|
+
Files are uploaded asynchronously.
|
|
2228
|
+
|
|
2229
|
+
:param team_id: Team ID in Supervisely.
|
|
2230
|
+
:type team_id: int
|
|
2231
|
+
:param local_dir: Path to local Directory.
|
|
2232
|
+
:type local_dir: str
|
|
2233
|
+
:param remote_dir: Path to Directory in Team Files.
|
|
2234
|
+
:type remote_dir: str
|
|
2235
|
+
:param change_name_if_conflict: Checks if given name already exists and adds suffix to the end of the name.
|
|
2236
|
+
:type change_name_if_conflict: bool, optional
|
|
2237
|
+
:param progress_size_cb: Function for tracking download progress.
|
|
2238
|
+
:type progress_size_cb: Progress, optional
|
|
2239
|
+
:param replace_if_conflict: If True, replace existing dir.
|
|
2240
|
+
:type replace_if_conflict: bool, optional
|
|
2241
|
+
:param enable_fallback: If True, the method will fallback to synchronous upload if an error occurs.
|
|
2242
|
+
:type enable_fallback: bool, optional
|
|
2243
|
+
:return: Path to Directory in Team Files
|
|
2244
|
+
:rtype: :class:`str`
|
|
2245
|
+
:Usage example:
|
|
2246
|
+
|
|
2247
|
+
.. code-block:: python
|
|
2248
|
+
|
|
2249
|
+
import supervisely as sly
|
|
2250
|
+
|
|
2251
|
+
os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
|
|
2252
|
+
os.environ['API_TOKEN'] = 'Your Supervisely API Token'
|
|
2253
|
+
api = sly.Api.from_env()
|
|
2254
|
+
|
|
2255
|
+
path_to_dir = "/My_App_Test/ds1"
|
|
2256
|
+
local_path = "/home/admin/Downloads/My_local_test"
|
|
2257
|
+
|
|
2258
|
+
api.file.upload_directory(9, local_path, path_to_dir)
|
|
2259
|
+
"""
|
|
2260
|
+
try:
|
|
2261
|
+
if not remote_dir.startswith("/"):
|
|
2262
|
+
remote_dir = "/" + remote_dir
|
|
2263
|
+
|
|
2264
|
+
if self.dir_exists(team_id, remote_dir):
|
|
2265
|
+
if change_name_if_conflict is True:
|
|
2266
|
+
res_remote_dir = self.get_free_dir_name(team_id, remote_dir)
|
|
2267
|
+
elif replace_if_conflict is True:
|
|
2268
|
+
res_remote_dir = remote_dir
|
|
2269
|
+
else:
|
|
2270
|
+
raise FileExistsError(
|
|
2271
|
+
f"Directory {remote_dir} already exists in your team (id={team_id})"
|
|
2272
|
+
)
|
|
2273
|
+
else:
|
|
2274
|
+
res_remote_dir = remote_dir
|
|
2275
|
+
|
|
2276
|
+
local_files = await list_files_recursively_async(local_dir)
|
|
2277
|
+
dir_prefix = local_dir.rstrip("/") + "/"
|
|
2278
|
+
remote_files = [
|
|
2279
|
+
res_remote_dir.rstrip("/") + "/" + file[len(dir_prefix) :] for file in local_files
|
|
2280
|
+
]
|
|
2281
|
+
|
|
2282
|
+
await self.upload_bulk_async(
|
|
2283
|
+
team_id=team_id,
|
|
2284
|
+
src_paths=local_files,
|
|
2285
|
+
dst_paths=remote_files,
|
|
2286
|
+
progress_cb=progress_size_cb,
|
|
2169
2287
|
)
|
|
2170
|
-
|
|
2171
|
-
|
|
2288
|
+
except Exception as e:
|
|
2289
|
+
if enable_fallback:
|
|
2290
|
+
logger.warning(
|
|
2291
|
+
f"Upload directory asynchronously failed. Fallback to synchronous upload.",
|
|
2292
|
+
exc_info=True,
|
|
2293
|
+
)
|
|
2294
|
+
res_remote_dir = self.upload_directory(
|
|
2295
|
+
team_id=team_id,
|
|
2296
|
+
local_dir=local_dir,
|
|
2297
|
+
remote_dir=res_remote_dir,
|
|
2298
|
+
change_name_if_conflict=change_name_if_conflict,
|
|
2299
|
+
progress_size_cb=progress_size_cb,
|
|
2300
|
+
replace_if_conflict=replace_if_conflict,
|
|
2301
|
+
)
|
|
2302
|
+
else:
|
|
2303
|
+
raise e
|
|
2304
|
+
return res_remote_dir
|
supervisely/api/task_api.py
CHANGED
|
@@ -818,14 +818,21 @@ class TaskApi(ModuleApiBase, ModuleWithStatus):
|
|
|
818
818
|
pass
|
|
819
819
|
|
|
820
820
|
def set_output_project(
|
|
821
|
-
self,
|
|
821
|
+
self,
|
|
822
|
+
task_id: int,
|
|
823
|
+
project_id: int,
|
|
824
|
+
project_name: Optional[str] = None,
|
|
825
|
+
project_preview: Optional[str] = None,
|
|
822
826
|
) -> Dict:
|
|
823
827
|
"""set_output_project"""
|
|
824
828
|
if project_name is None:
|
|
825
829
|
project = self._api.project.get_info_by_id(project_id, raise_error=True)
|
|
826
830
|
project_name = project.name
|
|
831
|
+
project_preview = project.image_preview_url
|
|
827
832
|
|
|
828
833
|
output = {ApiField.PROJECT: {ApiField.ID: project_id, ApiField.TITLE: project_name}}
|
|
834
|
+
if project_preview is not None:
|
|
835
|
+
output[ApiField.PROJECT][ApiField.PREVIEW] = project_preview
|
|
829
836
|
resp = self._api.post(
|
|
830
837
|
"tasks.output.set", {ApiField.TASK_ID: task_id, ApiField.OUTPUT: output}
|
|
831
838
|
)
|
|
@@ -1159,9 +1166,7 @@ class TaskApi(ModuleApiBase, ModuleWithStatus):
|
|
|
1159
1166
|
)
|
|
1160
1167
|
self._api.post("tasks.status.update", {ApiField.ID: task_id, ApiField.STATUS: status})
|
|
1161
1168
|
|
|
1162
|
-
def set_output_experiment(
|
|
1163
|
-
self, task_id: int, experiment_info: dict, project_name: str = None
|
|
1164
|
-
) -> Dict:
|
|
1169
|
+
def set_output_experiment(self, task_id: int, experiment_info: dict) -> Dict:
|
|
1165
1170
|
"""
|
|
1166
1171
|
Sets output for the task with experiment info.
|
|
1167
1172
|
|
|
@@ -1216,15 +1221,7 @@ class TaskApi(ModuleApiBase, ModuleWithStatus):
|
|
|
1216
1221
|
},
|
|
1217
1222
|
}
|
|
1218
1223
|
"""
|
|
1219
|
-
project_id = experiment_info.get("project_id")
|
|
1220
|
-
if project_id is None:
|
|
1221
|
-
raise ValueError("Key 'project_id' is required in experiment_info")
|
|
1222
|
-
if project_name is None:
|
|
1223
|
-
project = self._api.project.get_info_by_id(project_id, raise_error=True)
|
|
1224
|
-
project_name = project.name
|
|
1225
|
-
|
|
1226
1224
|
output = {
|
|
1227
|
-
ApiField.PROJECT: {ApiField.ID: project_id, ApiField.TITLE: project_name},
|
|
1228
1225
|
ApiField.EXPERIMENT: {ApiField.DATA: {**experiment_info}},
|
|
1229
1226
|
}
|
|
1230
1227
|
resp = self._api.post(
|
|
@@ -27,6 +27,7 @@ from fastapi import (
|
|
|
27
27
|
)
|
|
28
28
|
from fastapi.exception_handlers import http_exception_handler
|
|
29
29
|
from fastapi.responses import JSONResponse
|
|
30
|
+
from fastapi.routing import APIRouter
|
|
30
31
|
from fastapi.staticfiles import StaticFiles
|
|
31
32
|
|
|
32
33
|
import supervisely.io.env as sly_env
|
|
@@ -47,7 +48,7 @@ from supervisely.app.singleton import Singleton
|
|
|
47
48
|
from supervisely.app.widgets_context import JinjaWidgets
|
|
48
49
|
from supervisely.geometry.bitmap import Bitmap
|
|
49
50
|
from supervisely.io.fs import dir_exists, mkdir
|
|
50
|
-
from supervisely.sly_logger import logger
|
|
51
|
+
from supervisely.sly_logger import create_formatter, logger
|
|
51
52
|
|
|
52
53
|
# from supervisely.app.fastapi.request import Request
|
|
53
54
|
|
|
@@ -56,7 +57,9 @@ if TYPE_CHECKING:
|
|
|
56
57
|
|
|
57
58
|
import logging
|
|
58
59
|
|
|
59
|
-
|
|
60
|
+
SUPERVISELY_SERVER_PATH_PREFIX = sly_env.supervisely_server_path_prefix()
|
|
61
|
+
if SUPERVISELY_SERVER_PATH_PREFIX and not SUPERVISELY_SERVER_PATH_PREFIX.startswith("/"):
|
|
62
|
+
SUPERVISELY_SERVER_PATH_PREFIX = f"/{SUPERVISELY_SERVER_PATH_PREFIX}"
|
|
60
63
|
|
|
61
64
|
|
|
62
65
|
class ReadyzFilter(logging.Filter):
|
|
@@ -67,8 +70,24 @@ class ReadyzFilter(logging.Filter):
|
|
|
67
70
|
return True
|
|
68
71
|
|
|
69
72
|
|
|
70
|
-
|
|
71
|
-
uvicorn_logger.
|
|
73
|
+
def _init_uvicorn_logger():
|
|
74
|
+
uvicorn_logger = logging.getLogger("uvicorn.access")
|
|
75
|
+
for handler in uvicorn_logger.handlers:
|
|
76
|
+
handler.setFormatter(create_formatter())
|
|
77
|
+
uvicorn_logger.addFilter(ReadyzFilter())
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
_init_uvicorn_logger()
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class PrefixRouter(APIRouter):
|
|
84
|
+
def add_api_route(self, path, *args, **kwargs):
|
|
85
|
+
allowed_paths = ["/livez", "/is_alive", "/is_running", "/readyz", "/is_ready"]
|
|
86
|
+
if path in allowed_paths:
|
|
87
|
+
super().add_api_route(path, *args, **kwargs)
|
|
88
|
+
if SUPERVISELY_SERVER_PATH_PREFIX:
|
|
89
|
+
path = SUPERVISELY_SERVER_PATH_PREFIX + path
|
|
90
|
+
super().add_api_route(path, *args, **kwargs)
|
|
72
91
|
|
|
73
92
|
|
|
74
93
|
class Event:
|
|
@@ -864,6 +883,7 @@ def _init(
|
|
|
864
883
|
class _MainServer(metaclass=Singleton):
|
|
865
884
|
def __init__(self):
|
|
866
885
|
self._server = FastAPI()
|
|
886
|
+
self._server.router = PrefixRouter()
|
|
867
887
|
|
|
868
888
|
def get_server(self) -> FastAPI:
|
|
869
889
|
return self._server
|
supervisely/io/env.py
CHANGED
|
@@ -554,3 +554,19 @@ def semaphore_size() -> int:
|
|
|
554
554
|
postprocess_fn=lambda x: int(x),
|
|
555
555
|
raise_not_found=False,
|
|
556
556
|
)
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
def supervisely_server_path_prefix() -> str:
|
|
560
|
+
"""Returns routes prefix from environment variable using following
|
|
561
|
+
- SUPERVISELY_SERVER_PATH_PREFIX
|
|
562
|
+
|
|
563
|
+
:return: routes prefix
|
|
564
|
+
:rtype: str
|
|
565
|
+
"""
|
|
566
|
+
return _parse_from_env(
|
|
567
|
+
name="supervisely_server_path_prefix",
|
|
568
|
+
keys=["SUPERVISELY_SERVER_PATH_PREFIX"],
|
|
569
|
+
postprocess_fn=lambda x: x,
|
|
570
|
+
default="",
|
|
571
|
+
raise_not_found=False,
|
|
572
|
+
)
|
supervisely/io/fs.py
CHANGED
|
@@ -205,15 +205,19 @@ def list_files_recursively(
|
|
|
205
205
|
for filename in file_names:
|
|
206
206
|
yield os.path.join(dir_name, filename)
|
|
207
207
|
|
|
208
|
-
valid_extensions =
|
|
208
|
+
valid_extensions = (
|
|
209
|
+
valid_extensions
|
|
210
|
+
if ignore_valid_extensions_case is False
|
|
211
|
+
else [ext.lower() for ext in valid_extensions]
|
|
212
|
+
)
|
|
209
213
|
files = []
|
|
210
214
|
for file_path in file_path_generator():
|
|
211
215
|
file_ext = get_file_ext(file_path)
|
|
212
216
|
if ignore_valid_extensions_case:
|
|
213
217
|
file_ext.lower()
|
|
214
|
-
if (
|
|
215
|
-
|
|
216
|
-
)
|
|
218
|
+
if (valid_extensions is None or file_ext in valid_extensions) and (
|
|
219
|
+
filter_fn is None or filter_fn(file_path)
|
|
220
|
+
):
|
|
217
221
|
files.append(file_path)
|
|
218
222
|
return files
|
|
219
223
|
|
|
@@ -1558,3 +1562,76 @@ async def touch_async(path: str) -> None:
|
|
|
1558
1562
|
async with aiofiles.open(path, "a"):
|
|
1559
1563
|
loop = get_or_create_event_loop()
|
|
1560
1564
|
await loop.run_in_executor(None, os.utime, path, None)
|
|
1565
|
+
|
|
1566
|
+
|
|
1567
|
+
async def list_files_recursively_async(
|
|
1568
|
+
dir_path: str,
|
|
1569
|
+
valid_extensions: Optional[List[str]] = None,
|
|
1570
|
+
filter_fn: Optional[Callable[[str], bool]] = None,
|
|
1571
|
+
ignore_valid_extensions_case: bool = False,
|
|
1572
|
+
) -> List[str]:
|
|
1573
|
+
"""
|
|
1574
|
+
Recursively list files in the directory asynchronously.
|
|
1575
|
+
Returns list with all file paths.
|
|
1576
|
+
Can be filtered by valid extensions and filter function.
|
|
1577
|
+
|
|
1578
|
+
:param dir_path: Target directory path.
|
|
1579
|
+
:type dir_path: str
|
|
1580
|
+
:param valid_extensions: List of valid extensions. Default is None.
|
|
1581
|
+
:type valid_extensions: Optional[List[str]]
|
|
1582
|
+
:param filter_fn: Filter function. Default is None.
|
|
1583
|
+
:type filter_fn: Optional[Callable[[str], bool]]
|
|
1584
|
+
:param ignore_valid_extensions_case: Ignore case when checking valid extensions. Default is False.
|
|
1585
|
+
:type ignore_valid_extensions_case: bool
|
|
1586
|
+
:returns: List of file paths
|
|
1587
|
+
:rtype: List[str]
|
|
1588
|
+
|
|
1589
|
+
:Usage example:
|
|
1590
|
+
|
|
1591
|
+
.. code-block:: python
|
|
1592
|
+
|
|
1593
|
+
import supervisely as sly
|
|
1594
|
+
|
|
1595
|
+
dir_path = '/home/admin/work/projects/examples'
|
|
1596
|
+
loop = sly.utils.get_or_create_event_loop()
|
|
1597
|
+
coro = sly.fs.list_files_recursively_async(dir_path)
|
|
1598
|
+
if loop.is_running():
|
|
1599
|
+
future = asyncio.run_coroutine_threadsafe(coro, loop)
|
|
1600
|
+
files = future.result()
|
|
1601
|
+
else:
|
|
1602
|
+
files = loop.run_until_complete(coro)
|
|
1603
|
+
"""
|
|
1604
|
+
|
|
1605
|
+
def sync_file_list():
|
|
1606
|
+
if valid_extensions and ignore_valid_extensions_case:
|
|
1607
|
+
valid_extensions_set = set(map(str.lower, valid_extensions))
|
|
1608
|
+
else:
|
|
1609
|
+
valid_extensions_set = set(valid_extensions) if valid_extensions else None
|
|
1610
|
+
|
|
1611
|
+
files = []
|
|
1612
|
+
for dir_name, _, file_names in os.walk(dir_path):
|
|
1613
|
+
full_paths = [os.path.join(dir_name, filename) for filename in file_names]
|
|
1614
|
+
|
|
1615
|
+
if valid_extensions_set:
|
|
1616
|
+
full_paths = [
|
|
1617
|
+
fp
|
|
1618
|
+
for fp in full_paths
|
|
1619
|
+
if (
|
|
1620
|
+
ext := (
|
|
1621
|
+
os.path.splitext(fp)[1].lower()
|
|
1622
|
+
if ignore_valid_extensions_case
|
|
1623
|
+
else os.path.splitext(fp)[1]
|
|
1624
|
+
)
|
|
1625
|
+
)
|
|
1626
|
+
in valid_extensions_set
|
|
1627
|
+
]
|
|
1628
|
+
|
|
1629
|
+
if filter_fn:
|
|
1630
|
+
full_paths = [fp for fp in full_paths if filter_fn(fp)]
|
|
1631
|
+
|
|
1632
|
+
files.extend(full_paths)
|
|
1633
|
+
|
|
1634
|
+
return files
|
|
1635
|
+
|
|
1636
|
+
loop = get_or_create_event_loop()
|
|
1637
|
+
return await loop.run_in_executor(None, sync_file_list)
|
|
@@ -39,7 +39,7 @@ from supervisely import (
|
|
|
39
39
|
is_production,
|
|
40
40
|
logger,
|
|
41
41
|
)
|
|
42
|
-
from supervisely._utils import abs_url, get_filename_from_headers
|
|
42
|
+
from supervisely._utils import abs_url, get_filename_from_headers, sync_call
|
|
43
43
|
from supervisely.api.file_api import FileInfo
|
|
44
44
|
from supervisely.app import get_synced_data_dir
|
|
45
45
|
from supervisely.app.widgets import Progress
|
|
@@ -60,6 +60,7 @@ from supervisely.nn.utils import ModelSource
|
|
|
60
60
|
from supervisely.output import set_directory
|
|
61
61
|
from supervisely.project.download import (
|
|
62
62
|
copy_from_cache,
|
|
63
|
+
download_async_or_sync,
|
|
63
64
|
download_to_cache,
|
|
64
65
|
get_cache_size,
|
|
65
66
|
is_cached,
|
|
@@ -803,8 +804,9 @@ class TrainApp:
|
|
|
803
804
|
:type total_images: int
|
|
804
805
|
"""
|
|
805
806
|
with self.progress_bar_main(message="Downloading input data", total=total_images) as pbar:
|
|
807
|
+
logger.debug("Downloading project data without cache")
|
|
806
808
|
self.progress_bar_main.show()
|
|
807
|
-
|
|
809
|
+
download_async_or_sync(
|
|
808
810
|
api=self._api,
|
|
809
811
|
project_id=self.project_id,
|
|
810
812
|
dest_dir=self.project_dir,
|
|
@@ -834,6 +836,7 @@ class TrainApp:
|
|
|
834
836
|
|
|
835
837
|
logger.info(self._get_cache_log_message(cached, to_download))
|
|
836
838
|
with self.progress_bar_main(message="Downloading input data", total=total_images) as pbar:
|
|
839
|
+
logger.debug("Downloading project data with cache")
|
|
837
840
|
self.progress_bar_main.show()
|
|
838
841
|
download_to_cache(
|
|
839
842
|
api=self._api,
|
|
@@ -1546,6 +1549,7 @@ class TrainApp:
|
|
|
1546
1549
|
|
|
1547
1550
|
# Do not include this fields to uploaded file:
|
|
1548
1551
|
experiment_info["primary_metric"] = primary_metric_name
|
|
1552
|
+
experiment_info["project_preview"] = self.project_info.image_preview_url
|
|
1549
1553
|
return experiment_info
|
|
1550
1554
|
|
|
1551
1555
|
def _generate_hyperparameters(self, remote_dir: str, experiment_info: Dict) -> None:
|
|
@@ -1602,10 +1606,12 @@ class TrainApp:
|
|
|
1602
1606
|
logger.info(f"Demo directory '{local_demo_dir}' does not exist")
|
|
1603
1607
|
return
|
|
1604
1608
|
|
|
1605
|
-
logger.debug(f"Uploading demo files to Supervisely")
|
|
1606
1609
|
remote_demo_dir = join(remote_dir, "demo")
|
|
1607
1610
|
local_files = sly_fs.list_files_recursively(local_demo_dir)
|
|
1608
1611
|
total_size = sum([sly_fs.get_file_size(file_path) for file_path in local_files])
|
|
1612
|
+
logger.debug(
|
|
1613
|
+
f"Uploading demo files of total size {total_size} bytes to Supervisely Team Files directory '{remote_demo_dir}'"
|
|
1614
|
+
)
|
|
1609
1615
|
with self.progress_bar_main(
|
|
1610
1616
|
message="Uploading demo files to Team Files",
|
|
1611
1617
|
total=total_size,
|
|
@@ -1613,11 +1619,13 @@ class TrainApp:
|
|
|
1613
1619
|
unit_scale=True,
|
|
1614
1620
|
) as upload_artifacts_pbar:
|
|
1615
1621
|
self.progress_bar_main.show()
|
|
1616
|
-
remote_dir =
|
|
1617
|
-
self.
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1622
|
+
remote_dir = sync_call(
|
|
1623
|
+
self._api.file.upload_directory_async(
|
|
1624
|
+
team_id=self.team_id,
|
|
1625
|
+
local_dir=local_demo_dir,
|
|
1626
|
+
remote_dir=remote_demo_dir,
|
|
1627
|
+
progress_size_cb=upload_artifacts_pbar.update,
|
|
1628
|
+
)
|
|
1621
1629
|
)
|
|
1622
1630
|
self.progress_bar_main.hide()
|
|
1623
1631
|
|
|
@@ -1689,11 +1697,12 @@ class TrainApp:
|
|
|
1689
1697
|
Path: /experiments/{project_id}_{project_name}/{task_id}_{framework_name}/
|
|
1690
1698
|
Example path: /experiments/43192_Apples/68271_rt-detr/
|
|
1691
1699
|
"""
|
|
1692
|
-
logger.info(f"Uploading directory: '{self.output_dir}' to Supervisely")
|
|
1693
1700
|
task_id = self.task_id
|
|
1694
1701
|
|
|
1695
1702
|
remote_artifacts_dir = f"/{self._experiments_dir_name}/{self.project_id}_{self.project_name}/{task_id}_{self.framework_name}/"
|
|
1696
|
-
|
|
1703
|
+
logger.info(
|
|
1704
|
+
f"Uploading artifacts directory: '{self.output_dir}' to Supervisely Team Files directory '{remote_artifacts_dir}'"
|
|
1705
|
+
)
|
|
1697
1706
|
# Clean debug directory if exists
|
|
1698
1707
|
if task_id == "debug-session":
|
|
1699
1708
|
if self._api.file.dir_exists(self.team_id, f"{remote_artifacts_dir}/", True):
|
|
@@ -1724,11 +1733,13 @@ class TrainApp:
|
|
|
1724
1733
|
unit_scale=True,
|
|
1725
1734
|
) as upload_artifacts_pbar:
|
|
1726
1735
|
self.progress_bar_main.show()
|
|
1727
|
-
remote_dir =
|
|
1728
|
-
self.
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1736
|
+
remote_dir = sync_call(
|
|
1737
|
+
self._api.file.upload_directory_async(
|
|
1738
|
+
team_id=self.team_id,
|
|
1739
|
+
local_dir=self.output_dir,
|
|
1740
|
+
remote_dir=remote_artifacts_dir,
|
|
1741
|
+
progress_size_cb=upload_artifacts_pbar.update,
|
|
1742
|
+
)
|
|
1732
1743
|
)
|
|
1733
1744
|
self.progress_bar_main.hide()
|
|
1734
1745
|
|
|
@@ -1759,7 +1770,7 @@ class TrainApp:
|
|
|
1759
1770
|
# self.gui.training_logs.tensorboard_button.disable()
|
|
1760
1771
|
|
|
1761
1772
|
# Set artifacts to GUI
|
|
1762
|
-
self._api.task.set_output_experiment(self.task_id, experiment_info
|
|
1773
|
+
self._api.task.set_output_experiment(self.task_id, experiment_info)
|
|
1763
1774
|
set_directory(remote_dir)
|
|
1764
1775
|
self.gui.training_artifacts.artifacts_thumbnail.set(file_info)
|
|
1765
1776
|
self.gui.training_artifacts.artifacts_thumbnail.show()
|
|
@@ -1806,8 +1817,9 @@ class TrainApp:
|
|
|
1806
1817
|
self.gui.training_artifacts.trt_instruction.show()
|
|
1807
1818
|
|
|
1808
1819
|
# Show the inference demo widget if overview or any demo is available
|
|
1809
|
-
if self.gui.training_artifacts
|
|
1820
|
+
if hasattr(self.gui.training_artifacts, "inference_demo_field") and any(
|
|
1810
1821
|
[
|
|
1822
|
+
self.gui.training_artifacts.overview_demo_exists(demo_path),
|
|
1811
1823
|
self.gui.training_artifacts.pytorch_demo_exists(demo_path),
|
|
1812
1824
|
self.gui.training_artifacts.onnx_demo_exists(demo_path),
|
|
1813
1825
|
self.gui.training_artifacts.trt_demo_exists(demo_path),
|
|
@@ -1863,13 +1875,19 @@ class TrainApp:
|
|
|
1863
1875
|
:return: Evaluation report, report ID and evaluation metrics.
|
|
1864
1876
|
:rtype: tuple
|
|
1865
1877
|
"""
|
|
1866
|
-
lnk_file_info, report, report_id, eval_metrics
|
|
1878
|
+
lnk_file_info, report, report_id, eval_metrics, primary_metric_name = (
|
|
1879
|
+
None,
|
|
1880
|
+
None,
|
|
1881
|
+
None,
|
|
1882
|
+
{},
|
|
1883
|
+
None,
|
|
1884
|
+
)
|
|
1867
1885
|
if self._inference_class is None:
|
|
1868
1886
|
logger.warning(
|
|
1869
1887
|
"Inference class is not registered, model benchmark disabled. "
|
|
1870
1888
|
"Use 'register_inference_class' method to register inference class."
|
|
1871
1889
|
)
|
|
1872
|
-
return lnk_file_info, report, report_id, eval_metrics
|
|
1890
|
+
return lnk_file_info, report, report_id, eval_metrics, primary_metric_name
|
|
1873
1891
|
|
|
1874
1892
|
# Can't get task type from session. requires before session init
|
|
1875
1893
|
supported_task_types = [
|
|
@@ -1883,7 +1901,7 @@ class TrainApp:
|
|
|
1883
1901
|
f"Task type: '{task_type}' is not supported for Model Benchmark. "
|
|
1884
1902
|
f"Supported tasks: {', '.join(task_type)}"
|
|
1885
1903
|
)
|
|
1886
|
-
return lnk_file_info, report, report_id, eval_metrics
|
|
1904
|
+
return lnk_file_info, report, report_id, eval_metrics, primary_metric_name
|
|
1887
1905
|
|
|
1888
1906
|
logger.info("Running Model Benchmark evaluation")
|
|
1889
1907
|
try:
|
|
@@ -2051,7 +2069,7 @@ class TrainApp:
|
|
|
2051
2069
|
if diff_project_info:
|
|
2052
2070
|
self._api.project.remove(diff_project_info.id)
|
|
2053
2071
|
except Exception as e2:
|
|
2054
|
-
return lnk_file_info, report, report_id, eval_metrics
|
|
2072
|
+
return lnk_file_info, report, report_id, eval_metrics, primary_metric_name
|
|
2055
2073
|
return lnk_file_info, report, report_id, eval_metrics, primary_metric_name
|
|
2056
2074
|
|
|
2057
2075
|
# ----------------------------------------- #
|
|
@@ -2080,6 +2098,7 @@ class TrainApp:
|
|
|
2080
2098
|
self.project_info.id, version_id=project_version_id
|
|
2081
2099
|
)
|
|
2082
2100
|
|
|
2101
|
+
file_info = None
|
|
2083
2102
|
if self.model_source == ModelSource.CUSTOM:
|
|
2084
2103
|
file_info = self._api.file.get_info_by_path(
|
|
2085
2104
|
self.team_id,
|
|
@@ -2488,32 +2507,33 @@ class TrainApp:
|
|
|
2488
2507
|
def _upload_export_weights(
|
|
2489
2508
|
self, export_weights: Dict[str, str], remote_dir: str
|
|
2490
2509
|
) -> Dict[str, str]:
|
|
2510
|
+
"""Uploads export weights (any other specified formats) to Supervisely Team Files.
|
|
2511
|
+
The default export is handled by the `_upload_artifacts` method."""
|
|
2512
|
+
file_dest_paths = []
|
|
2513
|
+
size = 0
|
|
2514
|
+
for path in export_weights.values():
|
|
2515
|
+
file_name = sly_fs.get_file_name_with_ext(path)
|
|
2516
|
+
file_dest_paths.append(join(remote_dir, self._export_dir_name, file_name))
|
|
2517
|
+
size += sly_fs.get_file_size(path)
|
|
2491
2518
|
with self.progress_bar_main(
|
|
2492
|
-
message="Uploading export weights",
|
|
2493
|
-
total=
|
|
2519
|
+
message=f"Uploading {len(export_weights)} export weights",
|
|
2520
|
+
total=size,
|
|
2521
|
+
unit="B",
|
|
2522
|
+
unit_scale=True,
|
|
2494
2523
|
) as export_upload_main_pbar:
|
|
2524
|
+
logger.debug(f"Uploading {len(export_weights)} export weights of size {size} bytes")
|
|
2525
|
+
logger.debug(f"Destination paths: {file_dest_paths}")
|
|
2495
2526
|
self.progress_bar_main.show()
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
) as export_upload_secondary_pbar:
|
|
2505
|
-
self.progress_bar_secondary.show()
|
|
2506
|
-
destination_path = join(remote_dir, self._export_dir_name, file_name)
|
|
2507
|
-
self._api.file.upload(
|
|
2508
|
-
self.team_id,
|
|
2509
|
-
path,
|
|
2510
|
-
destination_path,
|
|
2511
|
-
export_upload_secondary_pbar,
|
|
2512
|
-
)
|
|
2513
|
-
export_upload_main_pbar.update(1)
|
|
2527
|
+
sync_call(
|
|
2528
|
+
self._api.file.upload_bulk_async(
|
|
2529
|
+
team_id=self.team_id,
|
|
2530
|
+
src_paths=export_weights.values(),
|
|
2531
|
+
dst_paths=file_dest_paths,
|
|
2532
|
+
progress_cb=export_upload_main_pbar.update,
|
|
2533
|
+
)
|
|
2534
|
+
)
|
|
2514
2535
|
|
|
2515
2536
|
self.progress_bar_main.hide()
|
|
2516
|
-
self.progress_bar_secondary.hide()
|
|
2517
2537
|
|
|
2518
2538
|
remote_export_weights = {
|
|
2519
2539
|
runtime: join(self._export_dir_name, sly_fs.get_file_name_with_ext(path))
|
supervisely/sly_logger.py
CHANGED
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
# coding: utf-8
|
|
2
2
|
|
|
3
|
-
import logging
|
|
4
|
-
import types
|
|
5
3
|
import datetime
|
|
4
|
+
import logging
|
|
6
5
|
import os
|
|
6
|
+
import types
|
|
7
7
|
from collections import namedtuple
|
|
8
8
|
from enum import Enum
|
|
9
|
-
#import simplejson
|
|
10
|
-
from pythonjsonlogger import jsonlogger
|
|
11
9
|
|
|
10
|
+
# import simplejson
|
|
11
|
+
from pythonjsonlogger import jsonlogger
|
|
12
12
|
|
|
13
13
|
###############################################################################
|
|
14
14
|
|
|
15
|
+
|
|
15
16
|
class ServiceType(Enum):
|
|
16
17
|
AGENT = 1
|
|
17
18
|
TASK = 2
|
|
@@ -47,20 +48,23 @@ class EventType(Enum):
|
|
|
47
48
|
|
|
48
49
|
|
|
49
50
|
# level name: level, default exc_info, description
|
|
50
|
-
LogLevelSpec = namedtuple(
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
51
|
+
LogLevelSpec = namedtuple(
|
|
52
|
+
"LogLevelSpec",
|
|
53
|
+
[
|
|
54
|
+
"int",
|
|
55
|
+
"add_exc_info",
|
|
56
|
+
"descr",
|
|
57
|
+
],
|
|
58
|
+
)
|
|
55
59
|
|
|
56
60
|
|
|
57
61
|
LOGGING_LEVELS = {
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
62
|
+
"FATAL": LogLevelSpec(50, True, "Critical error"),
|
|
63
|
+
"ERROR": LogLevelSpec(40, True, "Error"), # may be shown to end user
|
|
64
|
+
"WARN": LogLevelSpec(30, False, "Warning"), # may be shown to end user
|
|
65
|
+
"INFO": LogLevelSpec(20, False, "Info"), # may be shown to end user
|
|
66
|
+
"DEBUG": LogLevelSpec(10, False, "Debug"),
|
|
67
|
+
"TRACE": LogLevelSpec(5, False, "Trace"),
|
|
64
68
|
}
|
|
65
69
|
|
|
66
70
|
|
|
@@ -69,12 +73,9 @@ def _set_logging_levels(levels, the_logger):
|
|
|
69
73
|
logging.addLevelName(lvl, lvl_name.upper()) # two mappings
|
|
70
74
|
|
|
71
75
|
def construct_logger_member(lvl_val, default_exc_info):
|
|
72
|
-
return lambda self, msg, *args, exc_info=default_exc_info, **kwargs:
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
*args,
|
|
76
|
-
exc_info=exc_info,
|
|
77
|
-
**kwargs)
|
|
76
|
+
return lambda self, msg, *args, exc_info=default_exc_info, **kwargs: self.log(
|
|
77
|
+
lvl_val, str(msg), *args, exc_info=exc_info, **kwargs
|
|
78
|
+
)
|
|
78
79
|
|
|
79
80
|
func = construct_logger_member(lvl, def_exc_info)
|
|
80
81
|
bound_method = types.MethodType(func, the_logger)
|
|
@@ -86,16 +87,16 @@ def _set_logging_levels(levels, the_logger):
|
|
|
86
87
|
|
|
87
88
|
def _get_default_logging_fields():
|
|
88
89
|
supported_keys = [
|
|
89
|
-
|
|
90
|
+
"asctime",
|
|
90
91
|
# 'created',
|
|
91
92
|
# 'filename',
|
|
92
93
|
# 'funcName',
|
|
93
|
-
|
|
94
|
+
"levelname",
|
|
94
95
|
# 'levelno',
|
|
95
96
|
# 'lineno',
|
|
96
97
|
# 'module',
|
|
97
98
|
# 'msecs',
|
|
98
|
-
|
|
99
|
+
"message",
|
|
99
100
|
# 'name',
|
|
100
101
|
# 'pathname',
|
|
101
102
|
# 'process',
|
|
@@ -104,10 +105,10 @@ def _get_default_logging_fields():
|
|
|
104
105
|
# 'thread',
|
|
105
106
|
# 'threadName'
|
|
106
107
|
]
|
|
107
|
-
return
|
|
108
|
+
return " ".join(["%({0:s})".format(k) for k in supported_keys])
|
|
108
109
|
|
|
109
110
|
|
|
110
|
-
#def dumps_ignore_nan(obj, *args, **kwargs):
|
|
111
|
+
# def dumps_ignore_nan(obj, *args, **kwargs):
|
|
111
112
|
# return simplejson.dumps(obj, ignore_nan=True, *args, **kwargs)
|
|
112
113
|
|
|
113
114
|
|
|
@@ -115,21 +116,21 @@ class CustomJsonFormatter(jsonlogger.JsonFormatter):
|
|
|
115
116
|
additional_fields = {}
|
|
116
117
|
|
|
117
118
|
def __init__(self, format_string):
|
|
118
|
-
super().__init__(format_string)
|
|
119
|
+
super().__init__(format_string) # , json_serializer=dumps_ignore_nan)
|
|
119
120
|
|
|
120
121
|
def process_log_record(self, log_record):
|
|
121
|
-
log_record[
|
|
122
|
+
log_record["timestamp"] = log_record.pop("asctime", None)
|
|
122
123
|
|
|
123
|
-
levelname = log_record.pop(
|
|
124
|
+
levelname = log_record.pop("levelname", None)
|
|
124
125
|
if levelname is not None:
|
|
125
|
-
log_record[
|
|
126
|
+
log_record["level"] = levelname.lower()
|
|
126
127
|
|
|
127
|
-
e_info = log_record.pop(
|
|
128
|
+
e_info = log_record.pop("exc_info", None)
|
|
128
129
|
if e_info is not None:
|
|
129
|
-
if e_info ==
|
|
130
|
+
if e_info == "NoneType: None": # python logger is not ok here
|
|
130
131
|
pass
|
|
131
132
|
else:
|
|
132
|
-
log_record[
|
|
133
|
+
log_record["stack"] = e_info.split("\n")
|
|
133
134
|
|
|
134
135
|
return jsonlogger.JsonFormatter.process_log_record(self, log_record)
|
|
135
136
|
|
|
@@ -142,8 +143,8 @@ class CustomJsonFormatter(jsonlogger.JsonFormatter):
|
|
|
142
143
|
|
|
143
144
|
def formatTime(self, record, datefmt=None):
|
|
144
145
|
ct = datetime.datetime.fromtimestamp(record.created)
|
|
145
|
-
t = ct.strftime(
|
|
146
|
-
s =
|
|
146
|
+
t = ct.strftime("%Y-%m-%dT%H:%M:%S")
|
|
147
|
+
s = "%s.%03dZ" % (t, record.msecs)
|
|
147
148
|
return s
|
|
148
149
|
|
|
149
150
|
|
|
@@ -164,16 +165,20 @@ def _construct_logger(the_logger, loglevel_text):
|
|
|
164
165
|
###############################################################################
|
|
165
166
|
|
|
166
167
|
|
|
168
|
+
def create_formatter(logger_fmt_string=None):
|
|
169
|
+
if logger_fmt_string is None:
|
|
170
|
+
logger_fmt_string = _get_default_logging_fields()
|
|
171
|
+
return CustomJsonFormatter(logger_fmt_string)
|
|
172
|
+
|
|
173
|
+
|
|
167
174
|
def add_logger_handler(the_logger, log_handler): # default format
|
|
168
|
-
|
|
169
|
-
formatter = CustomJsonFormatter(logger_fmt_string)
|
|
175
|
+
formatter = create_formatter()
|
|
170
176
|
log_handler.setFormatter(formatter)
|
|
171
177
|
the_logger.addHandler(log_handler)
|
|
172
178
|
|
|
173
179
|
|
|
174
180
|
def add_default_logging_into_file(the_logger, log_dir):
|
|
175
|
-
fname =
|
|
176
|
-
datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S"))
|
|
181
|
+
fname = "log_{}.txt".format(datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S"))
|
|
177
182
|
ofpath = os.path.join(log_dir, fname)
|
|
178
183
|
|
|
179
184
|
log_handler_file = logging.FileHandler(filename=ofpath)
|
|
@@ -191,27 +196,26 @@ def change_formatters_default_values(the_logger, field_name, value):
|
|
|
191
196
|
|
|
192
197
|
|
|
193
198
|
def _get_loglevel_env():
|
|
194
|
-
loglevel = os.getenv(
|
|
199
|
+
loglevel = os.getenv("LOG_LEVEL", None)
|
|
195
200
|
if loglevel is None:
|
|
196
|
-
loglevel = os.getenv(
|
|
201
|
+
loglevel = os.getenv("LOGLEVEL", "INFO")
|
|
197
202
|
return loglevel.upper()
|
|
198
203
|
|
|
199
204
|
|
|
200
205
|
def set_global_logger():
|
|
201
206
|
loglevel = _get_loglevel_env() # use the env to set loglevel
|
|
202
|
-
the_logger = logging.getLogger(
|
|
207
|
+
the_logger = logging.getLogger("logger") # optional logger name
|
|
203
208
|
_construct_logger(the_logger, loglevel)
|
|
204
209
|
return the_logger
|
|
205
210
|
|
|
206
211
|
|
|
207
212
|
def get_task_logger(task_id, loglevel=None):
|
|
208
213
|
if loglevel is None:
|
|
209
|
-
loglevel = _get_loglevel_env()
|
|
210
|
-
logger_name =
|
|
214
|
+
loglevel = _get_loglevel_env() # use the env to set loglevel
|
|
215
|
+
logger_name = "task_{}".format(task_id)
|
|
211
216
|
the_logger = logging.getLogger(logger_name) # optional logger name
|
|
212
217
|
_construct_logger(the_logger, loglevel)
|
|
213
218
|
return the_logger
|
|
214
219
|
|
|
215
220
|
|
|
216
221
|
logger = set_global_logger()
|
|
217
|
-
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
supervisely/README.md,sha256=XM-DiMC6To3I9RjQZ0c61905EFRR_jnCUx2q3uNR-X8,3331
|
|
2
2
|
supervisely/__init__.py,sha256=mtgVKiRSlnRU7yKG0Re130mBL10wCzsNfOfi-w-Kj4c,10833
|
|
3
|
-
supervisely/_utils.py,sha256=
|
|
3
|
+
supervisely/_utils.py,sha256=hYzGRVAh-cB2RmqixHbaJQZHy4byNip4KZm2Gdt8P7k,16849
|
|
4
4
|
supervisely/function_wrapper.py,sha256=R5YajTQ0GnRp2vtjwfC9hINkzQc0JiyGsu8TER373xY,1912
|
|
5
|
-
supervisely/sly_logger.py,sha256=
|
|
5
|
+
supervisely/sly_logger.py,sha256=z92Vu5hmC0GgTIJO1n6kPDayRW9__8ix8hL6poDZj-Y,6274
|
|
6
6
|
supervisely/tiny_timer.py,sha256=hkpe_7FE6bsKL79blSs7WBaktuPavEVu67IpEPrfmjE,183
|
|
7
7
|
supervisely/annotation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
8
|
supervisely/annotation/annotation.py,sha256=5AG1AhebkmiYy2r7nKbz6TjdmCF4tuf9FtqUjLLs7aU,114659
|
|
@@ -22,10 +22,10 @@ supervisely/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
|
22
22
|
supervisely/api/advanced_api.py,sha256=Nd5cCnHFWc3PSUrCtENxTGtDjS37_lCHXsgXvUI3Ti8,2054
|
|
23
23
|
supervisely/api/agent_api.py,sha256=ShWAIlXcWXcyI9fqVuP5GZVCigCMJmjnvdGUfLspD6Y,8890
|
|
24
24
|
supervisely/api/annotation_api.py,sha256=kuk4qwojTJxYr2iqAKbW-QhWw_DFc4TsjA2Wc2MEaqw,68449
|
|
25
|
-
supervisely/api/api.py,sha256=
|
|
25
|
+
supervisely/api/api.py,sha256=6TczKT1t0MWlbArSW31RmeyWP04pqngfUO_NrG5FETE,66287
|
|
26
26
|
supervisely/api/app_api.py,sha256=RsbVej8WxWVn9cNo5s3Fqd1symsCdsfOaKVBKEUapRY,71927
|
|
27
27
|
supervisely/api/dataset_api.py,sha256=GH7prDRJKyJlTv_7_Y-RkTwJN7ED4EkXNqqmi3iIdI4,41352
|
|
28
|
-
supervisely/api/file_api.py,sha256=
|
|
28
|
+
supervisely/api/file_api.py,sha256=xVM4fFeIc52aKnxduCIU7L6Rgd7Rh36rzTJ8hVT8hw4,88925
|
|
29
29
|
supervisely/api/github_api.py,sha256=NIexNjEer9H5rf5sw2LEZd7C1WR-tK4t6IZzsgeAAwQ,623
|
|
30
30
|
supervisely/api/image_annotation_tool_api.py,sha256=YcUo78jRDBJYvIjrd-Y6FJAasLta54nnxhyaGyanovA,5237
|
|
31
31
|
supervisely/api/image_api.py,sha256=WIML_6N1qgOWBm3acexmGSWz4hAaSxlYmUtbytROaP8,192375
|
|
@@ -42,7 +42,7 @@ supervisely/api/remote_storage_api.py,sha256=qTuPhPsstgEjRm1g-ZInddik8BNC_38YvBB
|
|
|
42
42
|
supervisely/api/report_api.py,sha256=Om7CGulUbQ4BuJ16eDtz7luLe0JQNqab-LoLpUXu7YE,7123
|
|
43
43
|
supervisely/api/role_api.py,sha256=aBL4mxtn08LDPXQuS153-lQFN6N2kcwiz8MbescZ8Gk,3044
|
|
44
44
|
supervisely/api/storage_api.py,sha256=FPGYf3Rn3LBoe38RBNdoiURs306oshzvKOEOQ56XAbs,13030
|
|
45
|
-
supervisely/api/task_api.py,sha256=
|
|
45
|
+
supervisely/api/task_api.py,sha256=WUyovA322egcfjG6Iv_vX8RdfU_3Bw7qxcniYeLpqwA,53874
|
|
46
46
|
supervisely/api/team_api.py,sha256=bEoz3mrykvliLhKnzEy52vzdd_H8VBJCpxF-Bnek9Q8,19467
|
|
47
47
|
supervisely/api/user_api.py,sha256=4S97yIc6AMTZCa0N57lzETnpIE8CeqClvCb6kjUkgfc,24940
|
|
48
48
|
supervisely/api/video_annotation_tool_api.py,sha256=3A9-U8WJzrTShP_n9T8U01M9FzGYdeS51CCBTzUnooo,6686
|
|
@@ -93,7 +93,7 @@ supervisely/app/fastapi/index.html,sha256=6K8akK7_k9Au-BpZ7cM2qocuiegLdXz8UFPnWg
|
|
|
93
93
|
supervisely/app/fastapi/no_html_main.html,sha256=NhQP7noyORBx72lFh1CQKgBRupkWjiq6Gaw-9Hkvg7c,37
|
|
94
94
|
supervisely/app/fastapi/offline.py,sha256=CwMMkJ1frD6wiZS-SEoNDtQ1UJcJe1Ob6ohE3r4CQL8,7414
|
|
95
95
|
supervisely/app/fastapi/request.py,sha256=NU7rKmxJ1pfkDZ7_yHckRcRAueJRQIqCor11UO2OHr8,766
|
|
96
|
-
supervisely/app/fastapi/subapp.py,sha256=
|
|
96
|
+
supervisely/app/fastapi/subapp.py,sha256=5lMfFLYBfHzE1OmITHsogB9hScyTJFjGV45AKY67Hkg,45647
|
|
97
97
|
supervisely/app/fastapi/templating.py,sha256=JOAW8U-14GD47E286mzFi3mZSPbm_csJGqtXWLRM4rc,2929
|
|
98
98
|
supervisely/app/fastapi/utils.py,sha256=GZuTWLcVRGVx8TL3jVEYUOZIT2FawbwIe2kAOBLw9ho,398
|
|
99
99
|
supervisely/app/fastapi/websocket.py,sha256=TlRSPOAhRItTv1HGvdukK1ZvhRjMUxRa-lJlsRR9rJw,1308
|
|
@@ -706,9 +706,9 @@ supervisely/imaging/font.py,sha256=0XcmWhlw7y2PAhrWgcsfInyRWj0WnlFpMSEXXilR8UA,2
|
|
|
706
706
|
supervisely/imaging/image.py,sha256=1KNc4qRbP9OlI4Yta07Kc2ohAgSBJ_9alF9Jag74w30,41873
|
|
707
707
|
supervisely/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
708
708
|
supervisely/io/docker_utils.py,sha256=hb_HXGM8IYB0PF-nD7NxMwaHgzaxIFxofsUzQ_RCUZI,7935
|
|
709
|
-
supervisely/io/env.py,sha256=
|
|
709
|
+
supervisely/io/env.py,sha256=QQDDE79sz4inRHfcXoAKr3IzqkXQ0qleJ0YYojmHoCk,17712
|
|
710
710
|
supervisely/io/exception_handlers.py,sha256=_nAgMFeE94bCxEvWakR82hMtdOJUyn7Gc7OymMxI9WI,36484
|
|
711
|
-
supervisely/io/fs.py,sha256
|
|
711
|
+
supervisely/io/fs.py,sha256=DvLDzZUEpo7_ieSbt1gw8BYmoSsNUBcGKpXVs0Wqckk,55296
|
|
712
712
|
supervisely/io/fs_cache.py,sha256=985gvBGzveLcDudgz10E4EWVjP9jxdU1Pa0GFfCBoCA,6520
|
|
713
713
|
supervisely/io/github_utils.py,sha256=jGmvQJ5bjtACuSFABzrxL0jJdh14SezovrHp8T-9y8g,1779
|
|
714
714
|
supervisely/io/json.py,sha256=VvyqXZl22nb6_DJK3TUOPetd5xq9xwRFKumWqsGs7iI,8679
|
|
@@ -973,7 +973,7 @@ supervisely/nn/tracker/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
|
|
|
973
973
|
supervisely/nn/tracker/utils/gmc.py,sha256=3JX8979H3NA-YHNaRQyj9Z-xb9qtyMittPEjGw8y2Jo,11557
|
|
974
974
|
supervisely/nn/tracker/utils/kalman_filter.py,sha256=eSFmCjM0mikHCAFvj-KCVzw-0Jxpoc3Cfc2NWEjJC1Q,17268
|
|
975
975
|
supervisely/nn/training/__init__.py,sha256=gY4PCykJ-42MWKsqb9kl-skemKa8yB6t_fb5kzqR66U,111
|
|
976
|
-
supervisely/nn/training/train_app.py,sha256=
|
|
976
|
+
supervisely/nn/training/train_app.py,sha256=yVsMdMlV6OHCRMJ63-hPmTfdsC3Z1-0ohphsMtDMpPw,104944
|
|
977
977
|
supervisely/nn/training/gui/__init__.py,sha256=Nqnn8clbgv-5l0PgxcTOldg8mkMKrFn4TvPL-rYUUGg,1
|
|
978
978
|
supervisely/nn/training/gui/classes_selector.py,sha256=8UgzA4aogOAr1s42smwEcDbgaBj_i0JLhjwlZ9bFdIA,3772
|
|
979
979
|
supervisely/nn/training/gui/gui.py,sha256=CnT_QhihrxdSHKybpI0pXhPLwCaXEana_qdn0DhXByg,25558
|
|
@@ -1075,9 +1075,9 @@ supervisely/worker_proto/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
|
|
|
1075
1075
|
supervisely/worker_proto/worker_api_pb2.py,sha256=VQfi5JRBHs2pFCK1snec3JECgGnua3Xjqw_-b3aFxuM,59142
|
|
1076
1076
|
supervisely/worker_proto/worker_api_pb2_grpc.py,sha256=3BwQXOaP9qpdi0Dt9EKG--Lm8KGN0C5AgmUfRv77_Jk,28940
|
|
1077
1077
|
supervisely_lib/__init__.py,sha256=7-3QnN8Zf0wj8NCr2oJmqoQWMKKPKTECvjH9pd2S5vY,159
|
|
1078
|
-
supervisely-6.73.
|
|
1079
|
-
supervisely-6.73.
|
|
1080
|
-
supervisely-6.73.
|
|
1081
|
-
supervisely-6.73.
|
|
1082
|
-
supervisely-6.73.
|
|
1083
|
-
supervisely-6.73.
|
|
1078
|
+
supervisely-6.73.321.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
1079
|
+
supervisely-6.73.321.dist-info/METADATA,sha256=yVJfg3OU_JHg5N-hBOHneb0i5S2tBLYZsVQ9sdn67Co,33596
|
|
1080
|
+
supervisely-6.73.321.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
|
|
1081
|
+
supervisely-6.73.321.dist-info/entry_points.txt,sha256=U96-5Hxrp2ApRjnCoUiUhWMqijqh8zLR03sEhWtAcms,102
|
|
1082
|
+
supervisely-6.73.321.dist-info/top_level.txt,sha256=kcFVwb7SXtfqZifrZaSE3owHExX4gcNYe7Q2uoby084,28
|
|
1083
|
+
supervisely-6.73.321.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|