supervisely 6.73.298__py3-none-any.whl → 6.73.300__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of supervisely might be problematic. Click here for more details.
- supervisely/api/module_api.py +2 -0
- supervisely/api/task_api.py +61 -0
- supervisely/app/fastapi/index.html +1 -1
- supervisely/app/fastapi/templating.py +1 -1
- supervisely/app/widgets/__init__.py +2 -1
- supervisely/app/widgets/button/button.py +3 -1
- supervisely/app/widgets/button/template.html +4 -0
- supervisely/app/widgets/custom_models_selector/custom_models_selector.py +2 -2
- supervisely/app/widgets/experiment_selector/experiment_selector.py +2 -3
- supervisely/app/widgets/run_app_button/__init__.py +0 -0
- supervisely/app/widgets/run_app_button/run_app_button.py +300 -0
- supervisely/app/widgets/run_app_button/script.js +46 -0
- supervisely/app/widgets/run_app_button/template.html +10 -0
- supervisely/nn/artifacts/artifacts.py +3 -3
- supervisely/nn/benchmark/base_benchmark.py +14 -10
- supervisely/nn/experiments.py +42 -26
- supervisely/nn/inference/inference.py +91 -0
- supervisely/nn/training/gui/training_artifacts.py +13 -11
- supervisely/nn/training/gui/training_logs.py +39 -5
- supervisely/nn/training/gui/utils.py +11 -1
- supervisely/nn/training/train_app.py +70 -18
- {supervisely-6.73.298.dist-info → supervisely-6.73.300.dist-info}/METADATA +1 -1
- {supervisely-6.73.298.dist-info → supervisely-6.73.300.dist-info}/RECORD +27 -23
- {supervisely-6.73.298.dist-info → supervisely-6.73.300.dist-info}/LICENSE +0 -0
- {supervisely-6.73.298.dist-info → supervisely-6.73.300.dist-info}/WHEEL +0 -0
- {supervisely-6.73.298.dist-info → supervisely-6.73.300.dist-info}/entry_points.txt +0 -0
- {supervisely-6.73.298.dist-info → supervisely-6.73.300.dist-info}/top_level.txt +0 -0
supervisely/nn/experiments.py
CHANGED
|
@@ -2,7 +2,7 @@ from concurrent.futures import ThreadPoolExecutor
|
|
|
2
2
|
from dataclasses import dataclass, fields
|
|
3
3
|
from json import JSONDecodeError
|
|
4
4
|
from os.path import dirname, join
|
|
5
|
-
from typing import List
|
|
5
|
+
from typing import List, Optional
|
|
6
6
|
|
|
7
7
|
import requests
|
|
8
8
|
|
|
@@ -26,33 +26,39 @@ class ExperimentInfo:
|
|
|
26
26
|
"""Task ID in Supervisely"""
|
|
27
27
|
model_files: dict
|
|
28
28
|
"""Dictionary with paths to model files that needs to be downloaded for training"""
|
|
29
|
+
model_meta: str
|
|
30
|
+
"""Path to file with model metadata such as model name, project id, project name and classes used for training"""
|
|
29
31
|
checkpoints: List[str]
|
|
30
32
|
"""List of relative paths to checkpoints"""
|
|
31
33
|
best_checkpoint: str
|
|
32
34
|
"""Name of the best checkpoint. Defined by the user in the training app"""
|
|
33
|
-
export: dict
|
|
34
|
-
"""Dictionary with exported weights in different formats"""
|
|
35
|
-
app_state: str
|
|
36
|
-
"""Path to file with settings that were used in the app"""
|
|
37
|
-
model_meta: str
|
|
38
|
-
"""Path to file with model metadata such as model name, project id, project name and classes used for training"""
|
|
39
|
-
train_val_split: str
|
|
40
|
-
"""Path to train and validation splits, which contains IDs of the images used in each split"""
|
|
41
35
|
hyperparameters: str
|
|
42
36
|
"""Path to .yaml file with hyperparameters used in the experiment"""
|
|
43
37
|
artifacts_dir: str
|
|
44
38
|
"""Path to the directory with artifacts"""
|
|
45
|
-
|
|
39
|
+
export: Optional[dict] = None
|
|
40
|
+
"""Dictionary with exported weights in different formats"""
|
|
41
|
+
app_state: Optional[str] = None
|
|
42
|
+
"""Path to file with settings that were used in the app"""
|
|
43
|
+
train_val_split: Optional[str] = None
|
|
44
|
+
"""Path to train and validation splits, which contains IDs of the images used in each split"""
|
|
45
|
+
train_size: Optional[int] = None
|
|
46
|
+
"""Number of images in the training set"""
|
|
47
|
+
val_size: Optional[int] = None
|
|
48
|
+
"""Number of images in the validation set"""
|
|
49
|
+
datetime: Optional[str] = None
|
|
46
50
|
"""Date and time when the experiment was started"""
|
|
47
|
-
evaluation_report_id: int
|
|
48
|
-
"""ID of the evaluation report"""
|
|
49
|
-
|
|
51
|
+
evaluation_report_id: Optional[int] = None
|
|
52
|
+
"""ID of the model benchmark evaluation report"""
|
|
53
|
+
evaluation_report_link: Optional[str] = None
|
|
54
|
+
"""Link to the model benchmark evaluation report"""
|
|
55
|
+
evaluation_metrics: Optional[dict] = None
|
|
50
56
|
"""Evaluation metrics"""
|
|
57
|
+
logs: Optional[dict] = None
|
|
58
|
+
"""Dictionary with link and type of logger"""
|
|
51
59
|
|
|
52
60
|
|
|
53
|
-
def get_experiment_infos(
|
|
54
|
-
api: Api, team_id: int, framework_name: str
|
|
55
|
-
) -> List[ExperimentInfo]:
|
|
61
|
+
def get_experiment_infos(api: Api, team_id: int, framework_name: str) -> List[ExperimentInfo]:
|
|
56
62
|
"""
|
|
57
63
|
Get experiments from the specified framework folder for Train v2
|
|
58
64
|
|
|
@@ -79,9 +85,7 @@ def get_experiment_infos(
|
|
|
79
85
|
experiments_folder = "/experiments"
|
|
80
86
|
experiment_infos = []
|
|
81
87
|
|
|
82
|
-
file_infos = api.file.list(
|
|
83
|
-
team_id, experiments_folder, recursive=True, return_type="fileinfo"
|
|
84
|
-
)
|
|
88
|
+
file_infos = api.file.list(team_id, experiments_folder, recursive=True, return_type="fileinfo")
|
|
85
89
|
sorted_experiment_paths = []
|
|
86
90
|
for file_info in file_infos:
|
|
87
91
|
if not file_info.path.endswith(metadata_name):
|
|
@@ -101,11 +105,25 @@ def get_experiment_infos(
|
|
|
101
105
|
)
|
|
102
106
|
response.raise_for_status()
|
|
103
107
|
response_json = response.json()
|
|
104
|
-
required_fields = {
|
|
105
|
-
|
|
106
|
-
|
|
108
|
+
required_fields = {
|
|
109
|
+
field.name for field in fields(ExperimentInfo) if field.default is not None
|
|
110
|
+
}
|
|
111
|
+
optional_fields = {
|
|
112
|
+
field.name for field in fields(ExperimentInfo) if field.default is None
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
missing_optional_fields = optional_fields - response_json.keys()
|
|
116
|
+
if missing_optional_fields:
|
|
117
|
+
logger.debug(
|
|
118
|
+
f"Missing optional fields: {missing_optional_fields} for '{experiment_path}'"
|
|
119
|
+
)
|
|
120
|
+
for field in missing_optional_fields:
|
|
121
|
+
response_json[field] = None
|
|
122
|
+
|
|
123
|
+
missing_required_fields = required_fields - response_json.keys()
|
|
124
|
+
if missing_required_fields:
|
|
107
125
|
logger.debug(
|
|
108
|
-
f"Missing fields: {
|
|
126
|
+
f"Missing required fields: {missing_required_fields} for '{experiment_path}'. Skipping."
|
|
109
127
|
)
|
|
110
128
|
return None
|
|
111
129
|
return ExperimentInfo(**response_json)
|
|
@@ -118,9 +136,7 @@ def get_experiment_infos(
|
|
|
118
136
|
return None
|
|
119
137
|
|
|
120
138
|
with ThreadPoolExecutor() as executor:
|
|
121
|
-
experiment_infos = list(
|
|
122
|
-
executor.map(fetch_experiment_data, sorted_experiment_paths)
|
|
123
|
-
)
|
|
139
|
+
experiment_infos = list(executor.map(fetch_experiment_data, sorted_experiment_paths))
|
|
124
140
|
|
|
125
141
|
experiment_infos = [info for info in experiment_infos if info is not None]
|
|
126
142
|
return experiment_infos
|
|
@@ -670,6 +670,56 @@ class Inference:
|
|
|
670
670
|
self.update_gui(self._model_served)
|
|
671
671
|
self.gui.show_deployed_model_info(self)
|
|
672
672
|
|
|
673
|
+
def load_custom_checkpoint(self, model_files: dict, model_meta: dict, device: str = "cuda"):
|
|
674
|
+
"""
|
|
675
|
+
Loads local custom model checkpoint.
|
|
676
|
+
|
|
677
|
+
:param: model_files: dict with paths to model files
|
|
678
|
+
:type: model_files: dict
|
|
679
|
+
:param: model_meta: dict with model meta
|
|
680
|
+
:type: model_meta: dict
|
|
681
|
+
:param: device: device to load model on
|
|
682
|
+
:type: device: str
|
|
683
|
+
:return: None
|
|
684
|
+
:rtype: None
|
|
685
|
+
|
|
686
|
+
:Usage Example:
|
|
687
|
+
|
|
688
|
+
.. code-block:: python
|
|
689
|
+
|
|
690
|
+
model_files = {
|
|
691
|
+
"checkpoint": "supervisely_integration/serve/best.pth",
|
|
692
|
+
"config": "supervisely_integration/serve/model_config.yml",
|
|
693
|
+
}
|
|
694
|
+
model_meta = sly.json.load_json_file("model_meta.json")
|
|
695
|
+
|
|
696
|
+
model.load_custom_checkpoint(model_files, model_meta)
|
|
697
|
+
"""
|
|
698
|
+
|
|
699
|
+
checkpoint = model_files.get("checkpoint")
|
|
700
|
+
if checkpoint is None:
|
|
701
|
+
raise ValueError("Model checkpoint is not provided")
|
|
702
|
+
checkpoint_name = sly_fs.get_file_name_with_ext(model_files["checkpoint"])
|
|
703
|
+
|
|
704
|
+
self.checkpoint_info = CheckpointInfo(
|
|
705
|
+
checkpoint_name=checkpoint_name,
|
|
706
|
+
model_name=None,
|
|
707
|
+
architecture=None,
|
|
708
|
+
checkpoint_url=None,
|
|
709
|
+
custom_checkpoint_path=None,
|
|
710
|
+
model_source=ModelSource.CUSTOM,
|
|
711
|
+
)
|
|
712
|
+
|
|
713
|
+
deploy_params = {
|
|
714
|
+
"model_source": ModelSource.CUSTOM,
|
|
715
|
+
"model_files": model_files,
|
|
716
|
+
"model_info": {},
|
|
717
|
+
"device": device,
|
|
718
|
+
"runtime": RuntimeType.PYTORCH,
|
|
719
|
+
}
|
|
720
|
+
self._set_model_meta_custom_model({"model_meta": model_meta})
|
|
721
|
+
self._load_model(deploy_params)
|
|
722
|
+
|
|
673
723
|
def _load_model_headless(
|
|
674
724
|
self,
|
|
675
725
|
model_files: dict,
|
|
@@ -959,6 +1009,47 @@ class Inference:
|
|
|
959
1009
|
else:
|
|
960
1010
|
return self._inference_one_by_one_wrapper(source, settings)
|
|
961
1011
|
|
|
1012
|
+
def inference(
|
|
1013
|
+
self,
|
|
1014
|
+
source: Union[str, int, np.ndarray, List[str], List[int], List[np.ndarray]],
|
|
1015
|
+
settings: dict = None,
|
|
1016
|
+
) -> Union[Annotation, List[Annotation], dict, List[dict]]:
|
|
1017
|
+
"""
|
|
1018
|
+
Inference method for images. Provide image path or numpy array of image.
|
|
1019
|
+
|
|
1020
|
+
:param: source: image path,image id, numpy array of image or list of image paths, image ids or numpy arrays
|
|
1021
|
+
:type: source: Union[str, int, np.ndarray, List[str], List[int], List[np.ndarray]]
|
|
1022
|
+
:param: settings: inference settings
|
|
1023
|
+
:type: settings: dict
|
|
1024
|
+
:return: annotation or list of annotations
|
|
1025
|
+
:rtype: Union[Annotation, List[Annotation], dict, List[dict]]
|
|
1026
|
+
|
|
1027
|
+
:Usage Example:
|
|
1028
|
+
|
|
1029
|
+
.. code-block:: python
|
|
1030
|
+
|
|
1031
|
+
image_path = "/root/projects/demo/img/sample.jpg"
|
|
1032
|
+
ann = model.inference(image_path)
|
|
1033
|
+
"""
|
|
1034
|
+
input_is_list = True
|
|
1035
|
+
if not isinstance(source, list):
|
|
1036
|
+
input_is_list = False
|
|
1037
|
+
source = [source]
|
|
1038
|
+
|
|
1039
|
+
if settings is None:
|
|
1040
|
+
settings = self._get_inference_settings({})
|
|
1041
|
+
|
|
1042
|
+
if isinstance(source[0], int):
|
|
1043
|
+
ann_jsons = self._inference_batch_ids(
|
|
1044
|
+
self.api, {"batch_ids": source, "settings": settings}
|
|
1045
|
+
)
|
|
1046
|
+
anns = [Annotation.from_json(ann_json, self.model_meta) for ann_json in ann_jsons]
|
|
1047
|
+
else:
|
|
1048
|
+
anns, _ = self._inference_auto(source, settings)
|
|
1049
|
+
if not input_is_list:
|
|
1050
|
+
return anns[0]
|
|
1051
|
+
return anns
|
|
1052
|
+
|
|
962
1053
|
def _inference_batched_wrapper(
|
|
963
1054
|
self,
|
|
964
1055
|
source: List[Union[str, np.ndarray]],
|
|
@@ -2,7 +2,8 @@ import os
|
|
|
2
2
|
from typing import Any, Dict
|
|
3
3
|
|
|
4
4
|
import supervisely.io.env as sly_env
|
|
5
|
-
|
|
5
|
+
import supervisely.nn.training.gui.utils as gui_utils
|
|
6
|
+
from supervisely import Api, logger
|
|
6
7
|
from supervisely._utils import is_production
|
|
7
8
|
from supervisely.api.api import ApiField
|
|
8
9
|
from supervisely.app.widgets import (
|
|
@@ -37,7 +38,7 @@ class TrainingArtifacts:
|
|
|
37
38
|
self.success_message_text = (
|
|
38
39
|
"Training completed. Training artifacts were uploaded to Team Files. "
|
|
39
40
|
"You can find and open tensorboard logs in the artifacts folder via the "
|
|
40
|
-
"<a href='https://ecosystem.supervisely.com/apps/tensorboard-
|
|
41
|
+
"<a href='https://ecosystem.supervisely.com/apps/tensorboard-experiments-viewer' target='_blank'>Tensorboard Experiment Viewer</a> app."
|
|
41
42
|
)
|
|
42
43
|
self.app_options = app_options
|
|
43
44
|
|
|
@@ -90,6 +91,9 @@ class TrainingArtifacts:
|
|
|
90
91
|
model_demo_path = model_demo.get("path", None)
|
|
91
92
|
if model_demo_path is not None:
|
|
92
93
|
model_demo_gh_link = None
|
|
94
|
+
self.pytorch_instruction = None
|
|
95
|
+
self.onnx_instruction = None
|
|
96
|
+
self.trt_instruction = None
|
|
93
97
|
if is_production():
|
|
94
98
|
task_id = sly_env.task_id()
|
|
95
99
|
task_info = api.task.get_info_by_id(task_id)
|
|
@@ -98,15 +102,13 @@ class TrainingArtifacts:
|
|
|
98
102
|
model_demo_gh_link = app_info.repo
|
|
99
103
|
else:
|
|
100
104
|
app_name = sly_env.app_name()
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
app_info = apps[0]
|
|
109
|
-
model_demo_gh_link = app_info.repo
|
|
105
|
+
module_info = gui_utils.get_module_info_by_name(api, app_name)
|
|
106
|
+
if module_info is not None:
|
|
107
|
+
model_demo_gh_link = module_info["repo"]
|
|
108
|
+
else:
|
|
109
|
+
logger.warning(
|
|
110
|
+
f"App '{app_name}' not found in Supervisely Ecosystem. Demo artifacts will not be displayed."
|
|
111
|
+
)
|
|
110
112
|
|
|
111
113
|
if model_demo_gh_link is not None:
|
|
112
114
|
gh_branch = "blob/main"
|
|
@@ -1,9 +1,18 @@
|
|
|
1
1
|
from typing import Any, Dict
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
import supervisely.io.env as sly_env
|
|
4
|
+
import supervisely.nn.training.gui.utils as gui_utils
|
|
5
|
+
from supervisely import Api, logger
|
|
4
6
|
from supervisely._utils import is_production
|
|
5
|
-
from supervisely.app.widgets import
|
|
6
|
-
|
|
7
|
+
from supervisely.app.widgets import (
|
|
8
|
+
Button,
|
|
9
|
+
Card,
|
|
10
|
+
Container,
|
|
11
|
+
Progress,
|
|
12
|
+
RunAppButton,
|
|
13
|
+
TaskLogs,
|
|
14
|
+
Text,
|
|
15
|
+
)
|
|
7
16
|
|
|
8
17
|
|
|
9
18
|
class TrainingLogs:
|
|
@@ -21,7 +30,7 @@ class TrainingLogs:
|
|
|
21
30
|
self.validator_text.hide()
|
|
22
31
|
|
|
23
32
|
if is_production():
|
|
24
|
-
task_id =
|
|
33
|
+
task_id = sly_env.task_id(raise_not_found=False)
|
|
25
34
|
else:
|
|
26
35
|
task_id = None
|
|
27
36
|
|
|
@@ -39,11 +48,36 @@ class TrainingLogs:
|
|
|
39
48
|
plain=True,
|
|
40
49
|
icon="zmdi zmdi-chart",
|
|
41
50
|
link=self.tensorboard_link,
|
|
51
|
+
visible_by_vue_field="!isStaticVersion",
|
|
42
52
|
)
|
|
43
53
|
self.tensorboard_button.disable()
|
|
44
|
-
|
|
45
54
|
self.display_widgets.extend([self.validator_text, self.tensorboard_button])
|
|
46
55
|
|
|
56
|
+
# Offline session Tensorboard button
|
|
57
|
+
self.tensorboard_offline_button = None
|
|
58
|
+
if is_production():
|
|
59
|
+
workspace_id = sly_env.workspace_id()
|
|
60
|
+
app_name = "Tensorboard Experiments Viewer"
|
|
61
|
+
module_info = gui_utils.get_module_info_by_name(api, app_name)
|
|
62
|
+
if module_info is not None:
|
|
63
|
+
self.tensorboard_offline_button = RunAppButton(
|
|
64
|
+
workspace_id=workspace_id,
|
|
65
|
+
module_id=module_info["id"],
|
|
66
|
+
payload={},
|
|
67
|
+
text="Open Tensorboard",
|
|
68
|
+
button_type="info",
|
|
69
|
+
plain=True,
|
|
70
|
+
icon="zmdi zmdi-chart",
|
|
71
|
+
available_in_offline=True,
|
|
72
|
+
visible_by_vue_field="isStaticVersion",
|
|
73
|
+
)
|
|
74
|
+
self.tensorboard_offline_button.disable()
|
|
75
|
+
self.display_widgets.extend([self.tensorboard_offline_button])
|
|
76
|
+
else:
|
|
77
|
+
logger.warning(
|
|
78
|
+
f"App '{app_name}' not found. Tensorboard button will not be displayed in offline mode."
|
|
79
|
+
)
|
|
80
|
+
|
|
47
81
|
# Optional Show logs button
|
|
48
82
|
if app_options.get("show_logs_in_gui", False):
|
|
49
83
|
self.logs_button = Button(
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
from typing import Any, Callable, Dict, List, Optional, Tuple
|
|
1
|
+
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
|
2
2
|
|
|
3
|
+
from supervisely import Api
|
|
4
|
+
from supervisely.api.app_api import AppInfo
|
|
3
5
|
from supervisely.app import DataJson
|
|
4
6
|
from supervisely.app.widgets import Button, Card, Stepper, Text, Widget
|
|
5
7
|
|
|
@@ -126,3 +128,11 @@ def set_stepper_step(stepper: Stepper, button: Button, next_pos: int):
|
|
|
126
128
|
stepper.set_active_step(next_pos)
|
|
127
129
|
else:
|
|
128
130
|
stepper.set_active_step(next_pos - 1)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def get_module_info_by_name(api: Api, app_name: str) -> Union[Dict, None]:
|
|
134
|
+
all_modules = api.app.get_list_ecosystem_modules()
|
|
135
|
+
for module in all_modules:
|
|
136
|
+
if module["name"] == app_name:
|
|
137
|
+
app_info = api.app.get_info(module["id"])
|
|
138
|
+
return app_info
|
|
@@ -39,7 +39,7 @@ from supervisely import (
|
|
|
39
39
|
is_production,
|
|
40
40
|
logger,
|
|
41
41
|
)
|
|
42
|
-
from supervisely._utils import get_filename_from_headers
|
|
42
|
+
from supervisely._utils import abs_url, get_filename_from_headers
|
|
43
43
|
from supervisely.api.file_api import FileInfo
|
|
44
44
|
from supervisely.app import get_synced_data_dir
|
|
45
45
|
from supervisely.app.widgets import Progress
|
|
@@ -76,11 +76,11 @@ class TrainApp:
|
|
|
76
76
|
:param framework_name: Name of the ML framework used.
|
|
77
77
|
:type framework_name: str
|
|
78
78
|
:param models: List of model configurations.
|
|
79
|
-
:type models: List[Dict[str, Any]]
|
|
79
|
+
:type models: Union[str, List[Dict[str, Any]]]
|
|
80
80
|
:param hyperparameters: Path or string content of hyperparameters in YAML format.
|
|
81
81
|
:type hyperparameters: str
|
|
82
82
|
:param app_options: Options for the application layout and behavior.
|
|
83
|
-
:type app_options: Optional[Dict[str, Any]]
|
|
83
|
+
:type app_options: Optional[Union[str, Dict[str, Any]]]
|
|
84
84
|
:param work_dir: Path to the working directory for storing intermediate files.
|
|
85
85
|
:type work_dir: Optional[str]
|
|
86
86
|
"""
|
|
@@ -90,8 +90,8 @@ class TrainApp:
|
|
|
90
90
|
framework_name: str,
|
|
91
91
|
models: Union[str, List[Dict[str, Any]]],
|
|
92
92
|
hyperparameters: str,
|
|
93
|
-
app_options: Union[str, Dict[str, Any]] = None,
|
|
94
|
-
work_dir: str = None,
|
|
93
|
+
app_options: Optional[Union[str, Dict[str, Any]]] = None,
|
|
94
|
+
work_dir: Optional[str] = None,
|
|
95
95
|
):
|
|
96
96
|
|
|
97
97
|
# Init
|
|
@@ -120,6 +120,8 @@ class TrainApp:
|
|
|
120
120
|
self.task_id = sly_env.task_id()
|
|
121
121
|
else:
|
|
122
122
|
self._app_name = sly_env.app_name(raise_not_found=False)
|
|
123
|
+
if self._app_name is None:
|
|
124
|
+
self._app_name = "custom-app"
|
|
123
125
|
self.task_id = sly_env.task_id(raise_not_found=False)
|
|
124
126
|
if self.task_id is None:
|
|
125
127
|
self.task_id = "debug-session"
|
|
@@ -526,6 +528,7 @@ class TrainApp:
|
|
|
526
528
|
# Step 7. [Optional] Run Model Benchmark
|
|
527
529
|
mb_eval_lnk_file_info, mb_eval_report = None, None
|
|
528
530
|
mb_eval_report_id, eval_metrics = None, {}
|
|
531
|
+
evaluation_report_link, primary_metric_name = None, None
|
|
529
532
|
if self.is_model_benchmark_enabled:
|
|
530
533
|
try:
|
|
531
534
|
# Convert GT project
|
|
@@ -543,6 +546,7 @@ class TrainApp:
|
|
|
543
546
|
mb_eval_report,
|
|
544
547
|
mb_eval_report_id,
|
|
545
548
|
eval_metrics,
|
|
549
|
+
primary_metric_name,
|
|
546
550
|
) = self._run_model_benchmark(
|
|
547
551
|
self.output_dir,
|
|
548
552
|
remote_dir,
|
|
@@ -551,6 +555,7 @@ class TrainApp:
|
|
|
551
555
|
model_meta,
|
|
552
556
|
gt_project_id,
|
|
553
557
|
)
|
|
558
|
+
evaluation_report_link = abs_url(f"/model-benchmark?id={str(mb_eval_report_id)}")
|
|
554
559
|
except Exception as e:
|
|
555
560
|
logger.error(f"Model benchmark failed: {e}")
|
|
556
561
|
|
|
@@ -565,18 +570,24 @@ class TrainApp:
|
|
|
565
570
|
|
|
566
571
|
# Step 9. Generate and upload additional files
|
|
567
572
|
self._set_text_status("metadata")
|
|
568
|
-
self._generate_experiment_info(
|
|
569
|
-
remote_dir,
|
|
573
|
+
experiment_info = self._generate_experiment_info(
|
|
574
|
+
remote_dir,
|
|
575
|
+
experiment_info,
|
|
576
|
+
eval_metrics,
|
|
577
|
+
mb_eval_report_id,
|
|
578
|
+
evaluation_report_link,
|
|
579
|
+
primary_metric_name,
|
|
580
|
+
export_weights,
|
|
570
581
|
)
|
|
571
582
|
self._generate_app_state(remote_dir, experiment_info)
|
|
572
|
-
self._generate_hyperparameters(remote_dir, experiment_info)
|
|
583
|
+
experiment_info = self._generate_hyperparameters(remote_dir, experiment_info)
|
|
573
584
|
self._generate_train_val_splits(remote_dir, train_splits_data)
|
|
574
585
|
self._generate_model_meta(remote_dir, model_meta)
|
|
575
586
|
self._upload_demo_files(remote_dir)
|
|
576
587
|
|
|
577
588
|
# Step 10. Set output widgets
|
|
578
589
|
self._set_text_status("reset")
|
|
579
|
-
self._set_training_output(remote_dir, file_info, mb_eval_report)
|
|
590
|
+
self._set_training_output(experiment_info, remote_dir, file_info, mb_eval_report)
|
|
580
591
|
self._set_ws_progress_status("completed")
|
|
581
592
|
|
|
582
593
|
# Step 11. Workflow output
|
|
@@ -1378,7 +1389,9 @@ class TrainApp:
|
|
|
1378
1389
|
return experiment_info
|
|
1379
1390
|
|
|
1380
1391
|
# Generate experiment_info.json and app_state.json
|
|
1381
|
-
def _upload_file_to_team_files(
|
|
1392
|
+
def _upload_file_to_team_files(
|
|
1393
|
+
self, local_path: str, remote_path: str, message: str
|
|
1394
|
+
) -> Union[FileInfo, None]:
|
|
1382
1395
|
"""Helper function to upload a file with progress."""
|
|
1383
1396
|
logger.debug(f"Uploading '{local_path}' to Supervisely")
|
|
1384
1397
|
total_size = sly_fs.get_file_size(local_path)
|
|
@@ -1386,13 +1399,14 @@ class TrainApp:
|
|
|
1386
1399
|
message=message, total=total_size, unit="bytes", unit_scale=True
|
|
1387
1400
|
) as upload_artifacts_pbar:
|
|
1388
1401
|
self.progress_bar_main.show()
|
|
1389
|
-
self._api.file.upload(
|
|
1402
|
+
file_info = self._api.file.upload(
|
|
1390
1403
|
self.team_id,
|
|
1391
1404
|
local_path,
|
|
1392
1405
|
remote_path,
|
|
1393
1406
|
progress_cb=upload_artifacts_pbar,
|
|
1394
1407
|
)
|
|
1395
1408
|
self.progress_bar_main.hide()
|
|
1409
|
+
return file_info
|
|
1396
1410
|
|
|
1397
1411
|
def _generate_train_val_splits(self, remote_dir: str, splits_data: dict) -> None:
|
|
1398
1412
|
"""
|
|
@@ -1446,7 +1460,10 @@ class TrainApp:
|
|
|
1446
1460
|
|
|
1447
1461
|
if task_type == TaskType.OBJECT_DETECTION:
|
|
1448
1462
|
model_meta, _ = model_meta.to_detection_task(True)
|
|
1449
|
-
elif task_type in [
|
|
1463
|
+
elif task_type in [
|
|
1464
|
+
TaskType.INSTANCE_SEGMENTATION,
|
|
1465
|
+
TaskType.SEMANTIC_SEGMENTATION,
|
|
1466
|
+
]:
|
|
1450
1467
|
model_meta, _ = model_meta.to_segmentation_task() # @TODO: check background class
|
|
1451
1468
|
return model_meta
|
|
1452
1469
|
|
|
@@ -1456,8 +1473,10 @@ class TrainApp:
|
|
|
1456
1473
|
experiment_info: Dict,
|
|
1457
1474
|
eval_metrics: Dict = {},
|
|
1458
1475
|
evaluation_report_id: Optional[int] = None,
|
|
1476
|
+
evaluation_report_link: Optional[str] = None,
|
|
1477
|
+
primary_metric_name: str = None,
|
|
1459
1478
|
export_weights: Dict = {},
|
|
1460
|
-
) ->
|
|
1479
|
+
) -> dict:
|
|
1461
1480
|
"""
|
|
1462
1481
|
Generates and uploads the experiment_info.json file to the output directory.
|
|
1463
1482
|
|
|
@@ -1469,6 +1488,8 @@ class TrainApp:
|
|
|
1469
1488
|
:type eval_metrics: dict
|
|
1470
1489
|
:param evaluation_report_id: Evaluation report file ID.
|
|
1471
1490
|
:type evaluation_report_id: int
|
|
1491
|
+
:param evaluation_report_link: Evaluation report file link.
|
|
1492
|
+
:type evaluation_report_link: str
|
|
1472
1493
|
:param export_weights: Export data.
|
|
1473
1494
|
:type export_weights: dict
|
|
1474
1495
|
"""
|
|
@@ -1488,11 +1509,15 @@ class TrainApp:
|
|
|
1488
1509
|
"app_state": self._app_state_file,
|
|
1489
1510
|
"model_meta": self._model_meta_file,
|
|
1490
1511
|
"train_val_split": self._train_val_split_file,
|
|
1512
|
+
"train_size": len(self._train_split),
|
|
1513
|
+
"val_size": len(self._val_split),
|
|
1491
1514
|
"hyperparameters": self._hyperparameters_file,
|
|
1492
1515
|
"artifacts_dir": remote_dir,
|
|
1493
1516
|
"datetime": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
|
1494
1517
|
"evaluation_report_id": evaluation_report_id,
|
|
1518
|
+
"evaluation_report_link": evaluation_report_link,
|
|
1495
1519
|
"evaluation_metrics": eval_metrics,
|
|
1520
|
+
"logs": {"type": "tensorboard", "link": f"{remote_dir}logs/"},
|
|
1496
1521
|
}
|
|
1497
1522
|
|
|
1498
1523
|
remote_checkpoints_dir = join(remote_dir, self._remote_checkpoints_dir_name)
|
|
@@ -1519,6 +1544,10 @@ class TrainApp:
|
|
|
1519
1544
|
f"Uploading '{self._experiment_json_file}' to Team Files",
|
|
1520
1545
|
)
|
|
1521
1546
|
|
|
1547
|
+
# Do not include this fields to uploaded file:
|
|
1548
|
+
experiment_info["primary_metric"] = primary_metric_name
|
|
1549
|
+
return experiment_info
|
|
1550
|
+
|
|
1522
1551
|
def _generate_hyperparameters(self, remote_dir: str, experiment_info: Dict) -> None:
|
|
1523
1552
|
"""
|
|
1524
1553
|
Generates and uploads the hyperparameters.yaml file to the output directory.
|
|
@@ -1534,11 +1563,13 @@ class TrainApp:
|
|
|
1534
1563
|
with open(local_path, "w") as file:
|
|
1535
1564
|
file.write(self.hyperparameters_yaml)
|
|
1536
1565
|
|
|
1537
|
-
self._upload_file_to_team_files(
|
|
1566
|
+
file_info = self._upload_file_to_team_files(
|
|
1538
1567
|
local_path,
|
|
1539
1568
|
remote_path,
|
|
1540
1569
|
f"Uploading '{self._hyperparameters_file}' to Team Files",
|
|
1541
1570
|
)
|
|
1571
|
+
experiment_info["hyperparameters_id"] = file_info.id
|
|
1572
|
+
return experiment_info
|
|
1542
1573
|
|
|
1543
1574
|
def _generate_app_state(self, remote_dir: str, experiment_info: Dict) -> None:
|
|
1544
1575
|
"""
|
|
@@ -1702,10 +1733,20 @@ class TrainApp:
|
|
|
1702
1733
|
self.progress_bar_main.hide()
|
|
1703
1734
|
|
|
1704
1735
|
file_info = self._api.file.get_info_by_path(self.team_id, join(remote_dir, "open_app.lnk"))
|
|
1736
|
+
# Set offline tensorboard button payload
|
|
1737
|
+
if is_production():
|
|
1738
|
+
self.gui.training_logs.tensorboard_offline_button.payload = {
|
|
1739
|
+
"state": {"slyFolder": f"{join(remote_dir, 'logs')}"}
|
|
1740
|
+
}
|
|
1741
|
+
self.gui.training_logs.tensorboard_offline_button.enable()
|
|
1705
1742
|
return remote_dir, file_info
|
|
1706
1743
|
|
|
1707
1744
|
def _set_training_output(
|
|
1708
|
-
self,
|
|
1745
|
+
self,
|
|
1746
|
+
experiment_info: dict,
|
|
1747
|
+
remote_dir: str,
|
|
1748
|
+
file_info: FileInfo,
|
|
1749
|
+
mb_eval_report=None,
|
|
1709
1750
|
) -> None:
|
|
1710
1751
|
"""
|
|
1711
1752
|
Sets the training output in the GUI.
|
|
@@ -1718,6 +1759,7 @@ class TrainApp:
|
|
|
1718
1759
|
# self.gui.training_logs.tensorboard_button.disable()
|
|
1719
1760
|
|
|
1720
1761
|
# Set artifacts to GUI
|
|
1762
|
+
self._api.task.set_output_experiment(self.task_id, experiment_info)
|
|
1721
1763
|
set_directory(remote_dir)
|
|
1722
1764
|
self.gui.training_artifacts.artifacts_thumbnail.set(file_info)
|
|
1723
1765
|
self.gui.training_artifacts.artifacts_thumbnail.show()
|
|
@@ -1742,13 +1784,15 @@ class TrainApp:
|
|
|
1742
1784
|
if demo_path is not None:
|
|
1743
1785
|
# Show PyTorch demo if available
|
|
1744
1786
|
if self.gui.training_artifacts.pytorch_demo_exists(demo_path):
|
|
1745
|
-
self.gui.training_artifacts.pytorch_instruction
|
|
1787
|
+
if self.gui.training_artifacts.pytorch_instruction is not None:
|
|
1788
|
+
self.gui.training_artifacts.pytorch_instruction.show()
|
|
1746
1789
|
|
|
1747
1790
|
# Show ONNX demo if supported and available
|
|
1748
1791
|
if (
|
|
1749
1792
|
self._app_options.get("export_onnx_supported", False)
|
|
1750
1793
|
and self.gui.hyperparameters_selector.get_export_onnx_checkbox_value()
|
|
1751
1794
|
and self.gui.training_artifacts.onnx_demo_exists(demo_path)
|
|
1795
|
+
and self.gui.training_artifacts.onnx_instruction is not None
|
|
1752
1796
|
):
|
|
1753
1797
|
self.gui.training_artifacts.onnx_instruction.show()
|
|
1754
1798
|
|
|
@@ -1757,6 +1801,7 @@ class TrainApp:
|
|
|
1757
1801
|
self._app_options.get("export_tensorrt_supported", False)
|
|
1758
1802
|
and self.gui.hyperparameters_selector.get_export_tensorrt_checkbox_value()
|
|
1759
1803
|
and self.gui.training_artifacts.trt_demo_exists(demo_path)
|
|
1804
|
+
and self.gui.training_artifacts.trt_instruction is not None
|
|
1760
1805
|
):
|
|
1761
1806
|
self.gui.training_artifacts.trt_instruction.show()
|
|
1762
1807
|
|
|
@@ -1983,6 +2028,8 @@ class TrainApp:
|
|
|
1983
2028
|
report = bm.report
|
|
1984
2029
|
report_id = bm.report.id
|
|
1985
2030
|
eval_metrics = bm.key_metrics
|
|
2031
|
+
primary_metric_name = bm.primary_metric_name
|
|
2032
|
+
bm.upload_report_link(remote_artifacts_dir, report_id, self.output_dir)
|
|
1986
2033
|
|
|
1987
2034
|
# 8. UI updates
|
|
1988
2035
|
self.progress_bar_main.hide()
|
|
@@ -1991,6 +2038,7 @@ class TrainApp:
|
|
|
1991
2038
|
logger.info(
|
|
1992
2039
|
f"Predictions project name: {bm.dt_project_info.name}. Workspace_id: {bm.dt_project_info.workspace_id}"
|
|
1993
2040
|
)
|
|
2041
|
+
|
|
1994
2042
|
except Exception as e:
|
|
1995
2043
|
logger.error(f"Model benchmark failed. {repr(e)}", exc_info=True)
|
|
1996
2044
|
self._set_text_status("finalizing")
|
|
@@ -2004,7 +2052,7 @@ class TrainApp:
|
|
|
2004
2052
|
self._api.project.remove(diff_project_info.id)
|
|
2005
2053
|
except Exception as e2:
|
|
2006
2054
|
return lnk_file_info, report, report_id, eval_metrics
|
|
2007
|
-
return lnk_file_info, report, report_id, eval_metrics
|
|
2055
|
+
return lnk_file_info, report, report_id, eval_metrics, primary_metric_name
|
|
2008
2056
|
|
|
2009
2057
|
# ----------------------------------------- #
|
|
2010
2058
|
|
|
@@ -2476,7 +2524,11 @@ class TrainApp:
|
|
|
2476
2524
|
def _convert_and_split_gt_project(self, task_type: str):
|
|
2477
2525
|
# 1. Convert GT project to cv task
|
|
2478
2526
|
Project.download(
|
|
2479
|
-
self._api,
|
|
2527
|
+
self._api,
|
|
2528
|
+
self.project_info.id,
|
|
2529
|
+
"tmp_project",
|
|
2530
|
+
save_images=False,
|
|
2531
|
+
save_image_info=True,
|
|
2480
2532
|
)
|
|
2481
2533
|
project = Project("tmp_project", OpenMode.READ)
|
|
2482
2534
|
|