datamint 1.2.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of datamint might be problematic. Click here for more details.
- datamint/__init__.py +11 -0
- datamint-1.2.4.dist-info/METADATA +118 -0
- datamint-1.2.4.dist-info/RECORD +30 -0
- datamint-1.2.4.dist-info/WHEEL +4 -0
- datamint-1.2.4.dist-info/entry_points.txt +4 -0
- datamintapi/__init__.py +25 -0
- datamintapi/apihandler/annotation_api_handler.py +748 -0
- datamintapi/apihandler/api_handler.py +15 -0
- datamintapi/apihandler/base_api_handler.py +300 -0
- datamintapi/apihandler/dto/annotation_dto.py +149 -0
- datamintapi/apihandler/exp_api_handler.py +204 -0
- datamintapi/apihandler/root_api_handler.py +1013 -0
- datamintapi/client_cmd_tools/__init__.py +0 -0
- datamintapi/client_cmd_tools/datamint_config.py +168 -0
- datamintapi/client_cmd_tools/datamint_upload.py +483 -0
- datamintapi/configs.py +58 -0
- datamintapi/dataset/__init__.py +1 -0
- datamintapi/dataset/base_dataset.py +881 -0
- datamintapi/dataset/dataset.py +492 -0
- datamintapi/examples/__init__.py +1 -0
- datamintapi/examples/example_projects.py +75 -0
- datamintapi/experiment/__init__.py +1 -0
- datamintapi/experiment/_patcher.py +570 -0
- datamintapi/experiment/experiment.py +1049 -0
- datamintapi/logging.yaml +27 -0
- datamintapi/utils/dicom_utils.py +640 -0
- datamintapi/utils/io_utils.py +149 -0
- datamintapi/utils/logging_utils.py +55 -0
- datamintapi/utils/torchmetrics.py +70 -0
- datamintapi/utils/visualization.py +129 -0
|
File without changes
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import logging
|
|
3
|
+
from datamintapi import configs
|
|
4
|
+
from datamintapi.utils.logging_utils import load_cmdline_logging_config
|
|
5
|
+
|
|
6
|
+
# Create two loggings: one for the user and one for the developer
|
|
7
|
+
_LOGGER = logging.getLogger(__name__)
|
|
8
|
+
_USER_LOGGER = logging.getLogger('user_logger')
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def configure_default_url():
|
|
12
|
+
"""Configure the default API URL interactively."""
|
|
13
|
+
_USER_LOGGER.info("Current default URL: %s", configs.get_value(configs.APIURL_KEY, 'Not set'))
|
|
14
|
+
url = input("Enter the default API URL (leave empty to abort): ").strip()
|
|
15
|
+
if url == '':
|
|
16
|
+
return
|
|
17
|
+
|
|
18
|
+
# Basic URL validation
|
|
19
|
+
if not (url.startswith('http://') or url.startswith('https://')):
|
|
20
|
+
_USER_LOGGER.warning("URL should start with http:// or https://")
|
|
21
|
+
return
|
|
22
|
+
|
|
23
|
+
configs.set_value(configs.APIURL_KEY, url)
|
|
24
|
+
_USER_LOGGER.info("Default API URL set successfully.")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def ask_api_key(ask_to_save: bool) -> str | None:
|
|
28
|
+
"""Ask user for API key with improved guidance."""
|
|
29
|
+
_USER_LOGGER.info("💡 Get your API key from your Datamint administrator or the web app (https://app.datamint.io/team)")
|
|
30
|
+
|
|
31
|
+
api_key = input('API key (leave empty to abort): ').strip()
|
|
32
|
+
if api_key == '':
|
|
33
|
+
return None
|
|
34
|
+
|
|
35
|
+
if ask_to_save:
|
|
36
|
+
ans = input("Save the API key so it automatically loads next time? (y/n): ")
|
|
37
|
+
try:
|
|
38
|
+
if ans.lower() == 'y':
|
|
39
|
+
configs.set_value(configs.APIKEY_KEY, api_key)
|
|
40
|
+
_USER_LOGGER.info("✅ API key saved.")
|
|
41
|
+
except Exception as e:
|
|
42
|
+
_USER_LOGGER.error("❌ Error saving API key.")
|
|
43
|
+
_LOGGER.exception(e)
|
|
44
|
+
return api_key
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def show_all_configurations():
|
|
48
|
+
"""Display all current configurations in a user-friendly format."""
|
|
49
|
+
config = configs.read_config()
|
|
50
|
+
if config is not None and len(config) > 0:
|
|
51
|
+
_USER_LOGGER.info("📋 Current configurations:")
|
|
52
|
+
for key, value in config.items():
|
|
53
|
+
# Mask API key for security
|
|
54
|
+
if key == configs.APIKEY_KEY and value:
|
|
55
|
+
masked_value = f"{value[:3]}...{value[-3:]}" if len(value) > 6 else value
|
|
56
|
+
_USER_LOGGER.info(f" {key}: {masked_value}")
|
|
57
|
+
else:
|
|
58
|
+
_USER_LOGGER.info(f" {key}: {value}")
|
|
59
|
+
else:
|
|
60
|
+
_USER_LOGGER.info("No configurations found.")
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def clear_all_configurations():
|
|
64
|
+
"""Clear all configurations with confirmation."""
|
|
65
|
+
yesno = input('Are you sure you want to clear all configurations? (y/n): ')
|
|
66
|
+
if yesno.lower() == 'y':
|
|
67
|
+
configs.clear_all_configurations()
|
|
68
|
+
_USER_LOGGER.info("All configurations cleared.")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def configure_api_key():
|
|
72
|
+
api_key = ask_api_key(ask_to_save=False)
|
|
73
|
+
if api_key is None:
|
|
74
|
+
return
|
|
75
|
+
configs.set_value(configs.APIKEY_KEY, api_key)
|
|
76
|
+
_USER_LOGGER.info("✅ API key saved.")
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def test_connection():
|
|
80
|
+
"""Test the API connection with current settings."""
|
|
81
|
+
try:
|
|
82
|
+
from datamintapi import APIHandler
|
|
83
|
+
_USER_LOGGER.info("🔄 Testing connection...")
|
|
84
|
+
api = APIHandler()
|
|
85
|
+
# Simple test - try to get projects
|
|
86
|
+
projects = api.get_projects()
|
|
87
|
+
_USER_LOGGER.info(f"✅ Connection successful! Found {len(projects)} projects.")
|
|
88
|
+
except ImportError:
|
|
89
|
+
_USER_LOGGER.error("❌ Full API not available. Install with: pip install datamint-python-api[full]")
|
|
90
|
+
except Exception as e:
|
|
91
|
+
_USER_LOGGER.error(f"❌ Connection failed: {e}")
|
|
92
|
+
_USER_LOGGER.info("💡 Check your API key and URL settings")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def interactive_mode():
|
|
96
|
+
_USER_LOGGER.info("🔧 Datamint Configuration Tool")
|
|
97
|
+
|
|
98
|
+
if len(configs.read_config()) == 0:
|
|
99
|
+
_USER_LOGGER.info("👋 Welcome! Let's set up your API key first.")
|
|
100
|
+
configure_api_key()
|
|
101
|
+
|
|
102
|
+
while True:
|
|
103
|
+
_USER_LOGGER.info("\n📋 Select the action you want to perform:")
|
|
104
|
+
_USER_LOGGER.info(" (1) Configure the API key")
|
|
105
|
+
_USER_LOGGER.info(" (2) Configure the default URL")
|
|
106
|
+
_USER_LOGGER.info(" (3) Show all configuration settings")
|
|
107
|
+
_USER_LOGGER.info(" (4) Clear all configuration settings")
|
|
108
|
+
_USER_LOGGER.info(" (5) Test connection")
|
|
109
|
+
_USER_LOGGER.info(" (q) Exit")
|
|
110
|
+
choice = input("Enter your choice: ").lower().strip()
|
|
111
|
+
|
|
112
|
+
if choice == '1':
|
|
113
|
+
configure_api_key()
|
|
114
|
+
elif choice == '2':
|
|
115
|
+
configure_default_url()
|
|
116
|
+
elif choice == '3':
|
|
117
|
+
show_all_configurations()
|
|
118
|
+
elif choice == '4':
|
|
119
|
+
clear_all_configurations()
|
|
120
|
+
elif choice == '5':
|
|
121
|
+
test_connection()
|
|
122
|
+
elif choice in ('q', 'exit', 'quit'):
|
|
123
|
+
_USER_LOGGER.info("👋 Goodbye!")
|
|
124
|
+
break
|
|
125
|
+
else:
|
|
126
|
+
_USER_LOGGER.info("❌ Invalid choice. Please enter a number between 1 and 5 or 'q' to quit.")
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def main():
|
|
130
|
+
load_cmdline_logging_config()
|
|
131
|
+
parser = argparse.ArgumentParser(
|
|
132
|
+
description='🔧 Datamint API Configuration Tool',
|
|
133
|
+
epilog="""
|
|
134
|
+
Examples:
|
|
135
|
+
datamint-config # Interactive mode
|
|
136
|
+
datamint-config --api-key YOUR_KEY # Set API key
|
|
137
|
+
|
|
138
|
+
More Documentation: https://sonanceai.github.io/datamint-python-api/command_line_tools.html
|
|
139
|
+
""",
|
|
140
|
+
formatter_class=argparse.RawDescriptionHelpFormatter
|
|
141
|
+
)
|
|
142
|
+
parser.add_argument('--api-key', type=str, help='API key to set')
|
|
143
|
+
parser.add_argument('--default-url', '--url', type=str, help='Default URL to set')
|
|
144
|
+
parser.add_argument('-i', '--interactive', action='store_true',
|
|
145
|
+
help='Interactive mode (default if no other arguments provided)')
|
|
146
|
+
|
|
147
|
+
args = parser.parse_args()
|
|
148
|
+
|
|
149
|
+
if args.api_key is not None:
|
|
150
|
+
configs.set_value(configs.APIKEY_KEY, args.api_key)
|
|
151
|
+
_USER_LOGGER.info("✅ API key saved.")
|
|
152
|
+
|
|
153
|
+
if args.default_url is not None:
|
|
154
|
+
# Basic URL validation
|
|
155
|
+
if not (args.default_url.startswith('http://') or args.default_url.startswith('https://')):
|
|
156
|
+
_USER_LOGGER.error("❌ URL must start with http:// or https://")
|
|
157
|
+
return
|
|
158
|
+
configs.set_value(configs.APIURL_KEY, args.default_url)
|
|
159
|
+
_USER_LOGGER.info("✅ Default URL saved.")
|
|
160
|
+
|
|
161
|
+
no_arguments_provided = args.api_key is None and args.default_url is None
|
|
162
|
+
|
|
163
|
+
if no_arguments_provided or args.interactive:
|
|
164
|
+
interactive_mode()
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
if __name__ == "__main__":
|
|
168
|
+
main()
|
|
@@ -0,0 +1,483 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
from datamintapi.apihandler.api_handler import APIHandler
|
|
3
|
+
import os
|
|
4
|
+
from humanize import naturalsize
|
|
5
|
+
import logging
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import sys
|
|
8
|
+
from datamintapi.utils.dicom_utils import is_dicom
|
|
9
|
+
import fnmatch
|
|
10
|
+
from typing import Sequence, Generator, Optional, Any
|
|
11
|
+
from collections import defaultdict
|
|
12
|
+
from datamintapi import __version__ as datamintapi_version
|
|
13
|
+
from datamintapi import configs
|
|
14
|
+
from datamintapi.client_cmd_tools.datamint_config import ask_api_key
|
|
15
|
+
from datamintapi.utils.logging_utils import load_cmdline_logging_config
|
|
16
|
+
import yaml
|
|
17
|
+
|
|
18
|
+
# Create two loggings: one for the user and one for the developer
|
|
19
|
+
_LOGGER = logging.getLogger(__name__)
|
|
20
|
+
_USER_LOGGER = logging.getLogger('user_logger')
|
|
21
|
+
|
|
22
|
+
MAX_RECURSION_LIMIT = 1000
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _is_valid_path_argparse(x):
|
|
26
|
+
"""
|
|
27
|
+
argparse type that checks if the path exists
|
|
28
|
+
"""
|
|
29
|
+
if not os.path.exists(x):
|
|
30
|
+
raise argparse.ArgumentTypeError("{0} does not exist".format(x))
|
|
31
|
+
return x
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _tuple_int_type(x: str):
|
|
35
|
+
"""
|
|
36
|
+
argparse type that converts a string of two hexadecimal integers to a tuple of integers
|
|
37
|
+
"""
|
|
38
|
+
try:
|
|
39
|
+
x_processed = tuple(int(i, 16) for i in x.strip('()').split(','))
|
|
40
|
+
if len(x_processed) != 2:
|
|
41
|
+
raise ValueError
|
|
42
|
+
return x_processed
|
|
43
|
+
except ValueError:
|
|
44
|
+
raise argparse.ArgumentTypeError(
|
|
45
|
+
"Values must be two hexadecimal integers separated by a comma. Example (0x0008, 0x0050)"
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _mungfilename_type(arg):
|
|
50
|
+
if arg.lower() == 'all':
|
|
51
|
+
return 'all'
|
|
52
|
+
try:
|
|
53
|
+
ret = list(map(int, arg.split(',')))
|
|
54
|
+
# can only have positive values
|
|
55
|
+
if any(i <= 0 for i in ret):
|
|
56
|
+
raise ValueError
|
|
57
|
+
return ret
|
|
58
|
+
except ValueError:
|
|
59
|
+
raise argparse.ArgumentTypeError(
|
|
60
|
+
"Invalid value for --mungfilename. Expected 'all' or comma-separated positive integers.")
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _is_system_file(path: Path) -> bool:
|
|
64
|
+
"""
|
|
65
|
+
Check if a file is a system file that should be ignored
|
|
66
|
+
"""
|
|
67
|
+
# Common system files and folders to ignore
|
|
68
|
+
ignored_patterns = [
|
|
69
|
+
'.DS_Store',
|
|
70
|
+
'Thumbs.db',
|
|
71
|
+
'.git',
|
|
72
|
+
'__pycache__',
|
|
73
|
+
'*.pyc',
|
|
74
|
+
'.svn',
|
|
75
|
+
'.tmp',
|
|
76
|
+
'~*', # Temporary files created by some editors
|
|
77
|
+
'._*' # macOS resource fork files
|
|
78
|
+
]
|
|
79
|
+
|
|
80
|
+
# Check if path is inside a system folder
|
|
81
|
+
system_folders = ['__MACOSX', '$RECYCLE.BIN', 'System Volume Information']
|
|
82
|
+
if any(folder in path.parts for folder in system_folders):
|
|
83
|
+
return True
|
|
84
|
+
|
|
85
|
+
# Check if filename matches any ignored pattern
|
|
86
|
+
return any(fnmatch.fnmatch(path.name, pattern) for pattern in ignored_patterns)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def walk_to_depth(path: str,
|
|
90
|
+
depth: int,
|
|
91
|
+
exclude_pattern: str = None) -> Generator[Path, None, None]:
|
|
92
|
+
path = Path(path)
|
|
93
|
+
for child in path.iterdir():
|
|
94
|
+
if _is_system_file(child):
|
|
95
|
+
continue
|
|
96
|
+
|
|
97
|
+
if child.is_dir():
|
|
98
|
+
if depth != 0:
|
|
99
|
+
if exclude_pattern is not None and fnmatch.fnmatch(child.name, exclude_pattern):
|
|
100
|
+
continue
|
|
101
|
+
yield from walk_to_depth(child, depth-1, exclude_pattern)
|
|
102
|
+
else:
|
|
103
|
+
_LOGGER.debug(f"yielding {child} from {path}")
|
|
104
|
+
yield child
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def filter_files(files_path: Sequence[Path],
|
|
108
|
+
include_extensions,
|
|
109
|
+
exclude_extensions) -> list[Path]:
|
|
110
|
+
def fix_extension(ext: str) -> str:
|
|
111
|
+
if ext == "" or ext[0] == '.':
|
|
112
|
+
return ext
|
|
113
|
+
return '.' + ext
|
|
114
|
+
|
|
115
|
+
def normalize_extensions(exts_list: Sequence[str]) -> list[str]:
|
|
116
|
+
# explodes the extensions if they are separated by commas
|
|
117
|
+
exts_list = [ext.split(',') for ext in exts_list]
|
|
118
|
+
exts_list = [item for sublist in exts_list for item in sublist]
|
|
119
|
+
|
|
120
|
+
# adds a dot to the extensions if it does not have one
|
|
121
|
+
exts_list = [fix_extension(ext) for ext in exts_list]
|
|
122
|
+
|
|
123
|
+
return [fix_extension(ext) for ext in exts_list]
|
|
124
|
+
|
|
125
|
+
files_path = list(files_path)
|
|
126
|
+
# Filter out files less than 4 bytes
|
|
127
|
+
files_path2 = [f for f in files_path if f.stat().st_size >= 4]
|
|
128
|
+
if len(files_path) != len(files_path2):
|
|
129
|
+
_USER_LOGGER.info(f"Filtered out {len(files_path) - len(files_path2)} empty files")
|
|
130
|
+
files_path = files_path2
|
|
131
|
+
|
|
132
|
+
if include_extensions is not None:
|
|
133
|
+
include_extensions = normalize_extensions(include_extensions)
|
|
134
|
+
files_path = [f for f in files_path if f.suffix in include_extensions]
|
|
135
|
+
|
|
136
|
+
if exclude_extensions is not None:
|
|
137
|
+
exclude_extensions = normalize_extensions(exclude_extensions)
|
|
138
|
+
files_path = [f for f in files_path if f.suffix not in exclude_extensions]
|
|
139
|
+
|
|
140
|
+
return files_path
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def handle_api_key() -> str:
|
|
144
|
+
"""
|
|
145
|
+
Checks for API keys.
|
|
146
|
+
If it does not exist, it asks the user to input it.
|
|
147
|
+
Then, it asks the user if he wants to save the API key at a proper location in the machine
|
|
148
|
+
"""
|
|
149
|
+
api_key = configs.get_value(configs.APIKEY_KEY)
|
|
150
|
+
if api_key is None:
|
|
151
|
+
_USER_LOGGER.info("API key not found. Please provide it:")
|
|
152
|
+
api_key = ask_api_key(ask_to_save=True)
|
|
153
|
+
|
|
154
|
+
return api_key
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _find_segmentation_files(segmentation_root_path: str,
|
|
158
|
+
images_files: list[str],
|
|
159
|
+
segmentation_metainfo: dict = None
|
|
160
|
+
) -> Optional[list[dict]]:
|
|
161
|
+
"""
|
|
162
|
+
Find the segmentation files that match the images files based on the same folder structure
|
|
163
|
+
"""
|
|
164
|
+
|
|
165
|
+
if segmentation_root_path is None:
|
|
166
|
+
return None
|
|
167
|
+
|
|
168
|
+
if len(images_files) == 1 and os.path.isfile(images_files[0]) and os.path.isfile(segmentation_root_path):
|
|
169
|
+
return [{'files': [segmentation_root_path]}]
|
|
170
|
+
|
|
171
|
+
segmentation_files = []
|
|
172
|
+
acceptable_extensions = ['.nii.gz', '.nii', '.png']
|
|
173
|
+
|
|
174
|
+
if segmentation_metainfo is not None:
|
|
175
|
+
if 'segmentation_names' in segmentation_metainfo:
|
|
176
|
+
segnames = sorted(segmentation_metainfo['segmentation_names'],
|
|
177
|
+
key=lambda x: len(x))
|
|
178
|
+
else:
|
|
179
|
+
segnames = None
|
|
180
|
+
classnames = segmentation_metainfo.get('class_names', None)
|
|
181
|
+
if classnames is not None:
|
|
182
|
+
_LOGGER.debug(f"Number of class names: {len(classnames)}")
|
|
183
|
+
|
|
184
|
+
segmentation_root_path = Path(segmentation_root_path).absolute()
|
|
185
|
+
|
|
186
|
+
for imgpath in images_files:
|
|
187
|
+
imgpath_parent = Path(imgpath).absolute().parent
|
|
188
|
+
# Find the closest common parent between the image and the segmentation root
|
|
189
|
+
common_parent = []
|
|
190
|
+
for imgpath_part, segpath_part in zip(imgpath_parent.parts, segmentation_root_path.parent.parts):
|
|
191
|
+
if imgpath_part != segpath_part:
|
|
192
|
+
break
|
|
193
|
+
common_parent.append(imgpath_part)
|
|
194
|
+
if len(common_parent) == 0:
|
|
195
|
+
common_parent = Path('/')
|
|
196
|
+
else:
|
|
197
|
+
common_parent = Path(*common_parent)
|
|
198
|
+
|
|
199
|
+
_LOGGER.debug(f"_find_segmentation_files::common_parent: {common_parent}")
|
|
200
|
+
path_structure = imgpath_parent.relative_to(common_parent).parts[1:]
|
|
201
|
+
|
|
202
|
+
# path_structure = imgpath_parent.relative_to(root_path).parts[1:]
|
|
203
|
+
path_structure = Path(*path_structure)
|
|
204
|
+
|
|
205
|
+
real_seg_root_path = common_parent / Path(Path(segmentation_root_path).relative_to(common_parent).parts[0])
|
|
206
|
+
seg_path = real_seg_root_path / path_structure
|
|
207
|
+
# list all segmentation files (nii.gz, nii, png) in the same folder structure
|
|
208
|
+
seg_files = [fname for ext in acceptable_extensions for fname in seg_path.glob(f'*{ext}')]
|
|
209
|
+
if len(seg_files) == 0:
|
|
210
|
+
filename = Path(imgpath).stem
|
|
211
|
+
seg_path = seg_path / filename
|
|
212
|
+
seg_files = [fname for ext in acceptable_extensions for fname in seg_path.glob(f'*{ext}')]
|
|
213
|
+
|
|
214
|
+
if len(seg_files) > 0:
|
|
215
|
+
seginfo = {
|
|
216
|
+
'files': [str(f) for f in seg_files]
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
frame_indices = []
|
|
220
|
+
for segfile in seg_files:
|
|
221
|
+
if segfile.suffix == '.png':
|
|
222
|
+
try:
|
|
223
|
+
frame_index = int(segfile.stem)
|
|
224
|
+
except ValueError:
|
|
225
|
+
frame_index = None
|
|
226
|
+
|
|
227
|
+
frame_indices.append(frame_index)
|
|
228
|
+
|
|
229
|
+
if len(frame_indices) > 0:
|
|
230
|
+
seginfo['frame_index'] = frame_indices
|
|
231
|
+
|
|
232
|
+
if segmentation_metainfo is not None:
|
|
233
|
+
snames_associated = []
|
|
234
|
+
for segfile in seg_files:
|
|
235
|
+
if segnames is None:
|
|
236
|
+
snames_associated.append(classnames)
|
|
237
|
+
else:
|
|
238
|
+
for segname in segnames:
|
|
239
|
+
if segname in str(segfile):
|
|
240
|
+
if classnames is not None:
|
|
241
|
+
new_segname = {cid: f'{segname}_{cname}' for cid, cname in classnames.items()}
|
|
242
|
+
new_segname.update({'default': segname})
|
|
243
|
+
else:
|
|
244
|
+
new_segname = segname
|
|
245
|
+
snames_associated.append(new_segname)
|
|
246
|
+
break
|
|
247
|
+
else:
|
|
248
|
+
_USER_LOGGER.warning(f"Segmentation file {segname} does not match any segmentation name.")
|
|
249
|
+
snames_associated.append(None)
|
|
250
|
+
seginfo['names'] = snames_associated
|
|
251
|
+
|
|
252
|
+
segmentation_files.append(seginfo)
|
|
253
|
+
else:
|
|
254
|
+
segmentation_files.append(None)
|
|
255
|
+
|
|
256
|
+
return segmentation_files
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _parse_args() -> tuple[Any, list, Optional[list[dict]]]:
|
|
260
|
+
parser = argparse.ArgumentParser(
|
|
261
|
+
description='DatamintAPI command line tool for uploading DICOM files and other resources')
|
|
262
|
+
parser.add_argument('--path', type=_is_valid_path_argparse, metavar="FILE",
|
|
263
|
+
required=True,
|
|
264
|
+
help='Path to the resource file(s) or a directory')
|
|
265
|
+
parser.add_argument('-r', '--recursive', nargs='?', const=-1, # -1 means infinite
|
|
266
|
+
type=int,
|
|
267
|
+
help='Recurse folders looking for DICOMs. If a number is passed, recurse that number of levels.')
|
|
268
|
+
parser.add_argument('--exclude', type=str,
|
|
269
|
+
help='Exclude folders that match the specified pattern. \
|
|
270
|
+
Example: "*_not_to_upload" will exclude folders ending with "_not_to_upload')
|
|
271
|
+
parser.add_argument('--channel', '--name', type=str, required=False,
|
|
272
|
+
help='Channel name (arbritary) to upload the resources to. \
|
|
273
|
+
Useful for organizing the resources in the platform.')
|
|
274
|
+
parser.add_argument('--retain-pii', action='store_true', help='Do not anonymize DICOMs')
|
|
275
|
+
parser.add_argument('--retain-attribute', type=_tuple_int_type, action='append',
|
|
276
|
+
default=[],
|
|
277
|
+
help='Retain the value of a single attribute code specified as hexidecimal integers. \
|
|
278
|
+
Example: (0x0008, 0x0050) or just (0008, 0050)')
|
|
279
|
+
parser.add_argument('-l', '--label', type=str, action='append', help='Deprecated. Use --tag instead.')
|
|
280
|
+
parser.add_argument('--tag', type=str, action='append', help='A tag name to be applied to all files')
|
|
281
|
+
parser.add_argument('--publish', action='store_true',
|
|
282
|
+
help='Publish the uploaded resources, giving them the status "published" instead of "inbox"')
|
|
283
|
+
parser.add_argument('--mungfilename', type=_mungfilename_type,
|
|
284
|
+
help='Change the filename in the upload parameters. \
|
|
285
|
+
If set to "all", the filename becomes the folder names joined together with "_". \
|
|
286
|
+
If one or more integers are passed (comma-separated), append that depth of folder name to the filename.')
|
|
287
|
+
parser.add_argument('--include-extensions', type=str, nargs='+',
|
|
288
|
+
help='File extensions to be considered for uploading. Default: all file extensions.' +
|
|
289
|
+
' Example: --include-extensions dcm jpg png')
|
|
290
|
+
parser.add_argument('--exclude-extensions', type=str, nargs='+',
|
|
291
|
+
help='File extensions to be excluded from uploading. Default: none.' +
|
|
292
|
+
' Example: --exclude-extensions txt csv'
|
|
293
|
+
)
|
|
294
|
+
parser.add_argument('--segmentation_path', type=_is_valid_path_argparse, metavar="FILE",
|
|
295
|
+
required=False,
|
|
296
|
+
help='Path to the segmentation file(s) or a directory')
|
|
297
|
+
parser.add_argument('--segmentation_names', type=_is_valid_path_argparse, metavar="FILE",
|
|
298
|
+
required=False,
|
|
299
|
+
help='Path to a yaml file containing the segmentation names.' +
|
|
300
|
+
' The file may contain two keys: "segmentation_names" and "class_names".')
|
|
301
|
+
parser.add_argument('--yes', action='store_true',
|
|
302
|
+
help='Automatically answer yes to all prompts')
|
|
303
|
+
parser.add_argument('--transpose-segmentation', action='store_true', default=False,
|
|
304
|
+
help='Transpose the segmentation dimensions to match the image dimensions')
|
|
305
|
+
parser.add_argument('--version', action='version', version=f'%(prog)s {datamintapi_version}')
|
|
306
|
+
parser.add_argument('--verbose', action='store_true', help='Print debug messages', default=False)
|
|
307
|
+
args = parser.parse_args()
|
|
308
|
+
if args.verbose:
|
|
309
|
+
# Get the console handler and set to debug
|
|
310
|
+
logging.getLogger().handlers[0].setLevel(logging.DEBUG)
|
|
311
|
+
_LOGGER.setLevel(logging.DEBUG)
|
|
312
|
+
_USER_LOGGER.setLevel(logging.DEBUG)
|
|
313
|
+
|
|
314
|
+
if args.retain_pii and len(args.retain_attribute) > 0:
|
|
315
|
+
raise ValueError("Cannot use --retain-pii and --retain-attribute together.")
|
|
316
|
+
|
|
317
|
+
# include-extensions and exclude-extensions are mutually exclusive
|
|
318
|
+
if args.include_extensions is not None and args.exclude_extensions is not None:
|
|
319
|
+
raise ValueError("--include-extensions and --exclude-extensions are mutually exclusive.")
|
|
320
|
+
|
|
321
|
+
try:
|
|
322
|
+
|
|
323
|
+
if os.path.isfile(args.path):
|
|
324
|
+
file_path = [args.path]
|
|
325
|
+
if args.recursive is not None:
|
|
326
|
+
_USER_LOGGER.warning("Recursive flag ignored. Specified path is a file.")
|
|
327
|
+
else:
|
|
328
|
+
try:
|
|
329
|
+
recursive_depth = 0 if args.recursive is None else args.recursive
|
|
330
|
+
file_path = walk_to_depth(args.path, recursive_depth, args.exclude)
|
|
331
|
+
file_path = filter_files(file_path, args.include_extensions, args.exclude_extensions)
|
|
332
|
+
file_path = list(map(str, file_path)) # from Path to str
|
|
333
|
+
except Exception as e:
|
|
334
|
+
_LOGGER.error(f'Error in recursive search: {e}')
|
|
335
|
+
raise e
|
|
336
|
+
|
|
337
|
+
if len(file_path) == 0:
|
|
338
|
+
raise ValueError(f"No valid file was found in {args.path}")
|
|
339
|
+
|
|
340
|
+
if args.segmentation_names is not None:
|
|
341
|
+
with open(args.segmentation_names, 'r') as f:
|
|
342
|
+
segmentation_names = yaml.safe_load(f)
|
|
343
|
+
else:
|
|
344
|
+
segmentation_names = None
|
|
345
|
+
|
|
346
|
+
_LOGGER.debug(f'finding segmentations at {args.segmentation_path}')
|
|
347
|
+
segmentation_files = _find_segmentation_files(args.segmentation_path,
|
|
348
|
+
file_path,
|
|
349
|
+
segmentation_metainfo=segmentation_names)
|
|
350
|
+
|
|
351
|
+
_LOGGER.info(f"args parsed: {args}")
|
|
352
|
+
|
|
353
|
+
api_key = handle_api_key()
|
|
354
|
+
if api_key is None:
|
|
355
|
+
_USER_LOGGER.error("API key not provided. Aborting.")
|
|
356
|
+
sys.exit(1)
|
|
357
|
+
os.environ[configs.ENV_VARS[configs.APIKEY_KEY]] = api_key
|
|
358
|
+
|
|
359
|
+
if args.tag is not None and args.label is not None:
|
|
360
|
+
raise ValueError("Cannot use both --tag and --label. Use --tag instead. --label is deprecated.")
|
|
361
|
+
args.tag = args.tag if args.tag is not None else args.label
|
|
362
|
+
|
|
363
|
+
return args, file_path, segmentation_files
|
|
364
|
+
|
|
365
|
+
except Exception as e:
|
|
366
|
+
if args.verbose:
|
|
367
|
+
_LOGGER.exception(e)
|
|
368
|
+
raise e
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def print_input_summary(files_path: list[str],
|
|
372
|
+
args,
|
|
373
|
+
segfiles: Optional[list[dict]],
|
|
374
|
+
include_extensions=None):
|
|
375
|
+
### Create a summary of the upload ###
|
|
376
|
+
total_files = len(files_path)
|
|
377
|
+
total_size = sum(os.path.getsize(file) for file in files_path)
|
|
378
|
+
|
|
379
|
+
# Count number of files per extension
|
|
380
|
+
ext_dict = defaultdict(int)
|
|
381
|
+
for file in files_path:
|
|
382
|
+
ext_dict[os.path.splitext(file)[1]] += 1
|
|
383
|
+
|
|
384
|
+
# sorts the extensions by count
|
|
385
|
+
ext_counts = [(ext, count) for ext, count in ext_dict.items()]
|
|
386
|
+
ext_counts.sort(key=lambda x: x[1], reverse=True)
|
|
387
|
+
|
|
388
|
+
_USER_LOGGER.info(f"Number of files to be uploaded: {total_files}")
|
|
389
|
+
_USER_LOGGER.info(f"\t{files_path[0]}")
|
|
390
|
+
if total_files >= 2:
|
|
391
|
+
if total_files >= 3:
|
|
392
|
+
_USER_LOGGER.info("\t(...)")
|
|
393
|
+
_USER_LOGGER.info(f"\t{files_path[-1]}")
|
|
394
|
+
_USER_LOGGER.info(f"Total size of the upload: {naturalsize(total_size)}")
|
|
395
|
+
_USER_LOGGER.info(f"Number of files per extension:")
|
|
396
|
+
for ext, count in ext_counts:
|
|
397
|
+
if ext == '':
|
|
398
|
+
ext = 'no extension'
|
|
399
|
+
_USER_LOGGER.info(f"\t{ext}: {count}")
|
|
400
|
+
if len(ext_counts) > 1 and include_extensions is None:
|
|
401
|
+
_USER_LOGGER.warning("Multiple file extensions found!" +
|
|
402
|
+
" Make sure you are uploading the correct files.")
|
|
403
|
+
|
|
404
|
+
if segfiles is not None:
|
|
405
|
+
num_segfiles = sum([1 if seg is not None else 0 for seg in segfiles])
|
|
406
|
+
msg = f"Number of images with an associated segmentation: " +\
|
|
407
|
+
f"{num_segfiles} ({num_segfiles / total_files:.0%})"
|
|
408
|
+
if num_segfiles == 0:
|
|
409
|
+
_USER_LOGGER.warning(msg)
|
|
410
|
+
else:
|
|
411
|
+
_USER_LOGGER.info(msg)
|
|
412
|
+
# count number of segmentations files with names
|
|
413
|
+
if args.segmentation_names is not None and num_segfiles > 0:
|
|
414
|
+
segnames_count = sum([1 if 'names' in seg else 0 for seg in segfiles if seg is not None])
|
|
415
|
+
msg = f"Number of segmentations with associated name: " + \
|
|
416
|
+
f"{segnames_count} ({segnames_count / num_segfiles:.0%})"
|
|
417
|
+
if segnames_count == 0:
|
|
418
|
+
_USER_LOGGER.warning(msg)
|
|
419
|
+
else:
|
|
420
|
+
_USER_LOGGER.info(msg)
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def print_results_summary(files_path: list[str],
|
|
424
|
+
results: list[str | Exception]):
|
|
425
|
+
# Check for failed uploads
|
|
426
|
+
failure_files = [f for f, r in zip(files_path, results) if isinstance(r, Exception)]
|
|
427
|
+
_USER_LOGGER.info(f"\nUpload summary:")
|
|
428
|
+
_USER_LOGGER.info(f"\tTotal files: {len(files_path)}")
|
|
429
|
+
_USER_LOGGER.info(f"\tSuccessful uploads: {len(files_path) - len(failure_files)}")
|
|
430
|
+
_USER_LOGGER.info(f"\tFailed uploads: {len(failure_files)}")
|
|
431
|
+
if len(failure_files) > 0:
|
|
432
|
+
_USER_LOGGER.warning(f"\tFailed files: {failure_files}")
|
|
433
|
+
_USER_LOGGER.warning(f"\nFailures:")
|
|
434
|
+
for f, r in zip(files_path, results):
|
|
435
|
+
_LOGGER.debug(f"Failure: {f} - {r}")
|
|
436
|
+
if isinstance(r, Exception):
|
|
437
|
+
_USER_LOGGER.warning(f"\t{os.path.basename(f)}: {r}")
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def main():
|
|
441
|
+
load_cmdline_logging_config()
|
|
442
|
+
|
|
443
|
+
try:
|
|
444
|
+
args, files_path, segfiles = _parse_args()
|
|
445
|
+
except Exception as e:
|
|
446
|
+
_USER_LOGGER.error(f'Error validating arguments. {e}')
|
|
447
|
+
return
|
|
448
|
+
|
|
449
|
+
print_input_summary(files_path,
|
|
450
|
+
args=args,
|
|
451
|
+
segfiles=segfiles,
|
|
452
|
+
include_extensions=args.include_extensions)
|
|
453
|
+
|
|
454
|
+
if not args.yes:
|
|
455
|
+
confirmation = input("Do you want to proceed with the upload? (y/n): ")
|
|
456
|
+
if confirmation.lower() != "y":
|
|
457
|
+
_USER_LOGGER.info("Upload cancelled.")
|
|
458
|
+
return
|
|
459
|
+
#######################################
|
|
460
|
+
|
|
461
|
+
has_a_dicom_file = any(is_dicom(f) for f in files_path)
|
|
462
|
+
|
|
463
|
+
api_handler = APIHandler()
|
|
464
|
+
results = api_handler.upload_resources(channel=args.channel,
|
|
465
|
+
files_path=files_path,
|
|
466
|
+
tags=args.tag,
|
|
467
|
+
on_error='skip',
|
|
468
|
+
anonymize=args.retain_pii == False and has_a_dicom_file,
|
|
469
|
+
anonymize_retain_codes=args.retain_attribute,
|
|
470
|
+
mung_filename=args.mungfilename,
|
|
471
|
+
publish=args.publish,
|
|
472
|
+
segmentation_files=segfiles,
|
|
473
|
+
transpose_segmentation=args.transpose_segmentation,
|
|
474
|
+
assemble_dicoms=True
|
|
475
|
+
)
|
|
476
|
+
_USER_LOGGER.info('Upload finished!')
|
|
477
|
+
_LOGGER.debug(f"Number of results: {len(results)}")
|
|
478
|
+
|
|
479
|
+
print_results_summary(files_path, results)
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
if __name__ == '__main__':
|
|
483
|
+
main()
|