synapse-sdk 1.0.0a64__py3-none-any.whl → 1.0.0a66__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse-sdk might be problematic. Click here for more details.
- synapse_sdk/clients/backend/__init__.py +20 -1
- synapse_sdk/plugins/categories/base.py +1 -0
- synapse_sdk/plugins/categories/neural_net/base/inference.py +1 -4
- synapse_sdk/plugins/models.py +12 -6
- synapse_sdk/utils/converters/__init__.py +135 -72
- synapse_sdk/utils/converters/coco/to_dm.py +113 -0
- synapse_sdk/utils/storage/providers/http.py +190 -0
- synapse_sdk/utils/storage/registry.py +3 -0
- {synapse_sdk-1.0.0a64.dist-info → synapse_sdk-1.0.0a66.dist-info}/METADATA +1 -1
- {synapse_sdk-1.0.0a64.dist-info → synapse_sdk-1.0.0a66.dist-info}/RECORD +14 -12
- {synapse_sdk-1.0.0a64.dist-info → synapse_sdk-1.0.0a66.dist-info}/WHEEL +0 -0
- {synapse_sdk-1.0.0a64.dist-info → synapse_sdk-1.0.0a66.dist-info}/entry_points.txt +0 -0
- {synapse_sdk-1.0.0a64.dist-info → synapse_sdk-1.0.0a66.dist-info}/licenses/LICENSE +0 -0
- {synapse_sdk-1.0.0a64.dist-info → synapse_sdk-1.0.0a66.dist-info}/top_level.txt +0 -0
|
@@ -14,19 +14,38 @@ class BackendClient(
|
|
|
14
14
|
MLClientMixin,
|
|
15
15
|
HITLClientMixin,
|
|
16
16
|
):
|
|
17
|
+
"""BackendClient is a client for the synapse backend API.
|
|
18
|
+
|
|
19
|
+
* Access token overrides authorization token and tenant token.
|
|
20
|
+
|
|
21
|
+
Attrs:
|
|
22
|
+
access_token (str): The synapse access token for the synapse backend API.
|
|
23
|
+
authorization_token (str): The authorization token for the synapse backend API.
|
|
24
|
+
tenant_token (str): The tenant token for the synapse backend API.
|
|
25
|
+
agent_token (str): The agent token for the backend API.
|
|
26
|
+
"""
|
|
27
|
+
|
|
17
28
|
name = 'Backend'
|
|
18
29
|
access_token = None
|
|
30
|
+
authorization_token = None
|
|
31
|
+
tenant_token = None
|
|
19
32
|
agent_token = None
|
|
20
33
|
|
|
21
|
-
def __init__(self, base_url, access_token=None, agent_token=None, **kwargs):
|
|
34
|
+
def __init__(self, base_url, access_token=None, token=None, tenant=None, agent_token=None, **kwargs):
|
|
22
35
|
super().__init__(base_url)
|
|
23
36
|
self.access_token = access_token
|
|
37
|
+
self.authorization_token = token
|
|
38
|
+
self.tenant_token = tenant
|
|
24
39
|
self.agent_token = agent_token
|
|
25
40
|
|
|
26
41
|
def _get_headers(self):
|
|
27
42
|
headers = {}
|
|
28
43
|
if self.access_token:
|
|
29
44
|
headers['Synapse-Access-Token'] = f'Token {self.access_token}'
|
|
45
|
+
if self.authorization_token:
|
|
46
|
+
headers['Authorization'] = f'Token {self.authorization_token}'
|
|
47
|
+
if self.tenant_token:
|
|
48
|
+
headers['Synapse-Tenant'] = f'Token {self.tenant_token}'
|
|
30
49
|
if self.agent_token:
|
|
31
50
|
headers['SYNAPSE-Agent'] = f'Token {self.agent_token}'
|
|
32
51
|
return headers
|
|
@@ -5,7 +5,6 @@ from fastapi import FastAPI
|
|
|
5
5
|
from ray import serve
|
|
6
6
|
|
|
7
7
|
from synapse_sdk.clients.backend import BackendClient
|
|
8
|
-
from synapse_sdk.devtools.config import get_backend_config
|
|
9
8
|
from synapse_sdk.utils.file import unarchive
|
|
10
9
|
|
|
11
10
|
app = FastAPI()
|
|
@@ -21,9 +20,7 @@ class BaseInference:
|
|
|
21
20
|
@serve.multiplexed()
|
|
22
21
|
async def _load_model(self, model_id: str):
|
|
23
22
|
model_info = jwt.decode(model_id, self.backend_url, algorithms='HS256')
|
|
24
|
-
|
|
25
|
-
raise ValueError('Backend config not found. Please configure backend in devtools config.')
|
|
26
|
-
client = BackendClient(config['host'], access_token=config['token'])
|
|
23
|
+
client = BackendClient(self.backend_url, token=model_info['token'], tenant=model_info['tenant'])
|
|
27
24
|
model = client.get_model(model_info['model'])
|
|
28
25
|
with tempfile.TemporaryDirectory() as temp_path:
|
|
29
26
|
unarchive(model['file'], temp_path)
|
synapse_sdk/plugins/models.py
CHANGED
|
@@ -113,12 +113,18 @@ class Run:
|
|
|
113
113
|
def __init__(self, job_id, context):
|
|
114
114
|
self.job_id = job_id
|
|
115
115
|
self.context = context
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
116
|
+
config = get_backend_config()
|
|
117
|
+
if config:
|
|
118
|
+
self.client = BackendClient(
|
|
119
|
+
config['host'],
|
|
120
|
+
access_token=config['token'],
|
|
121
|
+
)
|
|
122
|
+
else:
|
|
123
|
+
self.client = BackendClient(
|
|
124
|
+
self.context['envs']['SYNAPSE_PLUGIN_RUN_HOST'],
|
|
125
|
+
token=self.context['envs'].get('SYNAPSE_PLUGIN_RUN_USER_TOKEN'),
|
|
126
|
+
tenant=self.context['envs'].get('SYNAPSE_PLUGIN_RUN_TENANT'),
|
|
127
|
+
)
|
|
122
128
|
self.set_logger()
|
|
123
129
|
|
|
124
130
|
def set_logger(self):
|
|
@@ -1,7 +1,72 @@
|
|
|
1
|
+
import json
|
|
1
2
|
import os
|
|
3
|
+
import shutil
|
|
4
|
+
import uuid
|
|
2
5
|
|
|
3
6
|
|
|
4
|
-
class
|
|
7
|
+
class BaseConverter:
|
|
8
|
+
"""Base class for shared logic between converters."""
|
|
9
|
+
|
|
10
|
+
def __init__(self, root_dir: str, is_categorized_dataset: bool = False) -> None:
|
|
11
|
+
self.root_dir: str = root_dir
|
|
12
|
+
self.is_categorized_dataset: bool = is_categorized_dataset
|
|
13
|
+
self.converted_data = None
|
|
14
|
+
|
|
15
|
+
@staticmethod
|
|
16
|
+
def ensure_dir(path: str) -> None:
|
|
17
|
+
"""Ensure that the directory exists, creating it if necessary."""
|
|
18
|
+
if not os.path.exists(path):
|
|
19
|
+
os.makedirs(path)
|
|
20
|
+
|
|
21
|
+
def _validate_required_dirs(self, dirs):
|
|
22
|
+
"""Validate that all required directories exist."""
|
|
23
|
+
for name, path in dirs.items():
|
|
24
|
+
if not os.path.exists(path):
|
|
25
|
+
raise FileNotFoundError(f'[ERROR] Required directory "{name}" does not exist at {path}')
|
|
26
|
+
|
|
27
|
+
def _validate_optional_dirs(self, dirs):
|
|
28
|
+
"""Validate optional directories and return those that exist."""
|
|
29
|
+
existing_dirs = {}
|
|
30
|
+
for name, path in dirs.items():
|
|
31
|
+
if os.path.exists(path):
|
|
32
|
+
existing_dirs[name] = path
|
|
33
|
+
else:
|
|
34
|
+
print(f'[WARNING] Optional directory "{name}" does not exist. Skipping.')
|
|
35
|
+
return existing_dirs
|
|
36
|
+
|
|
37
|
+
def _validate_splits(self, required_splits, optional_splits=[]):
|
|
38
|
+
"""Validate required and optional splits in the dataset."""
|
|
39
|
+
splits = {}
|
|
40
|
+
|
|
41
|
+
if self.is_categorized_dataset:
|
|
42
|
+
required_dirs = {split: os.path.join(self.root_dir, split) for split in required_splits}
|
|
43
|
+
self._validate_required_dirs(required_dirs)
|
|
44
|
+
splits.update(required_dirs)
|
|
45
|
+
|
|
46
|
+
optional_dirs = {split: os.path.join(self.root_dir, split) for split in optional_splits}
|
|
47
|
+
splits.update(self._validate_optional_dirs(optional_dirs))
|
|
48
|
+
else:
|
|
49
|
+
required_dirs = {
|
|
50
|
+
'json': os.path.join(self.root_dir, 'json'),
|
|
51
|
+
'original_file': os.path.join(self.root_dir, 'original_file'),
|
|
52
|
+
}
|
|
53
|
+
self._validate_required_dirs(required_dirs)
|
|
54
|
+
splits['root'] = self.root_dir
|
|
55
|
+
|
|
56
|
+
return splits
|
|
57
|
+
|
|
58
|
+
def _set_directories(self, split=None):
|
|
59
|
+
"""Set `self.json_dir` and `self.original_file_dir` based on the dataset split."""
|
|
60
|
+
if split:
|
|
61
|
+
split_dir = os.path.join(self.root_dir, split)
|
|
62
|
+
self.json_dir = os.path.join(split_dir, 'json')
|
|
63
|
+
self.original_file_dir = os.path.join(split_dir, 'original_file')
|
|
64
|
+
else:
|
|
65
|
+
self.json_dir = os.path.join(self.root_dir, 'json')
|
|
66
|
+
self.original_file_dir = os.path.join(self.root_dir, 'original_file')
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class FromDMConverter(BaseConverter):
|
|
5
70
|
"""Base class for converting data from DM format to a specific format.
|
|
6
71
|
|
|
7
72
|
Attrs:
|
|
@@ -42,10 +107,8 @@ class FromDMConverter:
|
|
|
42
107
|
"""
|
|
43
108
|
|
|
44
109
|
def __init__(self, root_dir: str, is_categorized_dataset: bool = False) -> None:
|
|
45
|
-
|
|
46
|
-
self.is_categorized_dataset: bool = is_categorized_dataset
|
|
110
|
+
super().__init__(root_dir, is_categorized_dataset)
|
|
47
111
|
self.version: str = '1.0'
|
|
48
|
-
self.converted_data = None
|
|
49
112
|
|
|
50
113
|
def convert(self):
|
|
51
114
|
"""Convert DM format to a specific format.
|
|
@@ -61,85 +124,85 @@ class FromDMConverter:
|
|
|
61
124
|
# Automatically call convert() if converted_data is not set
|
|
62
125
|
self.converted_data = self.convert()
|
|
63
126
|
|
|
64
|
-
@staticmethod
|
|
65
|
-
def ensure_dir(path: str) -> None:
|
|
66
|
-
"""Ensure that the directory exists, creating it if necessary."""
|
|
67
|
-
if not os.path.exists(path):
|
|
68
|
-
os.makedirs(path)
|
|
69
127
|
|
|
70
|
-
|
|
71
|
-
|
|
128
|
+
class ToDMConverter(BaseConverter):
|
|
129
|
+
"""Base class for converting data to DM format.
|
|
72
130
|
|
|
73
|
-
|
|
74
|
-
|
|
131
|
+
Attrs:
|
|
132
|
+
root_dir (str): Root directory containing data.
|
|
133
|
+
is_categorized_dataset (bool): Whether to handle train, test, valid splits.
|
|
134
|
+
converted_data: Holds the converted data after calling `convert()`.
|
|
75
135
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
raise FileNotFoundError(f'[ERROR] Required directory "{name}" does not exist at {path}')
|
|
136
|
+
Usage:
|
|
137
|
+
1. Subclass this base class and implement the `convert()` method.
|
|
138
|
+
2. Instantiate the converter with the required arguments.
|
|
139
|
+
3. Call `convert()` to perform the in-memory conversion and obtain the result as a dict or list of dicts.
|
|
140
|
+
4. Call `save_to_folder(output_dir)` to save the converted data and optionally copy original files.
|
|
82
141
|
|
|
83
|
-
|
|
84
|
-
|
|
142
|
+
Args:
|
|
143
|
+
root_dir (str): Path to the root directory containing data.
|
|
144
|
+
- If `is_categorized_dataset=True`, the directory should contain subdirectories for
|
|
145
|
+
`train`, `valid`, and optionally `test`.
|
|
146
|
+
- Each subdirectory should contain `annotations.json` and the corresponding image files.
|
|
147
|
+
- `train` and `valid` are required, while `test` is optional.
|
|
148
|
+
is_categorized_dataset (bool): Whether to handle train, test, valid splits.
|
|
85
149
|
|
|
86
|
-
|
|
87
|
-
|
|
150
|
+
Returns:
|
|
151
|
+
- convert(): Returns the converted data as a Python dict or a dictionary with keys for each split.
|
|
152
|
+
- save_to_folder(): Saves the converted data and optionally copies original files
|
|
153
|
+
to the specified output directory.
|
|
88
154
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
if os.path.exists(path):
|
|
95
|
-
existing_dirs[name] = path
|
|
96
|
-
else:
|
|
97
|
-
print(f'[WARNING] Optional directory "{name}" does not exist. Skipping.')
|
|
98
|
-
return existing_dirs
|
|
155
|
+
Example usage:
|
|
156
|
+
# Dataset with splits
|
|
157
|
+
converter = MyCustomToDMConverter(root_dir='/path/to/data', is_categorized_dataset=True)
|
|
158
|
+
converted = converter.convert() # Returns a dict with keys for `train`, `valid`, and optionally `test`
|
|
159
|
+
converter.save_to_folder('/my/target/output') # Writes files/folders to output location
|
|
99
160
|
|
|
100
|
-
|
|
101
|
-
|
|
161
|
+
# Dataset without splits
|
|
162
|
+
converter = MyCustomToDMConverter(root_dir='/path/to/data', is_categorized_dataset=False)
|
|
163
|
+
converted = converter.convert() # Returns a dict or a list, depending on the implementation
|
|
164
|
+
converter.save_to_folder('/my/target/output') # Writes files/folders to output location
|
|
165
|
+
"""
|
|
102
166
|
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
optional_splits (list): List of optional split names (e.g., ['test']).
|
|
167
|
+
def convert(self):
|
|
168
|
+
"""Convert data to DM format.
|
|
106
169
|
|
|
107
|
-
|
|
108
|
-
dict: A dictionary with split names as keys and their corresponding directories as values.
|
|
170
|
+
This method should be implemented by subclasses to perform the actual conversion.
|
|
109
171
|
"""
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
if self.is_categorized_dataset:
|
|
113
|
-
# Validate required splits
|
|
114
|
-
required_dirs = {split: os.path.join(self.root_dir, split) for split in required_splits}
|
|
115
|
-
self._validate_required_dirs(required_dirs)
|
|
116
|
-
splits.update(required_dirs)
|
|
117
|
-
|
|
118
|
-
# Validate optional splits
|
|
119
|
-
optional_dirs = {split: os.path.join(self.root_dir, split) for split in optional_splits}
|
|
120
|
-
splits.update(self._validate_optional_dirs(optional_dirs))
|
|
121
|
-
else:
|
|
122
|
-
# Validate `json` and `original_file` folders for non-split datasets
|
|
123
|
-
required_dirs = {
|
|
124
|
-
'json': os.path.join(self.root_dir, 'json'),
|
|
125
|
-
'original_file': os.path.join(self.root_dir, 'original_file'),
|
|
126
|
-
}
|
|
127
|
-
self._validate_required_dirs(required_dirs)
|
|
128
|
-
splits['root'] = self.root_dir
|
|
172
|
+
raise NotImplementedError
|
|
129
173
|
|
|
130
|
-
|
|
174
|
+
def _generate_unique_id(self):
|
|
175
|
+
"""Generate a unique 10-character UUID."""
|
|
176
|
+
return uuid.uuid4().hex[:10]
|
|
131
177
|
|
|
132
|
-
def
|
|
133
|
-
"""
|
|
178
|
+
def save_to_folder(self, output_dir: str) -> None:
|
|
179
|
+
"""Save converted DM schema data to the specified folder."""
|
|
180
|
+
self.ensure_dir(output_dir)
|
|
181
|
+
if self.converted_data is None:
|
|
182
|
+
self.converted_data = self.convert()
|
|
134
183
|
|
|
135
|
-
|
|
136
|
-
split
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
184
|
+
if self.is_categorized_dataset:
|
|
185
|
+
for split, img_dict in self.converted_data.items():
|
|
186
|
+
split_dir = os.path.join(output_dir, split)
|
|
187
|
+
json_dir = os.path.join(split_dir, 'json')
|
|
188
|
+
original_file_dir = os.path.join(split_dir, 'original_file')
|
|
189
|
+
self.ensure_dir(json_dir)
|
|
190
|
+
self.ensure_dir(original_file_dir)
|
|
191
|
+
for img_filename, (dm_json, img_src_path) in img_dict.items():
|
|
192
|
+
json_filename = os.path.splitext(img_filename)[0] + '.json'
|
|
193
|
+
with open(os.path.join(json_dir, json_filename), 'w', encoding='utf-8') as jf:
|
|
194
|
+
json.dump(dm_json, jf, indent=2, ensure_ascii=False)
|
|
195
|
+
if img_src_path and os.path.exists(img_src_path):
|
|
196
|
+
shutil.copy(img_src_path, os.path.join(original_file_dir, img_filename))
|
|
143
197
|
else:
|
|
144
|
-
|
|
145
|
-
|
|
198
|
+
json_dir = os.path.join(output_dir, 'json')
|
|
199
|
+
original_file_dir = os.path.join(output_dir, 'original_file')
|
|
200
|
+
self.ensure_dir(json_dir)
|
|
201
|
+
self.ensure_dir(original_file_dir)
|
|
202
|
+
for img_filename, (dm_json, img_src_path) in self.converted_data.items():
|
|
203
|
+
json_filename = os.path.splitext(img_filename)[0] + '.json'
|
|
204
|
+
with open(os.path.join(json_dir, json_filename), 'w', encoding='utf-8') as jf:
|
|
205
|
+
json.dump(dm_json, jf, indent=2, ensure_ascii=False)
|
|
206
|
+
if img_src_path and os.path.exists(img_src_path):
|
|
207
|
+
shutil.copy(img_src_path, os.path.join(original_file_dir, img_filename))
|
|
208
|
+
print(f'[DM] Data exported to {output_dir}')
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
from synapse_sdk.utils.converters import ToDMConverter
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class COCOToDMConverter(ToDMConverter):
|
|
8
|
+
"""Convert COCO format annotations to DM (Data Manager) format."""
|
|
9
|
+
|
|
10
|
+
def convert(self):
|
|
11
|
+
if self.is_categorized_dataset:
|
|
12
|
+
splits = self._validate_splits(['train', 'valid'], ['test'])
|
|
13
|
+
all_split_data = {}
|
|
14
|
+
for split, split_dir in splits.items():
|
|
15
|
+
annotation_path = os.path.join(split_dir, 'annotations.json')
|
|
16
|
+
if not os.path.exists(annotation_path):
|
|
17
|
+
raise FileNotFoundError(f'annotations.json not found in {split_dir}')
|
|
18
|
+
with open(annotation_path, 'r', encoding='utf-8') as f:
|
|
19
|
+
coco_data = json.load(f)
|
|
20
|
+
split_data = self._convert_coco_ann_to_dm(coco_data, split_dir)
|
|
21
|
+
all_split_data[split] = split_data
|
|
22
|
+
self.converted_data = all_split_data
|
|
23
|
+
return all_split_data
|
|
24
|
+
else:
|
|
25
|
+
annotation_path = os.path.join(self.root_dir, 'annotations.json')
|
|
26
|
+
if not os.path.exists(annotation_path):
|
|
27
|
+
raise FileNotFoundError(f'annotations.json not found in {self.root_dir}')
|
|
28
|
+
with open(annotation_path, 'r', encoding='utf-8') as f:
|
|
29
|
+
coco_data = json.load(f)
|
|
30
|
+
converted_data = self._convert_coco_ann_to_dm(coco_data, self.root_dir)
|
|
31
|
+
self.converted_data = converted_data
|
|
32
|
+
return converted_data
|
|
33
|
+
|
|
34
|
+
def _convert_coco_ann_to_dm(self, coco_data, base_dir):
|
|
35
|
+
"""Convert COCO annotations to DM format."""
|
|
36
|
+
dataset_type = coco_data.get('type', 'image') # Default to 'image' if type is not specified
|
|
37
|
+
if dataset_type == 'image':
|
|
38
|
+
return self._process_image_data(coco_data, base_dir)
|
|
39
|
+
else:
|
|
40
|
+
raise ValueError(f'Unsupported dataset type: {dataset_type}')
|
|
41
|
+
|
|
42
|
+
def _process_image_data(self, coco_data, img_base_dir):
|
|
43
|
+
"""Process COCO image data and convert to DM format."""
|
|
44
|
+
images = coco_data.get('images', [])
|
|
45
|
+
annotations = coco_data.get('annotations', [])
|
|
46
|
+
categories = coco_data.get('categories', [])
|
|
47
|
+
cat_map = {cat['id']: cat for cat in categories}
|
|
48
|
+
|
|
49
|
+
# Build image_id -> annotation list
|
|
50
|
+
ann_by_img_id = {}
|
|
51
|
+
for ann in annotations:
|
|
52
|
+
img_id = ann['image_id']
|
|
53
|
+
ann_by_img_id.setdefault(img_id, []).append(ann)
|
|
54
|
+
|
|
55
|
+
result = {}
|
|
56
|
+
for img in images:
|
|
57
|
+
img_id = img['id']
|
|
58
|
+
img_filename = img['file_name']
|
|
59
|
+
img_path = os.path.join(img_base_dir, img_filename)
|
|
60
|
+
anns = ann_by_img_id.get(img_id, [])
|
|
61
|
+
|
|
62
|
+
# DM image structure
|
|
63
|
+
dm_img = {
|
|
64
|
+
'bounding_box': [],
|
|
65
|
+
'keypoint': [],
|
|
66
|
+
'relation': [],
|
|
67
|
+
'group': [],
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
# Handle bounding_box
|
|
71
|
+
bbox_ids = []
|
|
72
|
+
for ann in anns:
|
|
73
|
+
cat = cat_map.get(ann['category_id'], {})
|
|
74
|
+
if 'bbox' in ann and ann['bbox']:
|
|
75
|
+
bbox_id = self._generate_unique_id()
|
|
76
|
+
bbox_ids.append(bbox_id)
|
|
77
|
+
dm_img['bounding_box'].append({
|
|
78
|
+
'id': bbox_id,
|
|
79
|
+
'classification': cat.get('name', str(ann['category_id'])),
|
|
80
|
+
'attrs': ann.get('attrs', []),
|
|
81
|
+
'data': list(ann['bbox']),
|
|
82
|
+
})
|
|
83
|
+
|
|
84
|
+
# Handle keypoints
|
|
85
|
+
for ann in anns:
|
|
86
|
+
cat = cat_map.get(ann['category_id'], {})
|
|
87
|
+
attrs = ann.get('attrs', [])
|
|
88
|
+
if 'keypoints' in ann and ann['keypoints']:
|
|
89
|
+
kp_names = cat.get('keypoints', [])
|
|
90
|
+
kps = ann['keypoints']
|
|
91
|
+
keypoint_ids = []
|
|
92
|
+
for idx in range(min(len(kps) // 3, len(kp_names))):
|
|
93
|
+
x, y, v = kps[idx * 3 : idx * 3 + 3]
|
|
94
|
+
kp_id = self._generate_unique_id()
|
|
95
|
+
keypoint_ids.append(kp_id)
|
|
96
|
+
dm_img['keypoint'].append({
|
|
97
|
+
'id': kp_id,
|
|
98
|
+
'classification': kp_names[idx] if idx < len(kp_names) else f'keypoint_{idx}',
|
|
99
|
+
'attrs': attrs,
|
|
100
|
+
'data': [x, y],
|
|
101
|
+
})
|
|
102
|
+
group_ids = bbox_ids + keypoint_ids
|
|
103
|
+
if group_ids:
|
|
104
|
+
dm_img['group'].append({
|
|
105
|
+
'id': self._generate_unique_id(),
|
|
106
|
+
'classification': cat.get('name', str(ann['category_id'])),
|
|
107
|
+
'attrs': attrs,
|
|
108
|
+
'data': group_ids,
|
|
109
|
+
})
|
|
110
|
+
|
|
111
|
+
dm_json = {'images': [dm_img]}
|
|
112
|
+
result[img_filename] = (dm_json, img_path)
|
|
113
|
+
return result
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from urllib.parse import urljoin, urlparse
|
|
3
|
+
|
|
4
|
+
import requests
|
|
5
|
+
|
|
6
|
+
from synapse_sdk.utils.storage.providers import BaseStorage
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class HTTPStorage(BaseStorage):
|
|
10
|
+
"""Storage provider for no-auth HTTP file servers (e.g., Django FileSystemStorage served over HTTP)."""
|
|
11
|
+
|
|
12
|
+
OPTION_CASTS = {
|
|
13
|
+
'timeout': int,
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
def __init__(self, connection_params: str | dict):
|
|
17
|
+
super().__init__(connection_params)
|
|
18
|
+
|
|
19
|
+
# Extract base URL
|
|
20
|
+
if isinstance(connection_params, dict):
|
|
21
|
+
self.base_url = self.query_params.get('base_url', '')
|
|
22
|
+
self.timeout = self.query_params.get('timeout', 30)
|
|
23
|
+
else:
|
|
24
|
+
# Parse URL like: http://example.com/media/
|
|
25
|
+
parsed = urlparse(connection_params)
|
|
26
|
+
self.base_url = f'{parsed.scheme}://{parsed.netloc}{parsed.path}'
|
|
27
|
+
self.timeout = self.query_params.get('timeout', 30)
|
|
28
|
+
|
|
29
|
+
# Ensure base_url ends with /
|
|
30
|
+
if not self.base_url.endswith('/'):
|
|
31
|
+
self.base_url += '/'
|
|
32
|
+
|
|
33
|
+
# Setup session for connection pooling
|
|
34
|
+
self.session = requests.Session()
|
|
35
|
+
|
|
36
|
+
def _get_full_url(self, path: str) -> str:
|
|
37
|
+
"""Get the full URL for a given path."""
|
|
38
|
+
# Remove leading slash from path to avoid double slashes
|
|
39
|
+
if path.startswith('/'):
|
|
40
|
+
path = path[1:]
|
|
41
|
+
return urljoin(self.base_url, path)
|
|
42
|
+
|
|
43
|
+
def upload(self, source: str, target: str) -> str:
|
|
44
|
+
"""Upload a file to the HTTP server.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
source: Local file path to upload
|
|
48
|
+
target: Target path on the HTTP server
|
|
49
|
+
|
|
50
|
+
Returns:
|
|
51
|
+
URL of the uploaded file
|
|
52
|
+
"""
|
|
53
|
+
url = self._get_full_url(target)
|
|
54
|
+
|
|
55
|
+
with open(source, 'rb') as f:
|
|
56
|
+
files = {'file': (os.path.basename(source), f)}
|
|
57
|
+
|
|
58
|
+
# Try PUT first (more RESTful), fallback to POST
|
|
59
|
+
response = self.session.put(url, files=files, timeout=self.timeout)
|
|
60
|
+
|
|
61
|
+
if response.status_code == 405: # Method not allowed
|
|
62
|
+
# Reset file pointer and try POST
|
|
63
|
+
f.seek(0)
|
|
64
|
+
response = self.session.post(url, files=files, timeout=self.timeout)
|
|
65
|
+
|
|
66
|
+
response.raise_for_status()
|
|
67
|
+
|
|
68
|
+
return url
|
|
69
|
+
|
|
70
|
+
def exists(self, target: str) -> bool:
|
|
71
|
+
"""Check if a file exists on the HTTP server.
|
|
72
|
+
|
|
73
|
+
Args:
|
|
74
|
+
target: Path to check
|
|
75
|
+
|
|
76
|
+
Returns:
|
|
77
|
+
True if file exists, False otherwise
|
|
78
|
+
"""
|
|
79
|
+
url = self._get_full_url(target)
|
|
80
|
+
|
|
81
|
+
try:
|
|
82
|
+
response = self.session.head(url, timeout=self.timeout)
|
|
83
|
+
return response.status_code == 200
|
|
84
|
+
except requests.RequestException:
|
|
85
|
+
return False
|
|
86
|
+
|
|
87
|
+
def get_url(self, target: str) -> str:
|
|
88
|
+
"""Get the URL for a file.
|
|
89
|
+
|
|
90
|
+
Args:
|
|
91
|
+
target: File path
|
|
92
|
+
|
|
93
|
+
Returns:
|
|
94
|
+
Full URL of the file
|
|
95
|
+
"""
|
|
96
|
+
return self._get_full_url(target)
|
|
97
|
+
|
|
98
|
+
def get_pathlib(self, path: str) -> 'HTTPPath':
|
|
99
|
+
"""Get a pathlib-like object for HTTP operations.
|
|
100
|
+
|
|
101
|
+
Args:
|
|
102
|
+
path: Path to wrap
|
|
103
|
+
|
|
104
|
+
Returns:
|
|
105
|
+
HTTPPath object
|
|
106
|
+
"""
|
|
107
|
+
return HTTPPath(self, path)
|
|
108
|
+
|
|
109
|
+
def get_path_file_count(self, pathlib_obj) -> int:
|
|
110
|
+
"""Get file count in a directory.
|
|
111
|
+
|
|
112
|
+
Note: This requires the HTTP server to provide directory listing functionality.
|
|
113
|
+
|
|
114
|
+
Args:
|
|
115
|
+
pathlib_obj: HTTPPath object
|
|
116
|
+
|
|
117
|
+
Returns:
|
|
118
|
+
File count
|
|
119
|
+
"""
|
|
120
|
+
# Most HTTP servers don't provide directory listing
|
|
121
|
+
# This would need custom server-side support
|
|
122
|
+
raise NotImplementedError('File counting requires server-side directory listing support')
|
|
123
|
+
|
|
124
|
+
def get_path_total_size(self, pathlib_obj) -> int:
|
|
125
|
+
"""Get total size of files in a directory.
|
|
126
|
+
|
|
127
|
+
Note: This requires the HTTP server to provide directory listing functionality.
|
|
128
|
+
|
|
129
|
+
Args:
|
|
130
|
+
pathlib_obj: HTTPPath object
|
|
131
|
+
|
|
132
|
+
Returns:
|
|
133
|
+
Total size in bytes
|
|
134
|
+
"""
|
|
135
|
+
# Most HTTP servers don't provide directory listing
|
|
136
|
+
# This would need custom server-side support
|
|
137
|
+
raise NotImplementedError('Size calculation requires server-side directory listing support')
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class HTTPPath:
|
|
141
|
+
"""A pathlib-like interface for HTTP paths."""
|
|
142
|
+
|
|
143
|
+
def __init__(self, storage: HTTPStorage, path: str):
|
|
144
|
+
self.storage = storage
|
|
145
|
+
self.path = path.strip('/')
|
|
146
|
+
|
|
147
|
+
def __str__(self):
|
|
148
|
+
return self.path
|
|
149
|
+
|
|
150
|
+
def __truediv__(self, other):
|
|
151
|
+
"""Join paths using / operator."""
|
|
152
|
+
new_path = f'{self.path}/{other}' if self.path else str(other)
|
|
153
|
+
return HTTPPath(self.storage, new_path)
|
|
154
|
+
|
|
155
|
+
def joinuri(self, *parts):
|
|
156
|
+
"""Join path parts."""
|
|
157
|
+
parts = [self.path] + [str(p) for p in parts]
|
|
158
|
+
new_path = '/'.join(p.strip('/') for p in parts if p)
|
|
159
|
+
return HTTPPath(self.storage, new_path)
|
|
160
|
+
|
|
161
|
+
@property
|
|
162
|
+
def name(self):
|
|
163
|
+
"""Get the final component of the path."""
|
|
164
|
+
return os.path.basename(self.path)
|
|
165
|
+
|
|
166
|
+
@property
|
|
167
|
+
def parent(self):
|
|
168
|
+
"""Get the parent directory."""
|
|
169
|
+
parent_path = os.path.dirname(self.path)
|
|
170
|
+
return HTTPPath(self.storage, parent_path)
|
|
171
|
+
|
|
172
|
+
def exists(self):
|
|
173
|
+
"""Check if this path exists."""
|
|
174
|
+
return self.storage.exists(self.path)
|
|
175
|
+
|
|
176
|
+
def is_file(self):
|
|
177
|
+
"""Check if this path is a file."""
|
|
178
|
+
# For HTTP, we assume it's a file if it exists
|
|
179
|
+
return self.exists()
|
|
180
|
+
|
|
181
|
+
def read_bytes(self):
|
|
182
|
+
"""Read file contents as bytes."""
|
|
183
|
+
url = self.storage.get_url(self.path)
|
|
184
|
+
response = self.storage.session.get(url, timeout=self.storage.timeout)
|
|
185
|
+
response.raise_for_status()
|
|
186
|
+
return response.content
|
|
187
|
+
|
|
188
|
+
def read_text(self, encoding='utf-8'):
|
|
189
|
+
"""Read file contents as text."""
|
|
190
|
+
return self.read_bytes().decode(encoding)
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
from synapse_sdk.utils.storage.providers.gcp import GCPStorage
|
|
2
|
+
from synapse_sdk.utils.storage.providers.http import HTTPStorage
|
|
2
3
|
from synapse_sdk.utils.storage.providers.s3 import S3Storage
|
|
3
4
|
from synapse_sdk.utils.storage.providers.sftp import SFTPStorage
|
|
4
5
|
|
|
@@ -8,4 +9,6 @@ STORAGE_PROVIDERS = {
|
|
|
8
9
|
'minio': S3Storage,
|
|
9
10
|
'gcp': GCPStorage,
|
|
10
11
|
'sftp': SFTPStorage,
|
|
12
|
+
'http': HTTPStorage,
|
|
13
|
+
'https': HTTPStorage,
|
|
11
14
|
}
|
|
@@ -30,7 +30,7 @@ synapse_sdk/clients/agent/__init__.py,sha256=FqYbtzMJdzRfuU2SA-Yxdc0JKmVP1wxH6Ol
|
|
|
30
30
|
synapse_sdk/clients/agent/core.py,sha256=x2jgORTjT7pJY67SLuc-5lMG6CD5OWpy8UgGeTf7IhA,270
|
|
31
31
|
synapse_sdk/clients/agent/ray.py,sha256=1EDl-bMN2CvKl07-qMidSWNOGpvIvzcWl7jDBCza65o,3248
|
|
32
32
|
synapse_sdk/clients/agent/service.py,sha256=s7KuPK_DB1nr2VHrigttV1WyFonaGHNrPvU8loRxHcE,478
|
|
33
|
-
synapse_sdk/clients/backend/__init__.py,sha256=
|
|
33
|
+
synapse_sdk/clients/backend/__init__.py,sha256=ZIOtyumZUw2u1k71rf5CjMCqhR1RwRTKJaZ19sCuTuE,1947
|
|
34
34
|
synapse_sdk/clients/backend/annotation.py,sha256=dX7os4zFxI3oyh8SzqB83eTW_mR07Hp2bhCHwe64AkE,1356
|
|
35
35
|
synapse_sdk/clients/backend/core.py,sha256=5XAOdo6JZ0drfk-FMPJ96SeTd9oja-VnTwzGXdvK7Bg,1027
|
|
36
36
|
synapse_sdk/clients/backend/data_collection.py,sha256=uI-_ByLh-Xez4VIIVRBO8FCNUpDcxhBcLxCVFb_aG7o,4104
|
|
@@ -108,11 +108,11 @@ synapse_sdk/devtools/web/src/views/PluginView.jsx,sha256=_-V8elSiEtsvKECeROtQopS
|
|
|
108
108
|
synapse_sdk/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
109
109
|
synapse_sdk/plugins/enums.py,sha256=ibixwqA3sCNSriG1jAtL54JQc_Zwo3MufwYUqGhVncc,523
|
|
110
110
|
synapse_sdk/plugins/exceptions.py,sha256=Qs7qODp_RRLO9y2otU2T4ryj5LFwIZODvSIXkAh91u0,691
|
|
111
|
-
synapse_sdk/plugins/models.py,sha256=
|
|
111
|
+
synapse_sdk/plugins/models.py,sha256=dte0Thx4O8KS5WKrtERbtOmyZ85MG_TFqE6FUCplkjk,4645
|
|
112
112
|
synapse_sdk/plugins/upload.py,sha256=VJOotYMayylOH0lNoAGeGHRkLdhP7jnC_A0rFQMvQpQ,3228
|
|
113
113
|
synapse_sdk/plugins/utils.py,sha256=4_K6jIl0WrsXOEhFp94faMOriSsddOhIiaXcawYYUUA,3300
|
|
114
114
|
synapse_sdk/plugins/categories/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
115
|
-
synapse_sdk/plugins/categories/base.py,sha256=
|
|
115
|
+
synapse_sdk/plugins/categories/base.py,sha256=ZBgRh3tTdSnqBrzjux8oi1_3PidHMmpmLClPXiTmU3Q,10690
|
|
116
116
|
synapse_sdk/plugins/categories/decorators.py,sha256=Gw6T-UHwpCKrSt596X-g2sZbY_Z1zbbogowClj7Pr5Q,518
|
|
117
117
|
synapse_sdk/plugins/categories/registry.py,sha256=KdQR8SUlLT-3kgYzDNWawS1uJnAhrcw2j4zFaTpilRs,636
|
|
118
118
|
synapse_sdk/plugins/categories/templates.py,sha256=FF5FerhkZMeW1YcKLY5cylC0SkWSYdJODA_Qcm4OGYQ,887
|
|
@@ -138,7 +138,7 @@ synapse_sdk/plugins/categories/neural_net/actions/test.py,sha256=JY25eg-Fo6WbgtM
|
|
|
138
138
|
synapse_sdk/plugins/categories/neural_net/actions/train.py,sha256=i406Ar0V74QwdvqI_g_DgHblB_SoGRPMsuwWcxfoeew,5429
|
|
139
139
|
synapse_sdk/plugins/categories/neural_net/actions/tune.py,sha256=C2zv3o0S-5Hjjsms8ULDGD-ad_DdNTqCPOcDqXa0v1Y,13494
|
|
140
140
|
synapse_sdk/plugins/categories/neural_net/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
141
|
-
synapse_sdk/plugins/categories/neural_net/base/inference.py,sha256=
|
|
141
|
+
synapse_sdk/plugins/categories/neural_net/base/inference.py,sha256=R5DASI6-5vzsjDOYxqeGGMBjnav5qHF4hNJT8zNUR3I,1097
|
|
142
142
|
synapse_sdk/plugins/categories/neural_net/templates/config.yaml,sha256=TMdvthf0zQYYTHf0IibKJ6InziRCWM4100C1DKkJVqU,1094
|
|
143
143
|
synapse_sdk/plugins/categories/neural_net/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
144
144
|
synapse_sdk/plugins/categories/neural_net/templates/plugin/inference.py,sha256=rXvECCewVlLSotv7UaIyyGfEv0OODlXBuk1JwBLNhh8,838
|
|
@@ -192,22 +192,24 @@ synapse_sdk/utils/http.py,sha256=yRxYfru8tMnBVeBK-7S0Ga13yOf8oRHquG5e8K_FWcI,475
|
|
|
192
192
|
synapse_sdk/utils/module_loading.py,sha256=chHpU-BZjtYaTBD_q0T7LcKWtqKvYBS4L0lPlKkoMQ8,1020
|
|
193
193
|
synapse_sdk/utils/network.py,sha256=WI8qn6KlKpHdMi45V57ofKJB8zusJrbQsxT74LwVfsY,1000
|
|
194
194
|
synapse_sdk/utils/string.py,sha256=rEwuZ9SAaZLcQ8TYiwNKr1h2u4CfnrQx7SUL8NWmChg,216
|
|
195
|
-
synapse_sdk/utils/converters/__init__.py,sha256=
|
|
195
|
+
synapse_sdk/utils/converters/__init__.py,sha256=7Lc8PeHGKKRlfhIyBbWXa258ha6qOcZCywGleFwD47E,9919
|
|
196
196
|
synapse_sdk/utils/converters/coco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
197
197
|
synapse_sdk/utils/converters/coco/from_dm.py,sha256=GH_y0zOYczbwByDusC4JiBl1M3utxpBYit7E_px7F-U,9620
|
|
198
|
+
synapse_sdk/utils/converters/coco/to_dm.py,sha256=Ve8LrcKVlzNysam3fidcgP5fdm0_UGbBgSPoj2dT_JA,4906
|
|
198
199
|
synapse_sdk/utils/pydantic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
199
200
|
synapse_sdk/utils/pydantic/config.py,sha256=1vYOcUI35GslfD1rrqhFkNXXJOXt4IDqOPSx9VWGfNE,123
|
|
200
201
|
synapse_sdk/utils/pydantic/errors.py,sha256=0v0T12eQBr1KrFiEOBu6KMaPK4aPEGEC6etPJGoR5b4,1061
|
|
201
202
|
synapse_sdk/utils/pydantic/validators.py,sha256=G47P8ObPhsePmd_QZDK8EdPnik2CbaYzr_N4Z6En8dc,193
|
|
202
203
|
synapse_sdk/utils/storage/__init__.py,sha256=HmZHqvoV-EogV2bE-Sw5XQRlrNuf3gfNL9irAJeRYsA,2195
|
|
203
|
-
synapse_sdk/utils/storage/registry.py,sha256=
|
|
204
|
+
synapse_sdk/utils/storage/registry.py,sha256=Kq3PYT7MbE_ZC4IOXBFA4GlP6DOlxMGTRybFbWBkzYU,451
|
|
204
205
|
synapse_sdk/utils/storage/providers/__init__.py,sha256=x7RGwZryT2FpVxS7fGWryRVpquHzAiIfTz-9uLgjleo,1860
|
|
205
206
|
synapse_sdk/utils/storage/providers/gcp.py,sha256=i2BQCu1Kej1If9SuNr2_lEyTcr5M_ncGITZrL0u5wEA,363
|
|
207
|
+
synapse_sdk/utils/storage/providers/http.py,sha256=2DhIulND47JOnS5ZY7MZUex7Su3peAPksGo1Wwg07L4,5828
|
|
206
208
|
synapse_sdk/utils/storage/providers/s3.py,sha256=W94rQvhGRXti3R4mYP7gmU5pcyCQpGFIBLvxxqLVdRM,2231
|
|
207
209
|
synapse_sdk/utils/storage/providers/sftp.py,sha256=_8s9hf0JXIO21gvm-JVS00FbLsbtvly4c-ETLRax68A,1426
|
|
208
|
-
synapse_sdk-1.0.
|
|
209
|
-
synapse_sdk-1.0.
|
|
210
|
-
synapse_sdk-1.0.
|
|
211
|
-
synapse_sdk-1.0.
|
|
212
|
-
synapse_sdk-1.0.
|
|
213
|
-
synapse_sdk-1.0.
|
|
210
|
+
synapse_sdk-1.0.0a66.dist-info/licenses/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
|
|
211
|
+
synapse_sdk-1.0.0a66.dist-info/METADATA,sha256=jilbqseoPooe0I1NltsyHG9dNts-61RKVCGBrS2fQEQ,1130
|
|
212
|
+
synapse_sdk-1.0.0a66.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
213
|
+
synapse_sdk-1.0.0a66.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
|
|
214
|
+
synapse_sdk-1.0.0a66.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
|
|
215
|
+
synapse_sdk-1.0.0a66.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|