sciveo 0.1.20__tar.gz → 0.1.22__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sciveo-0.1.20 → sciveo-0.1.22}/PKG-INFO +1 -1
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo/__init__.py +2 -2
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo/api/base.py +3 -3
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo/api/upload.py +5 -5
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo/cli.py +3 -3
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo/common/configuration.py +2 -2
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo/common/model.py +1 -1
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo/common/optimizers.py +4 -4
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo/common/sampling.py +9 -9
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo/content/dataset.py +1 -1
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo/content/experiment.py +11 -11
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo/content/project.py +9 -9
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo/content/runner.py +3 -3
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo/monitoring/monitor.py +6 -6
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo/monitoring/start.py +1 -1
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo/network/camera.py +2 -2
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo/network/sniffer.py +4 -4
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo/network/tools.py +6 -6
- {sciveo-0.1.20/sciveo/common → sciveo-0.1.22/sciveo}/tools/compress.py +3 -3
- {sciveo-0.1.20/sciveo/common → sciveo-0.1.22/sciveo}/tools/configuration.py +1 -1
- {sciveo-0.1.20/sciveo/common → sciveo-0.1.22/sciveo}/tools/crypto.py +1 -1
- {sciveo-0.1.20/sciveo/common → sciveo-0.1.22/sciveo}/tools/daemon.py +6 -6
- {sciveo-0.1.20/sciveo/common → sciveo-0.1.22/sciveo}/tools/hardware.py +2 -2
- {sciveo-0.1.20/sciveo/common → sciveo-0.1.22/sciveo}/tools/logger.py +15 -7
- {sciveo-0.1.20/sciveo/common → sciveo-0.1.22/sciveo}/tools/remote.py +7 -7
- {sciveo-0.1.20/sciveo/common → sciveo-0.1.22/sciveo}/tools/synchronized.py +1 -1
- {sciveo-0.1.20/sciveo/common → sciveo-0.1.22/sciveo}/tools/timers.py +2 -2
- sciveo-0.1.22/sciveo/version.py +2 -0
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo.egg-info/PKG-INFO +1 -1
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo.egg-info/SOURCES.txt +12 -13
- {sciveo-0.1.20 → sciveo-0.1.22}/test/test_compress.py +1 -1
- {sciveo-0.1.20 → sciveo-0.1.22}/test/test_crypto.py +1 -1
- {sciveo-0.1.20 → sciveo-0.1.22}/test/test_monitoring.py +1 -1
- {sciveo-0.1.20 → sciveo-0.1.22}/test/test_sampling.py +2 -2
- sciveo-0.1.20/sciveo/network/tunnel.py +0 -17
- sciveo-0.1.20/sciveo/version.py +0 -2
- {sciveo-0.1.20 → sciveo-0.1.22}/README.md +0 -0
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo/api/__init__.py +0 -0
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo/common/__init__.py +0 -0
- {sciveo-0.1.20/sciveo/common/tools → sciveo-0.1.22/sciveo/content}/__init__.py +0 -0
- {sciveo-0.1.20/sciveo/content → sciveo-0.1.22/sciveo/monitoring}/__init__.py +0 -0
- {sciveo-0.1.20/sciveo/monitoring → sciveo-0.1.22/sciveo/network}/__init__.py +0 -0
- {sciveo-0.1.20/sciveo/network → sciveo-0.1.22/sciveo/tools}/__init__.py +0 -0
- {sciveo-0.1.20/sciveo/common → sciveo-0.1.22/sciveo}/tools/formating.py +0 -0
- {sciveo-0.1.20/sciveo/common → sciveo-0.1.22/sciveo}/tools/random.py +0 -0
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo.egg-info/dependency_links.txt +0 -0
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo.egg-info/entry_points.txt +0 -0
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo.egg-info/requires.txt +0 -0
- {sciveo-0.1.20 → sciveo-0.1.22}/sciveo.egg-info/top_level.txt +0 -0
- {sciveo-0.1.20 → sciveo-0.1.22}/setup.cfg +0 -0
- {sciveo-0.1.20 → sciveo-0.1.22}/setup.py +0 -0
- {sciveo-0.1.20 → sciveo-0.1.22}/test/test_configuration.py +0 -0
- {sciveo-0.1.20 → sciveo-0.1.22}/test/test_runner.py +0 -0
|
@@ -16,8 +16,8 @@ try:
|
|
|
16
16
|
import os
|
|
17
17
|
import time
|
|
18
18
|
|
|
19
|
-
from sciveo.
|
|
20
|
-
from sciveo.
|
|
19
|
+
from sciveo.tools.logger import *
|
|
20
|
+
from sciveo.tools.daemon import TasksDaemon, __upload_content__
|
|
21
21
|
from sciveo.content.runner import ProjectRunner
|
|
22
22
|
from sciveo.content.dataset import Dataset
|
|
23
23
|
from sciveo.monitoring.start import MonitorStart
|
|
@@ -14,8 +14,8 @@ import json
|
|
|
14
14
|
from urllib import request, parse
|
|
15
15
|
from urllib.error import HTTPError
|
|
16
16
|
|
|
17
|
-
from sciveo.
|
|
18
|
-
from sciveo.
|
|
17
|
+
from sciveo.tools.logger import *
|
|
18
|
+
from sciveo.tools.configuration import GlobalConfiguration
|
|
19
19
|
|
|
20
20
|
|
|
21
21
|
class APIRemoteClient:
|
|
@@ -26,7 +26,7 @@ class APIRemoteClient:
|
|
|
26
26
|
base_url = self.config["api_base_url"]
|
|
27
27
|
self.base_url = f"{base_url}/api/v{ver}/"
|
|
28
28
|
self.headers = { "Auth-Token": self.config['secret_access_key'] }
|
|
29
|
-
debug(
|
|
29
|
+
debug(f"base url: {self.base_url}")
|
|
30
30
|
|
|
31
31
|
def POST_SCI(self, content_type, data, timeout=30):
|
|
32
32
|
return self.POST(f"sci/{content_type}/", data, timeout)
|
|
@@ -15,7 +15,7 @@ from urllib import request, parse
|
|
|
15
15
|
from urllib.error import HTTPError
|
|
16
16
|
import requests
|
|
17
17
|
|
|
18
|
-
from sciveo.
|
|
18
|
+
from sciveo.tools.logger import *
|
|
19
19
|
from sciveo.api.base import *
|
|
20
20
|
|
|
21
21
|
|
|
@@ -33,17 +33,17 @@ class APIFileUploader:
|
|
|
33
33
|
file_name = self.local_path.split("/")[-1]
|
|
34
34
|
url_presigned_post = f"aws/s3/presigned_post/?content_type={self.content_type}&file_name={file_name}"
|
|
35
35
|
presigned_post = APIFileUploader.api.GET(url_presigned_post)
|
|
36
|
-
debug(
|
|
36
|
+
debug("upload presigned_post", url_presigned_post, "=>", presigned_post)
|
|
37
37
|
if "fields" in presigned_post:
|
|
38
38
|
response = None
|
|
39
39
|
with open(self.local_path, 'rb') as fh:
|
|
40
40
|
files = { 'file': (presigned_post['fields']['key'], fh) }
|
|
41
41
|
response = requests.post(presigned_post['url'], data=presigned_post['fields'], files=files)
|
|
42
42
|
if response.status_code == 204:
|
|
43
|
-
debug(
|
|
43
|
+
debug(self.content_type, self.local_path, "uploaded")
|
|
44
44
|
url_append = f"content/append/?content_type={self.content_type}&name={file_name}&key={presigned_post['fields']['key']}&parent_id={self.parent_guid}"
|
|
45
45
|
result = APIFileUploader.api.GET(url_append)
|
|
46
|
-
debug(
|
|
46
|
+
debug("upload", url_append, result, "appended")
|
|
47
47
|
else:
|
|
48
|
-
error(
|
|
48
|
+
error("upload", self.content_type, self.local_path, "FAIL")
|
|
49
49
|
|
|
@@ -13,11 +13,11 @@
|
|
|
13
13
|
import os
|
|
14
14
|
import argparse
|
|
15
15
|
|
|
16
|
-
from sciveo.
|
|
17
|
-
from sciveo.
|
|
16
|
+
from sciveo.tools.logger import *
|
|
17
|
+
from sciveo.tools.timers import Timer
|
|
18
18
|
from sciveo.monitoring.start import MonitorStart
|
|
19
19
|
from sciveo.network.tools import NetworkTools
|
|
20
|
-
from sciveo.
|
|
20
|
+
from sciveo.tools.configuration import GlobalConfiguration
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
def main():
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
|
|
12
12
|
import json
|
|
13
13
|
|
|
14
|
-
from sciveo.
|
|
14
|
+
from sciveo.tools.logger import *
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
list_hyperparameters = list(set([
|
|
@@ -72,7 +72,7 @@ class Configuration:
|
|
|
72
72
|
setattr(self, c, None)
|
|
73
73
|
for k, v in configuration.items():
|
|
74
74
|
setattr(self, k, v)
|
|
75
|
-
debug(
|
|
75
|
+
debug("init", configuration)
|
|
76
76
|
|
|
77
77
|
def set(self, key, value):
|
|
78
78
|
self.configuration[key] = value
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
|
|
12
12
|
import numpy as np
|
|
13
13
|
|
|
14
|
-
from sciveo.
|
|
14
|
+
from sciveo.tools.logger import *
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
class BaseOptimizer:
|
|
@@ -27,7 +27,7 @@ class BaseOptimizer:
|
|
|
27
27
|
self.use_keys = set(use_keys)
|
|
28
28
|
self.remain_keys = self.keys - self.use_keys
|
|
29
29
|
self.iteration = 0
|
|
30
|
-
debug(
|
|
30
|
+
debug("init", self.use_keys, learning_rate, learning_rate_decay)
|
|
31
31
|
|
|
32
32
|
def x_to_list(self, x1, x2):
|
|
33
33
|
list_x1 = []
|
|
@@ -62,7 +62,7 @@ class BaseOptimizer:
|
|
|
62
62
|
x_new = x1 * (1 + self.learning_rate * grads)
|
|
63
63
|
|
|
64
64
|
self.on_iteration()
|
|
65
|
-
# debug(
|
|
65
|
+
# debug(self.iteration, "update", [x1.tolist(), y], "x_new", x_new.tolist(), "grads", grads.tolist(), "lr", self.learning_rate)
|
|
66
66
|
|
|
67
67
|
return delta_y, self.list_to_x(x, x_new)
|
|
68
68
|
|
|
@@ -109,6 +109,6 @@ class AdamOptimizer(BaseOptimizer):
|
|
|
109
109
|
x_new = x1 + self.learning_rate * m_hat / (np.sqrt(v_hat) + self.epsilon)
|
|
110
110
|
|
|
111
111
|
self.on_iteration()
|
|
112
|
-
# debug(
|
|
112
|
+
# debug(self.iteration, "update", [x1.tolist(), y], "x_new", x_new.tolist(), "grads", grads.tolist(), "lr", self.learning_rate)
|
|
113
113
|
|
|
114
114
|
return delta_y, self.list_to_x(x, x_new)
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
from itertools import product
|
|
13
13
|
import numpy as np
|
|
14
14
|
|
|
15
|
-
from sciveo.
|
|
15
|
+
from sciveo.tools.logger import *
|
|
16
16
|
from sciveo.common.configuration import Configuration
|
|
17
17
|
from sciveo.common.optimizers import BaseOptimizer, AdamOptimizer
|
|
18
18
|
|
|
@@ -79,7 +79,7 @@ class BaseSampler:
|
|
|
79
79
|
for k, v in self.configuration.items():
|
|
80
80
|
sample[k] = self.sample_field(v)
|
|
81
81
|
self.idx += 1
|
|
82
|
-
# debug(
|
|
82
|
+
# debug("next", sample)
|
|
83
83
|
return Configuration(sample)
|
|
84
84
|
|
|
85
85
|
def __call__(self):
|
|
@@ -149,7 +149,7 @@ class GridSampler(BaseSampler):
|
|
|
149
149
|
sample = next(self.sample_iterator)
|
|
150
150
|
self.idx += 1
|
|
151
151
|
sample = dict(zip(self.configuration_lists.keys(), sample))
|
|
152
|
-
# debug(
|
|
152
|
+
# debug("next", sample)
|
|
153
153
|
return Configuration(sample)
|
|
154
154
|
|
|
155
155
|
|
|
@@ -230,7 +230,7 @@ class AutoSampler(BaseSampler):
|
|
|
230
230
|
def __next__(self):
|
|
231
231
|
self.idx += 1
|
|
232
232
|
if len(self.project.list_experiments) < self.num_random_samples:
|
|
233
|
-
debug(
|
|
233
|
+
debug("next random sample from", type(self.random_sampler).__name__, len(self.project.list_experiments), "from", self.num_random_samples)
|
|
234
234
|
return next(self.random_sampler)
|
|
235
235
|
else:
|
|
236
236
|
# Find highest score experiment. Run with close to this current best experiment so to be able to calc gradient.
|
|
@@ -240,14 +240,14 @@ class AutoSampler(BaseSampler):
|
|
|
240
240
|
if E.data["experiment"]["eval"]["score"] > max_score:
|
|
241
241
|
max_score = E.data["experiment"]["eval"]["score"]
|
|
242
242
|
self.best_random_experiment = E
|
|
243
|
-
debug(
|
|
243
|
+
debug("next best experiment", E.name, E.data["experiment"]["config"])
|
|
244
244
|
new_params = {}
|
|
245
245
|
for k, v in E.data["experiment"]["config"].items():
|
|
246
246
|
if k in self.optimize_keys:
|
|
247
247
|
new_params[k] = v * self.next_sample_ratio
|
|
248
248
|
else:
|
|
249
249
|
new_params[k] = v
|
|
250
|
-
debug(
|
|
250
|
+
debug("next sample with ratio", self.next_sample_ratio, new_params)
|
|
251
251
|
return Configuration(new_params)
|
|
252
252
|
else:
|
|
253
253
|
# Already have best random experiment and its close experiment
|
|
@@ -267,12 +267,12 @@ class AutoSampler(BaseSampler):
|
|
|
267
267
|
|
|
268
268
|
for k, v in self.optimize_limits.items():
|
|
269
269
|
if new_params[k] < v[0] or new_params[k] > v[1]:
|
|
270
|
-
debug(
|
|
270
|
+
debug("next out of limits", k, new_params[k], v)
|
|
271
271
|
raise StopIteration
|
|
272
272
|
|
|
273
273
|
if np.linalg.norm(delta_score) > self.min_delta_score:
|
|
274
|
-
debug(
|
|
274
|
+
debug("next", type(self.optimizer).__name__, new_params, [delta_score, self.min_delta_score])
|
|
275
275
|
return Configuration(new_params)
|
|
276
276
|
else:
|
|
277
|
-
debug(
|
|
277
|
+
debug("next small improvement", new_params, "delta", delta_score)
|
|
278
278
|
raise StopIteration
|
|
@@ -15,10 +15,10 @@ import datetime
|
|
|
15
15
|
|
|
16
16
|
import numpy as np
|
|
17
17
|
|
|
18
|
-
from sciveo.
|
|
19
|
-
from sciveo.
|
|
20
|
-
from sciveo.
|
|
21
|
-
from sciveo.
|
|
18
|
+
from sciveo.tools.logger import *
|
|
19
|
+
from sciveo.tools.daemon import __upload_content__ as UPLOAD
|
|
20
|
+
from sciveo.tools.formating import format_elapsed_time
|
|
21
|
+
from sciveo.tools.hardware import HardwareInfo
|
|
22
22
|
from sciveo.common.configuration import Configuration
|
|
23
23
|
from sciveo.common.model import ModelInfo
|
|
24
24
|
from sciveo.content.dataset import Dataset
|
|
@@ -54,7 +54,7 @@ class Experiment:
|
|
|
54
54
|
# TODO: Consider better experiment naming
|
|
55
55
|
# perhaps include some eval
|
|
56
56
|
self.name = self.config.name
|
|
57
|
-
debug(
|
|
57
|
+
debug("init", self.name)
|
|
58
58
|
|
|
59
59
|
def on_guid(self, guid):
|
|
60
60
|
self.guid = guid
|
|
@@ -66,7 +66,7 @@ class Experiment:
|
|
|
66
66
|
self.data["experiment"]["end_at"] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
67
67
|
self.data["experiment"]["elapsed"] = format_elapsed_time(self.elapsed)
|
|
68
68
|
self.data["experiment"]["dataset"] = Dataset.get().info
|
|
69
|
-
debug(
|
|
69
|
+
debug("close", self.name)
|
|
70
70
|
|
|
71
71
|
def append(self, d):
|
|
72
72
|
self.data["experiment"]["log"].append(d)
|
|
@@ -152,14 +152,14 @@ class RemoteExperiment(Experiment):
|
|
|
152
152
|
else:
|
|
153
153
|
remote_data["project_name"] = self.project_name
|
|
154
154
|
result = self.api.POST_SCI("experiment", remote_data)
|
|
155
|
-
debug(
|
|
155
|
+
debug("init", self.name, "api", result)
|
|
156
156
|
if result and "error" not in result:
|
|
157
157
|
if result["name"] == self.name:
|
|
158
158
|
self.on_guid(result["guid"])
|
|
159
159
|
else:
|
|
160
|
-
error(
|
|
160
|
+
error("Project name mismatch", result, self.name)
|
|
161
161
|
else:
|
|
162
|
-
error(
|
|
162
|
+
error("init", self.name, "api", result)
|
|
163
163
|
|
|
164
164
|
def close(self):
|
|
165
165
|
super().close()
|
|
@@ -169,7 +169,7 @@ class RemoteExperiment(Experiment):
|
|
|
169
169
|
"data": self.data
|
|
170
170
|
}
|
|
171
171
|
result = self.api.POST_SCI("experiment", remote_data)
|
|
172
|
-
debug(
|
|
172
|
+
debug("close", self.name, "api", result)
|
|
173
173
|
|
|
174
174
|
def on_guid(self, guid):
|
|
175
175
|
super().on_guid(guid)
|
|
@@ -179,7 +179,7 @@ class RemoteExperiment(Experiment):
|
|
|
179
179
|
if self.guid is not None:
|
|
180
180
|
UPLOAD(content_type, local_path, self.guid)
|
|
181
181
|
else:
|
|
182
|
-
error(
|
|
182
|
+
error("upload", local_path)
|
|
183
183
|
def upload_image(self, local_path):
|
|
184
184
|
self.upload("image", local_path)
|
|
185
185
|
def upload_file(self, local_path):
|
|
@@ -14,7 +14,7 @@ import time
|
|
|
14
14
|
|
|
15
15
|
import numpy as np
|
|
16
16
|
|
|
17
|
-
from sciveo.
|
|
17
|
+
from sciveo.tools.logger import *
|
|
18
18
|
from sciveo.api.base import *
|
|
19
19
|
from sciveo.common.configuration import Configuration
|
|
20
20
|
from sciveo.content.experiment import Experiment, RemoteExperiment
|
|
@@ -40,15 +40,15 @@ class ProjectBase:
|
|
|
40
40
|
}
|
|
41
41
|
}
|
|
42
42
|
|
|
43
|
-
debug(
|
|
43
|
+
debug("init", self.project_name)
|
|
44
44
|
|
|
45
45
|
def open(self):
|
|
46
|
-
debug(
|
|
46
|
+
debug("open", self.project_name)
|
|
47
47
|
self.current_experiment = Experiment(self.project_name, self.guid, self.config)
|
|
48
48
|
return self.current_experiment
|
|
49
49
|
|
|
50
50
|
def close(self):
|
|
51
|
-
debug(
|
|
51
|
+
debug("close", self.project_name)
|
|
52
52
|
self.project_data["project"]["dataset"] = Dataset.get().info
|
|
53
53
|
self.current_experiment.close()
|
|
54
54
|
self.list_experiments.append(self.current_experiment)
|
|
@@ -76,18 +76,18 @@ class RemoteProject(ProjectBase):
|
|
|
76
76
|
if parent_id is not None:
|
|
77
77
|
remote_data["parent_id"] = parent_id
|
|
78
78
|
result = self.api.POST_SCI("project", remote_data)
|
|
79
|
-
debug(
|
|
79
|
+
debug("init api", result)
|
|
80
80
|
if result and "error" not in result:
|
|
81
81
|
if result["name"] == project_name:
|
|
82
82
|
self.guid = result["guid"]
|
|
83
83
|
self.list_content_size = result["list_content_size"]
|
|
84
84
|
else:
|
|
85
|
-
error(
|
|
85
|
+
error("Project name mismatch", result, project_name)
|
|
86
86
|
else:
|
|
87
|
-
error(
|
|
87
|
+
error("api", remote_data, result)
|
|
88
88
|
|
|
89
89
|
def open(self):
|
|
90
|
-
debug(
|
|
90
|
+
debug("open", self.project_name)
|
|
91
91
|
self.current_experiment = RemoteExperiment(self.project_name, self.guid, self.config)
|
|
92
92
|
return self.current_experiment
|
|
93
93
|
|
|
@@ -101,7 +101,7 @@ class RemoteProject(ProjectBase):
|
|
|
101
101
|
"data": self.project_data
|
|
102
102
|
}
|
|
103
103
|
result = self.api.POST_SCI("project", remote_data)
|
|
104
|
-
debug(
|
|
104
|
+
debug("close", self.project_name, "api", result)
|
|
105
105
|
|
|
106
106
|
|
|
107
107
|
class LocalProject(ProjectBase):
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
|
|
12
12
|
import os
|
|
13
13
|
|
|
14
|
-
from sciveo.
|
|
14
|
+
from sciveo.tools.logger import *
|
|
15
15
|
from sciveo.common.sampling import RandomSampler, GridSampler, AutoSampler
|
|
16
16
|
from sciveo.content.project import RemoteProject, LocalProject
|
|
17
17
|
|
|
@@ -48,7 +48,7 @@ class ProjectRunner:
|
|
|
48
48
|
else:
|
|
49
49
|
self.configuration_sampler = RandomSampler(configuration)
|
|
50
50
|
|
|
51
|
-
debug(
|
|
51
|
+
debug(f"start remote[{remote}] count[{self.count}] sampler[{sampler}]", configuration)
|
|
52
52
|
|
|
53
53
|
def get(self, a):
|
|
54
54
|
return self.kwargs.get(a, self.arguments[a])
|
|
@@ -66,5 +66,5 @@ class ProjectRunner:
|
|
|
66
66
|
|
|
67
67
|
self.project.config = configuration_sample
|
|
68
68
|
self.project.config.set_name(f"[{self.project.list_content_size + i + 1}]")
|
|
69
|
-
debug(
|
|
69
|
+
debug("run", i, self.project.config)
|
|
70
70
|
self.function()
|
|
@@ -20,10 +20,10 @@ import re
|
|
|
20
20
|
import uuid
|
|
21
21
|
import numpy as np
|
|
22
22
|
|
|
23
|
-
from sciveo.
|
|
24
|
-
from sciveo.
|
|
25
|
-
from sciveo.
|
|
26
|
-
from sciveo.
|
|
23
|
+
from sciveo.tools.logger import *
|
|
24
|
+
from sciveo.tools.daemon import DaemonBase
|
|
25
|
+
from sciveo.tools.hardware import *
|
|
26
|
+
from sciveo.tools.formating import format_memory_size
|
|
27
27
|
from sciveo.api.base import APIRemoteClient
|
|
28
28
|
|
|
29
29
|
|
|
@@ -73,7 +73,7 @@ class BaseMonitor(DaemonBase):
|
|
|
73
73
|
|
|
74
74
|
self.data["serial"] = MonitorTools.serial()
|
|
75
75
|
|
|
76
|
-
debug(
|
|
76
|
+
debug(f"init monitor with period={period}", self.data["serial"], "initial_cpu_usage", initial_cpu_usage)
|
|
77
77
|
|
|
78
78
|
def __call__(self):
|
|
79
79
|
return self.data
|
|
@@ -91,7 +91,7 @@ class BaseMonitor(DaemonBase):
|
|
|
91
91
|
|
|
92
92
|
api_result = self.api.POST_SCI("monitor", {"data": self.data})
|
|
93
93
|
|
|
94
|
-
debug(
|
|
94
|
+
debug(self(), "api_result", api_result)
|
|
95
95
|
|
|
96
96
|
def tail_logs(self):
|
|
97
97
|
for log_name, log_path in self.list_logs:
|
|
@@ -15,8 +15,8 @@ import cv2
|
|
|
15
15
|
import numpy as np
|
|
16
16
|
from scapy.all import sniff, IP, TCP, UDP
|
|
17
17
|
|
|
18
|
-
from sciveo.
|
|
19
|
-
from sciveo.
|
|
18
|
+
from sciveo.tools.logger import *
|
|
19
|
+
from sciveo.tools.timers import FPSCounter
|
|
20
20
|
from sciveo.network.tools import StreamSniffer
|
|
21
21
|
|
|
22
22
|
|
|
@@ -13,8 +13,8 @@ import socket
|
|
|
13
13
|
import threading
|
|
14
14
|
from scapy.all import sniff, IP, TCP
|
|
15
15
|
|
|
16
|
-
from sciveo.
|
|
17
|
-
from sciveo.
|
|
16
|
+
from sciveo.tools.logger import *
|
|
17
|
+
from sciveo.tools.timers import Timer
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
class StreamSniffer:
|
|
@@ -30,12 +30,12 @@ class StreamSniffer:
|
|
|
30
30
|
self.sniff_thread.start()
|
|
31
31
|
|
|
32
32
|
def stop(self):
|
|
33
|
-
debug(
|
|
33
|
+
debug("stopping...")
|
|
34
34
|
self.running = False
|
|
35
35
|
self.sniff_thread.join()
|
|
36
36
|
|
|
37
37
|
def sniff_packets(self):
|
|
38
|
-
debug(
|
|
38
|
+
debug("start sniffing on", self.iface)
|
|
39
39
|
sniff(iface=self.iface, prn=self.on_packet, stop_filter=self.should_stop)
|
|
40
40
|
|
|
41
41
|
def on_packet(self, packet):
|
|
@@ -13,8 +13,8 @@ import socket
|
|
|
13
13
|
import ipaddress
|
|
14
14
|
import threading
|
|
15
15
|
|
|
16
|
-
from sciveo.
|
|
17
|
-
from sciveo.
|
|
16
|
+
from sciveo.tools.logger import *
|
|
17
|
+
from sciveo.tools.timers import Timer
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
class NetworkTools:
|
|
@@ -48,7 +48,7 @@ class NetworkTools:
|
|
|
48
48
|
if ip.startswith(net_class):
|
|
49
49
|
list_local_ips.append(ip)
|
|
50
50
|
except Exception as e:
|
|
51
|
-
warning(
|
|
51
|
+
warning("netifaces not installed")
|
|
52
52
|
return list_local_ips
|
|
53
53
|
|
|
54
54
|
def generate_ip_list(self, base_ip):
|
|
@@ -62,7 +62,7 @@ class NetworkTools:
|
|
|
62
62
|
|
|
63
63
|
if network is None:
|
|
64
64
|
list_local_ips = self.get_local_nets()
|
|
65
|
-
# debug(
|
|
65
|
+
# debug("scan_port", "list_local_ips", list_local_ips)
|
|
66
66
|
for local_ip in list_local_ips:
|
|
67
67
|
list_ip = self.generate_ip_list(local_ip)
|
|
68
68
|
self.scan_port_hosts(list_ip, port)
|
|
@@ -76,7 +76,7 @@ class NetworkTools:
|
|
|
76
76
|
if self.arguments["localhost"]:
|
|
77
77
|
self.scan_port_hosts(["127.0.0.1"], port)
|
|
78
78
|
self.data["scan"][port].sort(key=lambda ip: int(ip.split('.')[-1]))
|
|
79
|
-
debug(
|
|
79
|
+
debug(f"scan_port [{port}] elapsed time {t.stop():.1f}s", self.data["scan"][port])
|
|
80
80
|
return self.data["scan"][port]
|
|
81
81
|
|
|
82
82
|
def scan_port_hosts(self, list_ip, port=22):
|
|
@@ -97,6 +97,6 @@ class NetworkTools:
|
|
|
97
97
|
if result == 0:
|
|
98
98
|
with self.data_lock:
|
|
99
99
|
self.data["scan"][port].append(ip)
|
|
100
|
-
# debug(
|
|
100
|
+
# debug("scan_ports", ip, port, result)
|
|
101
101
|
except socket.error:
|
|
102
102
|
pass
|
|
@@ -2,7 +2,7 @@ import json
|
|
|
2
2
|
import gzip
|
|
3
3
|
import base64
|
|
4
4
|
|
|
5
|
-
from sciveo.
|
|
5
|
+
from sciveo.tools.logger import *
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
class CompressJsonData:
|
|
@@ -13,7 +13,7 @@ class CompressJsonData:
|
|
|
13
13
|
json_data = json.dumps(data).encode('utf-8')
|
|
14
14
|
compressed_data = gzip.compress(json_data)
|
|
15
15
|
encoded_data = base64.b64encode(compressed_data).decode('utf-8')
|
|
16
|
-
debug(
|
|
16
|
+
debug("compress", len(json_data), "->", len(encoded_data))
|
|
17
17
|
return encoded_data
|
|
18
18
|
|
|
19
19
|
def decompress(self, encoded_data):
|
|
@@ -21,6 +21,6 @@ class CompressJsonData:
|
|
|
21
21
|
decompressed_data = gzip.decompress(compressed_data)
|
|
22
22
|
json_data = decompressed_data.decode('utf-8')
|
|
23
23
|
data = json.loads(json_data)
|
|
24
|
-
debug(
|
|
24
|
+
debug("decompress", len(encoded_data), "->", len(json_data))
|
|
25
25
|
return data
|
|
26
26
|
|
|
@@ -13,8 +13,8 @@ import threading
|
|
|
13
13
|
import signal
|
|
14
14
|
import time
|
|
15
15
|
|
|
16
|
-
from sciveo.
|
|
17
|
-
from sciveo.
|
|
16
|
+
from sciveo.tools.logger import *
|
|
17
|
+
from sciveo.tools.synchronized import ListQueue
|
|
18
18
|
from sciveo.api.upload import APIFileUploader
|
|
19
19
|
|
|
20
20
|
|
|
@@ -26,12 +26,12 @@ class SignalHandler:
|
|
|
26
26
|
signal.signal(signal.SIGUSR1, self.handler_usr1)
|
|
27
27
|
|
|
28
28
|
def handler_int(self, signum, frame):
|
|
29
|
-
info(
|
|
29
|
+
info("INT signal received", signum, frame)
|
|
30
30
|
for daemon in self.daemons:
|
|
31
31
|
daemon.stop()
|
|
32
32
|
|
|
33
33
|
def handler_usr1(self, signum, frame):
|
|
34
|
-
info(
|
|
34
|
+
info("USR1 signal received", signum, frame)
|
|
35
35
|
|
|
36
36
|
|
|
37
37
|
class DaemonBase:
|
|
@@ -73,14 +73,14 @@ class DaemonBase:
|
|
|
73
73
|
try:
|
|
74
74
|
self.loop()
|
|
75
75
|
except Exception as e:
|
|
76
|
-
error(
|
|
76
|
+
error(e)
|
|
77
77
|
time.sleep(self.period)
|
|
78
78
|
|
|
79
79
|
def safe_run(self):
|
|
80
80
|
try:
|
|
81
81
|
self.run()
|
|
82
82
|
except Exception as e:
|
|
83
|
-
error(
|
|
83
|
+
error(e)
|
|
84
84
|
|
|
85
85
|
|
|
86
86
|
class TasksDaemon(DaemonBase):
|
|
@@ -15,8 +15,8 @@ import uuid
|
|
|
15
15
|
import random
|
|
16
16
|
import string
|
|
17
17
|
|
|
18
|
-
from sciveo.
|
|
19
|
-
from sciveo.
|
|
18
|
+
from sciveo.tools.logger import *
|
|
19
|
+
from sciveo.tools.formating import format_memory_size
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
def new_guid_uuid():
|
|
@@ -11,9 +11,10 @@
|
|
|
11
11
|
|
|
12
12
|
import os
|
|
13
13
|
import logging
|
|
14
|
+
import inspect
|
|
14
15
|
from threading import Lock
|
|
15
16
|
|
|
16
|
-
from sciveo.
|
|
17
|
+
from sciveo.tools.configuration import GlobalConfiguration
|
|
17
18
|
|
|
18
19
|
|
|
19
20
|
SCIVEO_LOGGER_NAME = "sciveo-log"
|
|
@@ -35,15 +36,22 @@ def _sciveo_get_logger(name):
|
|
|
35
36
|
logger.propagate = False
|
|
36
37
|
return logger
|
|
37
38
|
|
|
39
|
+
def _sciveo_append_classname(*args):
|
|
40
|
+
frame = inspect.currentframe().f_back.f_back
|
|
41
|
+
class_name = frame.f_locals.get('self', None).__class__.__name__ if 'self' in frame.f_locals else None
|
|
42
|
+
if class_name is not None:
|
|
43
|
+
args = (class_name,) + args
|
|
44
|
+
return args
|
|
45
|
+
|
|
38
46
|
def debug(*args):
|
|
39
|
-
_sciveo_get_logger(SCIVEO_LOGGER_NAME).debug(args)
|
|
47
|
+
_sciveo_get_logger(SCIVEO_LOGGER_NAME).debug(_sciveo_append_classname(*args))
|
|
40
48
|
def info(*args):
|
|
41
|
-
_sciveo_get_logger(SCIVEO_LOGGER_NAME).info(args)
|
|
49
|
+
_sciveo_get_logger(SCIVEO_LOGGER_NAME).info(_sciveo_append_classname(*args))
|
|
42
50
|
def warning(*args):
|
|
43
|
-
_sciveo_get_logger(SCIVEO_LOGGER_NAME).warning(args)
|
|
51
|
+
_sciveo_get_logger(SCIVEO_LOGGER_NAME).warning(_sciveo_append_classname(*args))
|
|
44
52
|
def error(*args):
|
|
45
|
-
_sciveo_get_logger(SCIVEO_LOGGER_NAME).error(args)
|
|
53
|
+
_sciveo_get_logger(SCIVEO_LOGGER_NAME).error(_sciveo_append_classname(*args))
|
|
46
54
|
def critical(*args):
|
|
47
|
-
_sciveo_get_logger(SCIVEO_LOGGER_NAME).critical(args)
|
|
55
|
+
_sciveo_get_logger(SCIVEO_LOGGER_NAME).critical(_sciveo_append_classname(*args))
|
|
48
56
|
def exception(e, *args):
|
|
49
|
-
_sciveo_get_logger(SCIVEO_LOGGER_NAME).exception(args)
|
|
57
|
+
_sciveo_get_logger(SCIVEO_LOGGER_NAME).exception(_sciveo_append_classname(*args))
|
|
@@ -15,14 +15,14 @@ import cv2
|
|
|
15
15
|
import base64
|
|
16
16
|
import requests
|
|
17
17
|
|
|
18
|
-
from sciveo.
|
|
19
|
-
from sciveo.
|
|
20
|
-
from sciveo.
|
|
18
|
+
from sciveo.tools.logger import *
|
|
19
|
+
from sciveo.tools.configuration import GlobalConfiguration
|
|
20
|
+
from sciveo.tools.compress import CompressJsonData
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
class PredictorRemoteClient:
|
|
24
24
|
def __init__(self, url="http://localhost:8901", verify=True, auth_token=None, api_prefix=None):
|
|
25
|
-
debug(
|
|
25
|
+
debug("init url", url, verify)
|
|
26
26
|
config = GlobalConfiguration.get()
|
|
27
27
|
if auth_token is None:
|
|
28
28
|
auth_token = config["api_auth_token"]
|
|
@@ -36,7 +36,7 @@ class PredictorRemoteClient:
|
|
|
36
36
|
|
|
37
37
|
def predict(self, params):
|
|
38
38
|
try:
|
|
39
|
-
debug(
|
|
39
|
+
debug("predict", params)
|
|
40
40
|
headers = {"Authorization": f"Bearer {self.auth_token}"}
|
|
41
41
|
response = requests.post(self.url, json=params, headers=headers, verify=self.verify)
|
|
42
42
|
|
|
@@ -48,10 +48,10 @@ class PredictorRemoteClient:
|
|
|
48
48
|
predicted = CompressJsonData().decompress(predicted)
|
|
49
49
|
data[params["predictor"]] = predicted
|
|
50
50
|
else:
|
|
51
|
-
error(
|
|
51
|
+
error(f"Request [{self.url}] failed with status code {response.status_code}")
|
|
52
52
|
data = {"error": response.status_code}
|
|
53
53
|
except Exception as e:
|
|
54
|
-
exception(
|
|
54
|
+
exception("predict", e)
|
|
55
55
|
data = {"error": str(e)}
|
|
56
56
|
return data
|
|
57
57
|
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
|
|
12
12
|
import time
|
|
13
13
|
|
|
14
|
-
from sciveo.
|
|
14
|
+
from sciveo.tools.logger import *
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
class FPSCounter:
|
|
@@ -25,7 +25,7 @@ class FPSCounter:
|
|
|
25
25
|
self.n += 1
|
|
26
26
|
t2 = time.time()
|
|
27
27
|
if t2 - self.t1 > self.period:
|
|
28
|
-
debug(
|
|
28
|
+
debug(self.tag, "FPS", self.n / (t2 - self.t1))
|
|
29
29
|
self.n = 0
|
|
30
30
|
self.t1 = time.time()
|
|
31
31
|
|
|
@@ -17,18 +17,6 @@ sciveo/common/configuration.py
|
|
|
17
17
|
sciveo/common/model.py
|
|
18
18
|
sciveo/common/optimizers.py
|
|
19
19
|
sciveo/common/sampling.py
|
|
20
|
-
sciveo/common/tools/__init__.py
|
|
21
|
-
sciveo/common/tools/compress.py
|
|
22
|
-
sciveo/common/tools/configuration.py
|
|
23
|
-
sciveo/common/tools/crypto.py
|
|
24
|
-
sciveo/common/tools/daemon.py
|
|
25
|
-
sciveo/common/tools/formating.py
|
|
26
|
-
sciveo/common/tools/hardware.py
|
|
27
|
-
sciveo/common/tools/logger.py
|
|
28
|
-
sciveo/common/tools/random.py
|
|
29
|
-
sciveo/common/tools/remote.py
|
|
30
|
-
sciveo/common/tools/synchronized.py
|
|
31
|
-
sciveo/common/tools/timers.py
|
|
32
20
|
sciveo/content/__init__.py
|
|
33
21
|
sciveo/content/dataset.py
|
|
34
22
|
sciveo/content/experiment.py
|
|
@@ -41,7 +29,18 @@ sciveo/network/__init__.py
|
|
|
41
29
|
sciveo/network/camera.py
|
|
42
30
|
sciveo/network/sniffer.py
|
|
43
31
|
sciveo/network/tools.py
|
|
44
|
-
sciveo/
|
|
32
|
+
sciveo/tools/__init__.py
|
|
33
|
+
sciveo/tools/compress.py
|
|
34
|
+
sciveo/tools/configuration.py
|
|
35
|
+
sciveo/tools/crypto.py
|
|
36
|
+
sciveo/tools/daemon.py
|
|
37
|
+
sciveo/tools/formating.py
|
|
38
|
+
sciveo/tools/hardware.py
|
|
39
|
+
sciveo/tools/logger.py
|
|
40
|
+
sciveo/tools/random.py
|
|
41
|
+
sciveo/tools/remote.py
|
|
42
|
+
sciveo/tools/synchronized.py
|
|
43
|
+
sciveo/tools/timers.py
|
|
45
44
|
test/test_compress.py
|
|
46
45
|
test/test_configuration.py
|
|
47
46
|
test/test_crypto.py
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
import unittest
|
|
13
13
|
import numpy as np
|
|
14
14
|
|
|
15
|
-
from sciveo.
|
|
15
|
+
from sciveo.tools.logger import *
|
|
16
16
|
from sciveo.common.sampling import RandomSampler, GridSampler, AutoSampler
|
|
17
17
|
from sciveo.content.project import RemoteProject, LocalProject
|
|
18
18
|
from sciveo.common.optimizers import AdamOptimizer, BaseOptimizer
|
|
@@ -247,7 +247,7 @@ class TestSamplerAuto(unittest.TestCase):
|
|
|
247
247
|
E.score(i * 0.0001)
|
|
248
248
|
E.name = f"E[{i}]"
|
|
249
249
|
project.close()
|
|
250
|
-
# debug(
|
|
250
|
+
# debug(i, configuration_sample.configuration)
|
|
251
251
|
if i > 5:
|
|
252
252
|
break
|
|
253
253
|
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# Pavlin Georgiev, Softel Labs
|
|
3
|
-
#
|
|
4
|
-
# This is a proprietary file and may not be copied,
|
|
5
|
-
# distributed, or modified without express permission
|
|
6
|
-
# from the owner. For licensing inquiries, please
|
|
7
|
-
# contact pavlin@softel.bg.
|
|
8
|
-
#
|
|
9
|
-
# 2024
|
|
10
|
-
#
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
import os
|
|
14
|
-
import subprocess
|
|
15
|
-
|
|
16
|
-
from sciveo.common.tools.logger import *
|
|
17
|
-
|
sciveo-0.1.20/sciveo/version.py
DELETED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|