sciveo 0.1.7__tar.gz → 0.1.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sciveo-0.1.7 → sciveo-0.1.9}/PKG-INFO +4 -2
- {sciveo-0.1.7 → sciveo-0.1.9}/README.md +3 -1
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/api/base.py +6 -3
- sciveo-0.1.9/sciveo/cli.py +39 -0
- sciveo-0.1.9/sciveo/common/tools/configuration.py +65 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/common/tools/logger.py +4 -1
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/monitoring/monitor.py +1 -1
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/monitoring/network.py +29 -62
- sciveo-0.1.9/sciveo/version.py +2 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo.egg-info/PKG-INFO +4 -2
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo.egg-info/SOURCES.txt +3 -0
- sciveo-0.1.9/sciveo.egg-info/entry_points.txt +2 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/setup.py +6 -0
- sciveo-0.1.7/sciveo/version.py +0 -2
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/__init__.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/api/__init__.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/api/upload.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/common/__init__.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/common/configuration.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/common/model.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/common/optimizers.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/common/sampling.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/common/tools/__init__.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/common/tools/daemon.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/common/tools/formating.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/common/tools/hardware.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/common/tools/synchronized.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/common/tools/timers.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/content/__init__.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/content/dataset.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/content/experiment.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/content/project.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/content/runner.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/monitoring/__init__.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo/monitoring/start.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo.egg-info/dependency_links.txt +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo.egg-info/requires.txt +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/sciveo.egg-info/top_level.txt +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/setup.cfg +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/test/test_configuration.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/test/test_monitoring.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/test/test_runner.py +0 -0
- {sciveo-0.1.7 → sciveo-0.1.9}/test/test_sampling.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: sciveo
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.9
|
|
4
4
|
Description-Content-Type: text/markdown
|
|
5
5
|
Provides-Extra: mon
|
|
6
6
|
Provides-Extra: net
|
|
@@ -49,10 +49,12 @@ When have sciveo account:
|
|
|
49
49
|
```shell
|
|
50
50
|
export SCIVEO_SECRET_ACCESS_KEY='my_sciveo_user_auth_token'
|
|
51
51
|
```
|
|
52
|
+
or create a file like ~/.sciveo/some_file_name where put:
|
|
53
|
+
secret_access_key=my_sciveo_user_auth_token
|
|
52
54
|
|
|
53
55
|
When using sciveo Monitoring just run, using suitable python environment
|
|
54
56
|
```shell
|
|
55
|
-
|
|
57
|
+
sciveo monitor --period 120
|
|
56
58
|
```
|
|
57
59
|
|
|
58
60
|
Monitoring started along with other python code.
|
|
@@ -41,10 +41,12 @@ When have sciveo account:
|
|
|
41
41
|
```shell
|
|
42
42
|
export SCIVEO_SECRET_ACCESS_KEY='my_sciveo_user_auth_token'
|
|
43
43
|
```
|
|
44
|
+
or create a file like ~/.sciveo/some_file_name where put:
|
|
45
|
+
secret_access_key=my_sciveo_user_auth_token
|
|
44
46
|
|
|
45
47
|
When using sciveo Monitoring just run, using suitable python environment
|
|
46
48
|
```shell
|
|
47
|
-
|
|
49
|
+
sciveo monitor --period 120
|
|
48
50
|
```
|
|
49
51
|
|
|
50
52
|
Monitoring started along with other python code.
|
|
@@ -15,14 +15,17 @@ from urllib import request, parse
|
|
|
15
15
|
from urllib.error import HTTPError
|
|
16
16
|
|
|
17
17
|
from sciveo.common.tools.logger import *
|
|
18
|
+
from sciveo.common.tools.configuration import GlobalConfiguration
|
|
18
19
|
|
|
19
20
|
|
|
20
21
|
class APIRemoteClient:
|
|
21
22
|
def __init__(self, base_url=None, ver=1):
|
|
23
|
+
self.config = GlobalConfiguration.get()
|
|
24
|
+
self.auth_token = None
|
|
22
25
|
if base_url is None:
|
|
23
|
-
base_url =
|
|
26
|
+
base_url = self.config["api_base_url"]
|
|
24
27
|
self.base_url = f"{base_url}/api/v{ver}/"
|
|
25
|
-
self.headers = { "Auth-Token":
|
|
28
|
+
self.headers = { "Auth-Token": self.config['secret_access_key'] }
|
|
26
29
|
debug(type(self).__name__, f"base url: {self.base_url}")
|
|
27
30
|
|
|
28
31
|
def POST_SCI(self, content_type, data, timeout=30):
|
|
@@ -44,7 +47,7 @@ class APIRemoteClient:
|
|
|
44
47
|
return result
|
|
45
48
|
|
|
46
49
|
def GET(self, url, timeout=30):
|
|
47
|
-
url = f"{self.base_url}{url}&auth_token={
|
|
50
|
+
url = f"{self.base_url}{url}&auth_token={self.config['secret_access_key']}"
|
|
48
51
|
result = False
|
|
49
52
|
try:
|
|
50
53
|
# debug("GET", url)
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
#
|
|
3
|
+
# Pavlin Georgiev, Softel Labs
|
|
4
|
+
#
|
|
5
|
+
# This is a proprietary file and may not be copied,
|
|
6
|
+
# distributed, or modified without express permission
|
|
7
|
+
# from the owner. For licensing inquiries, please
|
|
8
|
+
# contact pavlin@softel.bg.
|
|
9
|
+
#
|
|
10
|
+
# 2024
|
|
11
|
+
#
|
|
12
|
+
|
|
13
|
+
import os
|
|
14
|
+
import argparse
|
|
15
|
+
|
|
16
|
+
from sciveo.common.tools.logger import *
|
|
17
|
+
from sciveo.monitoring.start import MonitorStart
|
|
18
|
+
from sciveo.common.tools.configuration import GlobalConfiguration
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def main():
|
|
22
|
+
config = GlobalConfiguration.get()
|
|
23
|
+
|
|
24
|
+
parser = argparse.ArgumentParser(description='sciveo CLI')
|
|
25
|
+
parser.add_argument('command', choices=['monitor', 'net'], help='Command to execute')
|
|
26
|
+
parser.add_argument('--period', type=int, default=120, help='Period in seconds')
|
|
27
|
+
parser.add_argument('--block', type=bool, default=True, help='Block flag')
|
|
28
|
+
parser.add_argument('--auth', type=str, default=config['secret_access_key'], help='Auth secret access key')
|
|
29
|
+
args = parser.parse_args()
|
|
30
|
+
|
|
31
|
+
if args.command == 'monitor':
|
|
32
|
+
MonitorStart(period=args.period, block=args.block)()
|
|
33
|
+
elif args.command == 'net':
|
|
34
|
+
warning(args.command, "not implemented")
|
|
35
|
+
else:
|
|
36
|
+
warning(args.command, "not implemented")
|
|
37
|
+
|
|
38
|
+
if __name__ == '__main__':
|
|
39
|
+
main()
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Pavlin Georgiev, Softel Labs
|
|
3
|
+
#
|
|
4
|
+
# This is a proprietary file and may not be copied,
|
|
5
|
+
# distributed, or modified without express permission
|
|
6
|
+
# from the owner. For licensing inquiries, please
|
|
7
|
+
# contact pavlin@softel.bg.
|
|
8
|
+
#
|
|
9
|
+
# 2024
|
|
10
|
+
#
|
|
11
|
+
|
|
12
|
+
import os
|
|
13
|
+
import threading
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class GlobalConfiguration:
|
|
17
|
+
config = None
|
|
18
|
+
lock_config = threading.Lock()
|
|
19
|
+
|
|
20
|
+
def __init__(self) -> None:
|
|
21
|
+
home = os.path.expanduser('~')
|
|
22
|
+
self.base_path = os.path.join(home, '.sciveo')
|
|
23
|
+
self.data = {}
|
|
24
|
+
|
|
25
|
+
self.default = {
|
|
26
|
+
"api_base_url": "https://sciveo.com",
|
|
27
|
+
"log_min_level": "DEBUG"
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
try:
|
|
31
|
+
self.read_local_files()
|
|
32
|
+
self.read_environment()
|
|
33
|
+
except Exception as e:
|
|
34
|
+
error(type(self).__name__, "Exception", e)
|
|
35
|
+
|
|
36
|
+
@staticmethod
|
|
37
|
+
def get():
|
|
38
|
+
with GlobalConfiguration.lock_config:
|
|
39
|
+
if GlobalConfiguration.config is None:
|
|
40
|
+
GlobalConfiguration.config = GlobalConfiguration()
|
|
41
|
+
return GlobalConfiguration.config
|
|
42
|
+
|
|
43
|
+
def __getitem__(self, key):
|
|
44
|
+
key = key.lower()
|
|
45
|
+
return self.data.get(key, self.default.get(key, ""))
|
|
46
|
+
|
|
47
|
+
def read_environment(self):
|
|
48
|
+
for k, v in os.environ.items():
|
|
49
|
+
k = k.lower()
|
|
50
|
+
if k.startswith("sciveo_"):
|
|
51
|
+
k = k.replace("sciveo_", "")
|
|
52
|
+
self.data[k] = v
|
|
53
|
+
|
|
54
|
+
def read_local_files(self):
|
|
55
|
+
if os.path.exists(self.base_path):
|
|
56
|
+
for path, _, files in os.walk(self.base_path):
|
|
57
|
+
for file_name in files:
|
|
58
|
+
with open(os.path.join(path, file_name), 'r') as fp:
|
|
59
|
+
lines = fp.readlines()
|
|
60
|
+
for line in lines:
|
|
61
|
+
parts = line.strip().split('=')
|
|
62
|
+
if len(parts) == 2:
|
|
63
|
+
key = parts[0].strip().lower().replace("sciveo_", "")
|
|
64
|
+
value = parts[1].strip()
|
|
65
|
+
self.data[key] = value
|
|
@@ -13,8 +13,11 @@ import os
|
|
|
13
13
|
import logging
|
|
14
14
|
import threading
|
|
15
15
|
|
|
16
|
+
from sciveo.common.tools.configuration import GlobalConfiguration
|
|
16
17
|
|
|
17
|
-
|
|
18
|
+
|
|
19
|
+
config = GlobalConfiguration.get()
|
|
20
|
+
log_min_level = config["LOG_MIN_LEVEL"]
|
|
18
21
|
|
|
19
22
|
def get_logger(name):
|
|
20
23
|
logger = logging.getLogger(name)
|
|
@@ -110,7 +110,7 @@ class BaseMonitor(DaemonBase):
|
|
|
110
110
|
temperatures = psutil.sensors_temperatures()
|
|
111
111
|
for k, v in temperatures.items():
|
|
112
112
|
self.data["TEMP"][k] = [t.current for t in v]
|
|
113
|
-
except Exception:
|
|
113
|
+
except Exception as e:
|
|
114
114
|
pass
|
|
115
115
|
|
|
116
116
|
def get_memory(self):
|
|
@@ -10,41 +10,12 @@
|
|
|
10
10
|
#
|
|
11
11
|
|
|
12
12
|
import socket
|
|
13
|
-
import
|
|
13
|
+
import threading
|
|
14
14
|
|
|
15
15
|
from sciveo.common.tools.logger import *
|
|
16
16
|
from sciveo.common.tools.timers import Timer
|
|
17
17
|
|
|
18
18
|
|
|
19
|
-
class NetworkScanner:
|
|
20
|
-
def __init__(self, timeout=1.0):
|
|
21
|
-
self.timeout = timeout
|
|
22
|
-
|
|
23
|
-
async def scan_port(self, ip, port):
|
|
24
|
-
try:
|
|
25
|
-
reader, writer = await asyncio.wait_for(asyncio.open_connection(ip, port), timeout=self.timeout)
|
|
26
|
-
writer.close()
|
|
27
|
-
await writer.wait_closed()
|
|
28
|
-
return (ip, port)
|
|
29
|
-
except Exception as e:
|
|
30
|
-
# print("Exception", e)
|
|
31
|
-
return None
|
|
32
|
-
|
|
33
|
-
async def scan_all_ports(self, ip, ports):
|
|
34
|
-
tasks = [self.scan_port(ip, port) for port in ports]
|
|
35
|
-
return await asyncio.gather(*tasks)
|
|
36
|
-
|
|
37
|
-
async def scan_async(self, ips, ports):
|
|
38
|
-
results = []
|
|
39
|
-
for ip in ips:
|
|
40
|
-
results.extend(await self.scan_all_ports(ip, ports))
|
|
41
|
-
results = [x for x in results if x is not None]
|
|
42
|
-
return results
|
|
43
|
-
|
|
44
|
-
def scan(self, ips, ports):
|
|
45
|
-
return asyncio.run(self.scan_async(ips, ports))
|
|
46
|
-
|
|
47
|
-
|
|
48
19
|
class NetworkTools:
|
|
49
20
|
def __init__(self, **kwargs):
|
|
50
21
|
self.default_arguments = {
|
|
@@ -60,6 +31,9 @@ class NetworkTools:
|
|
|
60
31
|
for i in range(16, 32):
|
|
61
32
|
self.net_classes.append(f"172.{i}.")
|
|
62
33
|
|
|
34
|
+
self.data = {"scan": {}}
|
|
35
|
+
self.data_lock = threading.Lock()
|
|
36
|
+
|
|
63
37
|
def get_local_nets(self):
|
|
64
38
|
list_local_ips = []
|
|
65
39
|
try:
|
|
@@ -83,48 +57,41 @@ class NetworkTools:
|
|
|
83
57
|
|
|
84
58
|
def scan_port(self, port=22):
|
|
85
59
|
t = Timer()
|
|
86
|
-
result = []
|
|
87
60
|
list_local_ips = self.get_local_nets()
|
|
88
61
|
debug(type(self).__name__, "scan_port", "list_local_ips", list_local_ips)
|
|
89
62
|
for local_ip in list_local_ips:
|
|
90
63
|
list_ip = self.generate_ip_list(local_ip)
|
|
91
|
-
|
|
64
|
+
self.scan_port_hosts(list_ip, port)
|
|
92
65
|
if self.arguments["localhost"]:
|
|
93
|
-
|
|
94
|
-
debug(type(self).__name__, "scan_port elapsed time", t.stop(), "
|
|
95
|
-
return
|
|
66
|
+
self.scan_port_hosts(["127.0.0.1"], port)
|
|
67
|
+
debug(type(self).__name__, f"scan_port [{port}] elapsed time", t.stop(), self.data["scan"][port])
|
|
68
|
+
return self.data["scan"][port]
|
|
96
69
|
|
|
97
70
|
def scan_port_hosts(self, list_ip, port=22):
|
|
98
|
-
list_hosts = []
|
|
99
71
|
timeout = self.arguments["timeout"]
|
|
72
|
+
list_threads = []
|
|
100
73
|
for ip in list_ip:
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
74
|
+
t = threading.Thread(target=self.scan_host_port, args=(ip, port, timeout))
|
|
75
|
+
t.start()
|
|
76
|
+
list_threads.append(t)
|
|
77
|
+
for t in list_threads:
|
|
78
|
+
t.join()
|
|
79
|
+
|
|
80
|
+
def scan_host_port(self, ip, port, timeout):
|
|
81
|
+
try:
|
|
82
|
+
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
|
83
|
+
sock.settimeout(timeout)
|
|
84
|
+
result = sock.connect_ex((ip, port))
|
|
85
|
+
if result == 0:
|
|
86
|
+
with self.data_lock:
|
|
87
|
+
self.data["scan"].setdefault(port, [])
|
|
88
|
+
self.data["scan"][port].append(ip)
|
|
89
|
+
# debug(type(self).__name__, "scan_ports", ip, port, result)
|
|
90
|
+
except socket.error:
|
|
91
|
+
pass
|
|
111
92
|
|
|
112
93
|
|
|
113
94
|
if __name__ == "__main__":
|
|
114
|
-
t1 = Timer()
|
|
115
95
|
net = NetworkTools(timeout=0.5, localhost=True)
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
print(result, "elapsed", t1)
|
|
119
|
-
|
|
120
|
-
t2 = Timer()
|
|
121
|
-
list_local_ips = net.get_local_nets()
|
|
122
|
-
list_scan_ips = []
|
|
123
|
-
for local_ip in list_local_ips:
|
|
124
|
-
list_scan_ips += net.generate_ip_list(local_ip)
|
|
125
|
-
# print(list_scan_ips)
|
|
126
|
-
ns = NetworkScanner(timeout=0.5)
|
|
127
|
-
result = ns.scan(list_scan_ips, [22])
|
|
128
|
-
t2 = t2.stop()
|
|
129
|
-
print(result, "elapsed", t2)
|
|
130
|
-
print("elapsed", t1, t2, t1 - t2)
|
|
96
|
+
list_hosts = net.scan_port(port=22)
|
|
97
|
+
print(list_hosts)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: sciveo
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.9
|
|
4
4
|
Description-Content-Type: text/markdown
|
|
5
5
|
Provides-Extra: mon
|
|
6
6
|
Provides-Extra: net
|
|
@@ -49,10 +49,12 @@ When have sciveo account:
|
|
|
49
49
|
```shell
|
|
50
50
|
export SCIVEO_SECRET_ACCESS_KEY='my_sciveo_user_auth_token'
|
|
51
51
|
```
|
|
52
|
+
or create a file like ~/.sciveo/some_file_name where put:
|
|
53
|
+
secret_access_key=my_sciveo_user_auth_token
|
|
52
54
|
|
|
53
55
|
When using sciveo Monitoring just run, using suitable python environment
|
|
54
56
|
```shell
|
|
55
|
-
|
|
57
|
+
sciveo monitor --period 120
|
|
56
58
|
```
|
|
57
59
|
|
|
58
60
|
Monitoring started along with other python code.
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
README.md
|
|
2
2
|
setup.py
|
|
3
3
|
sciveo/__init__.py
|
|
4
|
+
sciveo/cli.py
|
|
4
5
|
sciveo/version.py
|
|
5
6
|
sciveo.egg-info/PKG-INFO
|
|
6
7
|
sciveo.egg-info/SOURCES.txt
|
|
7
8
|
sciveo.egg-info/dependency_links.txt
|
|
9
|
+
sciveo.egg-info/entry_points.txt
|
|
8
10
|
sciveo.egg-info/requires.txt
|
|
9
11
|
sciveo.egg-info/top_level.txt
|
|
10
12
|
sciveo/api/__init__.py
|
|
@@ -16,6 +18,7 @@ sciveo/common/model.py
|
|
|
16
18
|
sciveo/common/optimizers.py
|
|
17
19
|
sciveo/common/sampling.py
|
|
18
20
|
sciveo/common/tools/__init__.py
|
|
21
|
+
sciveo/common/tools/configuration.py
|
|
19
22
|
sciveo/common/tools/daemon.py
|
|
20
23
|
sciveo/common/tools/formating.py
|
|
21
24
|
sciveo/common/tools/hardware.py
|
sciveo-0.1.7/sciveo/version.py
DELETED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|