queuemo-quality 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- queuemo_quality-1.0.0.dist-info/METADATA +23 -0
- queuemo_quality-1.0.0.dist-info/RECORD +11 -0
- queuemo_quality-1.0.0.dist-info/WHEEL +5 -0
- queuemo_quality-1.0.0.dist-info/entry_points.txt +4 -0
- queuemo_quality-1.0.0.dist-info/top_level.txt +1 -0
- train/__init__.py +22 -0
- train/cli.py +63 -0
- train/loco.py +132 -0
- train/performance.py +101 -0
- train/pusher.py +175 -0
- train/race.py +93 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: queuemo-quality
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Author-email: Helmut Konrad Schewe <helmutus@outlook.com>
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/anaticulae/queuemo-quality
|
|
7
|
+
Project-URL: Repository, https://github.com/anaticulae/queuemo-quality
|
|
8
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
12
|
+
Requires-Python: >=3.12
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
Requires-Dist: configos<2.0.0,>=1.0.4
|
|
15
|
+
Requires-Dist: protoerror<4.0.0,>=3.20.4
|
|
16
|
+
Requires-Dist: queuemo<2.0.0,>=1.28.1
|
|
17
|
+
Requires-Dist: utilo<3.0.0,>=2.112.3
|
|
18
|
+
Provides-Extra: dev
|
|
19
|
+
Requires-Dist: hoverpower==1.7.0; extra == "dev"
|
|
20
|
+
Requires-Dist: utilotest==1.1.1; extra == "dev"
|
|
21
|
+
Requires-Dist: viewo==2.31.1; extra == "dev"
|
|
22
|
+
|
|
23
|
+
# train
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
train/__init__.py,sha256=rxj1gtWlbzZa8SMmyNmC_mQzC_N2JN5TWbHA2g7MyXE,919
|
|
2
|
+
train/cli.py,sha256=fFoB_Prse0R_CuprfOo0r3Yv5EygxStIC-PSavd7kGk,1936
|
|
3
|
+
train/loco.py,sha256=_X8tPZU8vXGD9vMPkpJAjt98ut1xxjTXWTGOycR4rVo,3946
|
|
4
|
+
train/performance.py,sha256=sEaRQm8DcWseflY_REb3nU7qig9WlxDLepkL9eQ6Yr4,3069
|
|
5
|
+
train/pusher.py,sha256=HVHNIPOf60RLHFDp9shgiKiNyLDqVgtZ2qpRwYZNg24,4704
|
|
6
|
+
train/race.py,sha256=cGi8nkWNIcA1ljFpJNIYflYfn_ImLeQFvONFBHS7IRY,2640
|
|
7
|
+
queuemo_quality-1.0.0.dist-info/METADATA,sha256=QXtAgT0STI122dpjIZt5QzhEuEgmxdKzeo7eHKjHNak,874
|
|
8
|
+
queuemo_quality-1.0.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
9
|
+
queuemo_quality-1.0.0.dist-info/entry_points.txt,sha256=EyPOLJWA7piabhjaNlI7Tsqtul3pCoTFrwHzCcID9UU,91
|
|
10
|
+
queuemo_quality-1.0.0.dist-info/top_level.txt,sha256=9uEqXwMETbwSSbVD4SZtqKrNOB8f-gkLKceJw9bv7Xg,6
|
|
11
|
+
queuemo_quality-1.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
train
|
train/__init__.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
#==============================================================================
|
|
2
|
+
# C O P Y R I G H T
|
|
3
|
+
#------------------------------------------------------------------------------
|
|
4
|
+
# Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
|
|
5
|
+
# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
|
|
6
|
+
# use or distribution is an offensive act against international law and may
|
|
7
|
+
# be prosecuted under federal law. Its content is company confidential.
|
|
8
|
+
#==============================================================================
|
|
9
|
+
|
|
10
|
+
import importlib.metadata
|
|
11
|
+
import os
|
|
12
|
+
|
|
13
|
+
import configos
|
|
14
|
+
|
|
15
|
+
__version__ = importlib.metadata.version('queuemo-quality')
|
|
16
|
+
|
|
17
|
+
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
|
|
18
|
+
PROCESS = 'train'
|
|
19
|
+
|
|
20
|
+
DEV_SERVER_PORT = configos.env(name='DEV_SERVER_PORT', default=8080)
|
|
21
|
+
# 169.254.149.20
|
|
22
|
+
DEV_SERVER_URL = configos.env(name='HEAVEN', default='localhost')
|
train/cli.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# =============================================================================
|
|
2
|
+
# C O P Y R I G H T
|
|
3
|
+
# -----------------------------------------------------------------------------
|
|
4
|
+
# Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
|
|
5
|
+
# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
|
|
6
|
+
# use or distribution is an offensive act against international law and may
|
|
7
|
+
# be prosecuted under federal law. Its content is company confidential.
|
|
8
|
+
# =============================================================================
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
import sys
|
|
12
|
+
|
|
13
|
+
import utilo
|
|
14
|
+
|
|
15
|
+
import train
|
|
16
|
+
import train.loco
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@utilo.saveme
|
|
20
|
+
def main():
|
|
21
|
+
commands = [
|
|
22
|
+
utilo.NumberedParameter(
|
|
23
|
+
shortcut='p',
|
|
24
|
+
message='used port',
|
|
25
|
+
default=train.DEV_SERVER_PORT,
|
|
26
|
+
),
|
|
27
|
+
utilo.NumberedParameter(shortcut='r', message='runtime', default=3600),
|
|
28
|
+
utilo.Parameter(longcut='workspace', message='workspace'),
|
|
29
|
+
]
|
|
30
|
+
parser = utilo.cli.create_parser(
|
|
31
|
+
todo=commands,
|
|
32
|
+
config=utilo.ParserConfiguration(
|
|
33
|
+
multiprocessed=True,
|
|
34
|
+
inputparameter=False,
|
|
35
|
+
outputparameter=False,
|
|
36
|
+
prefix=False,
|
|
37
|
+
verboseflag=True,
|
|
38
|
+
),
|
|
39
|
+
version=train.__version__,
|
|
40
|
+
)
|
|
41
|
+
args = utilo.parse(parser)
|
|
42
|
+
|
|
43
|
+
port = int(args['p'])
|
|
44
|
+
runtime = int(args['r'])
|
|
45
|
+
jobs = int(args['job'])
|
|
46
|
+
workspace = os.getcwd() if args['workspace'] is None else args['workspace']
|
|
47
|
+
if not os.path.exists(workspace):
|
|
48
|
+
utilo.error(f'workspace does not exists: {workspace}')
|
|
49
|
+
sys.exit(utilo.FAILURE)
|
|
50
|
+
|
|
51
|
+
loco = train.loco.Locomotive(
|
|
52
|
+
root=workspace,
|
|
53
|
+
port=port,
|
|
54
|
+
runtime=runtime,
|
|
55
|
+
jobs=jobs,
|
|
56
|
+
)
|
|
57
|
+
if loco.started():
|
|
58
|
+
utilo.error(f'server already started: {port} | {workspace}')
|
|
59
|
+
sys.exit(utilo.FAILURE)
|
|
60
|
+
|
|
61
|
+
loco.start()
|
|
62
|
+
loco.wait(runtime)
|
|
63
|
+
sys.exit(utilo.SUCCESS)
|
train/loco.py
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# =============================================================================
|
|
2
|
+
# C O P Y R I G H T
|
|
3
|
+
# -----------------------------------------------------------------------------
|
|
4
|
+
# Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
|
|
5
|
+
# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
|
|
6
|
+
# use or distribution is an offensive act against international law and may
|
|
7
|
+
# be prosecuted under federal law. Its content is company confidential.
|
|
8
|
+
# =============================================================================
|
|
9
|
+
|
|
10
|
+
import collections
|
|
11
|
+
import os
|
|
12
|
+
import subprocess # nosec
|
|
13
|
+
|
|
14
|
+
import utilo
|
|
15
|
+
|
|
16
|
+
import train
|
|
17
|
+
|
|
18
|
+
Status = collections.namedtuple('Status', 'viewo, queuemo')
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Locomotive:
|
|
22
|
+
|
|
23
|
+
def __init__(
|
|
24
|
+
self,
|
|
25
|
+
root: str,
|
|
26
|
+
port: int,
|
|
27
|
+
runtime: int = 1,
|
|
28
|
+
jobs: int = 1,
|
|
29
|
+
verbose: bool = False,
|
|
30
|
+
):
|
|
31
|
+
self.root = root
|
|
32
|
+
self.port = port
|
|
33
|
+
self.runtime = runtime
|
|
34
|
+
self.jobs = jobs
|
|
35
|
+
self.viewo = None
|
|
36
|
+
self.queuemo = None
|
|
37
|
+
self.verbose = verbose
|
|
38
|
+
|
|
39
|
+
def state(self) -> Status:
|
|
40
|
+
viewo = check_process(self.viewo)
|
|
41
|
+
queuemo = None
|
|
42
|
+
if self.queuemo:
|
|
43
|
+
queuemo = check_process(self.queuemo)
|
|
44
|
+
return Status(viewo=viewo, queuemo=queuemo)
|
|
45
|
+
|
|
46
|
+
def started(self) -> bool:
|
|
47
|
+
completed = utilo.run(
|
|
48
|
+
f'curl http://{train.DEV_SERVER_URL}:{self.port}',
|
|
49
|
+
expect=None,
|
|
50
|
+
)
|
|
51
|
+
return not completed.returncode
|
|
52
|
+
|
|
53
|
+
def close(self) -> Status:
|
|
54
|
+
self.viewo.terminate()
|
|
55
|
+
if self.queuemo:
|
|
56
|
+
self.queuemo.terminate()
|
|
57
|
+
|
|
58
|
+
def start(self, queuemo_start: bool = True):
|
|
59
|
+
assert not self.started(), 'already running'
|
|
60
|
+
assert self.viewo is None and self.queuemo is None
|
|
61
|
+
|
|
62
|
+
tmp = os.path.join(self.root, 'tmp')
|
|
63
|
+
os.makedirs(tmp)
|
|
64
|
+
todo = os.path.join(self.root, 'todo')
|
|
65
|
+
os.makedirs(todo)
|
|
66
|
+
ready = os.path.join(self.root, 'ready')
|
|
67
|
+
os.makedirs(ready)
|
|
68
|
+
|
|
69
|
+
silent = subprocess.PIPE if not self.verbose else None
|
|
70
|
+
|
|
71
|
+
env = dict(os.environ)
|
|
72
|
+
env['SHARED_SPACE'] = str(self.root)
|
|
73
|
+
env['SHARED_TODO'] = str(todo)
|
|
74
|
+
env['SHARED_READY'] = str(ready)
|
|
75
|
+
env['SHARED_TMP'] = str(tmp)
|
|
76
|
+
|
|
77
|
+
cmd = f'viewo --port {self.port} -VVV'.split()
|
|
78
|
+
# HINT: DO NOT USE WITH TO AVOID WAITING PROBLEM
|
|
79
|
+
# pylint:disable=R1732
|
|
80
|
+
self.viewo = subprocess.Popen( # nosec
|
|
81
|
+
cmd,
|
|
82
|
+
env=env,
|
|
83
|
+
stdout=silent,
|
|
84
|
+
stderr=silent,
|
|
85
|
+
)
|
|
86
|
+
# ensure that process is started correctly
|
|
87
|
+
assert check_process(self.viewo) is None
|
|
88
|
+
if queuemo_start:
|
|
89
|
+
cmd = f'queuemo --runtime={self.runtime} --jobs={self.jobs}'.split()
|
|
90
|
+
self.queuemo = subprocess.Popen( # nosec
|
|
91
|
+
cmd,
|
|
92
|
+
env=env,
|
|
93
|
+
stdout=silent,
|
|
94
|
+
stderr=silent,
|
|
95
|
+
)
|
|
96
|
+
# ensure that process is started correctly
|
|
97
|
+
assert check_process(self.queuemo) is None # TODO: VERIFY THIS
|
|
98
|
+
|
|
99
|
+
def wait(self, seconds=10) -> Status:
|
|
100
|
+
result = None
|
|
101
|
+
if seconds is None:
|
|
102
|
+
seconds = 3600
|
|
103
|
+
for _ in range(seconds):
|
|
104
|
+
result = self.state()
|
|
105
|
+
if result.queuemo is not None:
|
|
106
|
+
assert result.viewo is None, str(result)
|
|
107
|
+
self.close()
|
|
108
|
+
# update starte
|
|
109
|
+
result = self.state()
|
|
110
|
+
break
|
|
111
|
+
return result
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def create(root: str, port: int, runtime: int = 1):
|
|
115
|
+
return Locomotive(
|
|
116
|
+
root=root,
|
|
117
|
+
port=port,
|
|
118
|
+
runtime=runtime,
|
|
119
|
+
verbose=True,
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def check_process(process) -> int:
|
|
124
|
+
try:
|
|
125
|
+
outs, errs = process.communicate(timeout=1.0)
|
|
126
|
+
if outs:
|
|
127
|
+
utilo.info(outs)
|
|
128
|
+
if errs:
|
|
129
|
+
utilo.info(errs)
|
|
130
|
+
except subprocess.TimeoutExpired:
|
|
131
|
+
pass
|
|
132
|
+
return process.returncode
|
train/performance.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# =============================================================================
|
|
2
|
+
# C O P Y R I G H T
|
|
3
|
+
# -----------------------------------------------------------------------------
|
|
4
|
+
# Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
|
|
5
|
+
# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
|
|
6
|
+
# use or distribution is an offensive act against international law and may
|
|
7
|
+
# be prosecuted under federal law. Its content is company confidential.
|
|
8
|
+
# =============================================================================
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
import subprocess
|
|
12
|
+
import sys
|
|
13
|
+
import time
|
|
14
|
+
|
|
15
|
+
import utilo
|
|
16
|
+
|
|
17
|
+
import train
|
|
18
|
+
import train.loco
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def wait(process):
|
|
22
|
+
completed = train.loco.check_process(process)
|
|
23
|
+
while completed is None: # pylint:disable=W0149
|
|
24
|
+
completed = train.loco.check_process(process)
|
|
25
|
+
time.sleep(1.0)
|
|
26
|
+
return completed
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def upload(
|
|
30
|
+
file: str,
|
|
31
|
+
server: str = train.DEV_SERVER_URL,
|
|
32
|
+
port: int = train.DEV_SERVER_PORT,
|
|
33
|
+
public: bool = False,
|
|
34
|
+
verbose: bool = False,
|
|
35
|
+
):
|
|
36
|
+
"""\
|
|
37
|
+
Args:
|
|
38
|
+
file(str): file upload source
|
|
39
|
+
server(str): ip adress
|
|
40
|
+
port(int): server port
|
|
41
|
+
public(bool): make document accessible by public user
|
|
42
|
+
verbose(bool): if True log curl invocation
|
|
43
|
+
"""
|
|
44
|
+
base, filename = os.path.split(file)
|
|
45
|
+
public = '?public=True' if public else ''
|
|
46
|
+
api = f'http://{server}:{port}/api/v0/documents{public}'
|
|
47
|
+
cmd = f'curl -F "file=@{filename}" {api}'
|
|
48
|
+
if verbose:
|
|
49
|
+
utilo.log(cmd)
|
|
50
|
+
completed = utilo.run(cmd, cwd=base, expect=None)
|
|
51
|
+
if completed.returncode:
|
|
52
|
+
utilo.error(cmd)
|
|
53
|
+
utilo.error(completed.stderr)
|
|
54
|
+
utilo.error('check connection')
|
|
55
|
+
sys.exit(utilo.FAILURE)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
TWO_HOURS = 3600 * 2
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def run(
|
|
62
|
+
workspace,
|
|
63
|
+
source,
|
|
64
|
+
port=train.DEV_SERVER_PORT,
|
|
65
|
+
runtime=TWO_HOURS,
|
|
66
|
+
) -> str:
|
|
67
|
+
if not utilo.hasprog('train'):
|
|
68
|
+
utilo.error('install `train` please')
|
|
69
|
+
sys.exit(127)
|
|
70
|
+
cmd = f'train --workspace {workspace} -j=1 -p={port} -r={runtime}'.split()
|
|
71
|
+
utilo.log(cmd)
|
|
72
|
+
with subprocess.Popen( # nosec
|
|
73
|
+
cmd,
|
|
74
|
+
stdout=subprocess.PIPE,
|
|
75
|
+
stderr=subprocess.PIPE,
|
|
76
|
+
) as process:
|
|
77
|
+
waitforstartup(port)
|
|
78
|
+
returncode = train.loco.check_process(process)
|
|
79
|
+
assert returncode is None, returncode
|
|
80
|
+
# upload test file
|
|
81
|
+
upload(file=source, port=port, public=True)
|
|
82
|
+
# wait for finished process
|
|
83
|
+
returncode = wait(process)
|
|
84
|
+
assert returncode == utilo.SUCCESS, returncode
|
|
85
|
+
ready = os.path.join(workspace, 'ready')
|
|
86
|
+
count = utilo.file_count(ready)
|
|
87
|
+
assert count >= 15, ready
|
|
88
|
+
return ready
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def waitforstartup(port: int, server: str = train.DEV_SERVER_URL):
|
|
92
|
+
cmd = f'curl http://{server}:{port}'
|
|
93
|
+
# ensure correct startup
|
|
94
|
+
for _ in range(100):
|
|
95
|
+
# curl till server is online
|
|
96
|
+
completed = utilo.run(cmd, expect=None)
|
|
97
|
+
if completed.returncode == utilo.SUCCESS:
|
|
98
|
+
break
|
|
99
|
+
else:
|
|
100
|
+
utilo.error(f'timeout: {cmd}')
|
|
101
|
+
sys.exit(utilo.FAILURE)
|
train/pusher.py
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
# =============================================================================
|
|
2
|
+
# C O P Y R I G H T
|
|
3
|
+
# -----------------------------------------------------------------------------
|
|
4
|
+
# Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
|
|
5
|
+
# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
|
|
6
|
+
# use or distribution is an offensive act against international law and may
|
|
7
|
+
# be prosecuted under federal law. Its content is company confidential.
|
|
8
|
+
# =============================================================================
|
|
9
|
+
|
|
10
|
+
import hoverpower
|
|
11
|
+
import utilo
|
|
12
|
+
|
|
13
|
+
import train
|
|
14
|
+
import train.performance
|
|
15
|
+
|
|
16
|
+
try:
|
|
17
|
+
import hardcore # pylint:disable=E0401
|
|
18
|
+
except ModuleNotFoundError:
|
|
19
|
+
hardcore = None # pylint:disable=C0103
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@utilo.saveme
|
|
23
|
+
def main():
|
|
24
|
+
args = create()
|
|
25
|
+
jobs = select(args)
|
|
26
|
+
if not jobs:
|
|
27
|
+
utilo.error('nothing todo')
|
|
28
|
+
return utilo.FAILURE
|
|
29
|
+
port = train.DEV_SERVER_PORT
|
|
30
|
+
if args['port']:
|
|
31
|
+
port = int(args['port'])
|
|
32
|
+
host = train.DEV_SERVER_URL
|
|
33
|
+
if args['host']:
|
|
34
|
+
host = args['host']
|
|
35
|
+
upload(
|
|
36
|
+
jobs=jobs,
|
|
37
|
+
port=port,
|
|
38
|
+
host=host,
|
|
39
|
+
)
|
|
40
|
+
return utilo.SUCCESS
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def create():
|
|
44
|
+
hoverpowers = utilo.Parameter(
|
|
45
|
+
shortcut='p',
|
|
46
|
+
longcut='hoverpower',
|
|
47
|
+
message='hoverpower files',
|
|
48
|
+
)
|
|
49
|
+
hardcores = utilo.Parameter(
|
|
50
|
+
shortcut='hc',
|
|
51
|
+
longcut='hardcore',
|
|
52
|
+
message='hardcore files',
|
|
53
|
+
)
|
|
54
|
+
commands = [
|
|
55
|
+
hoverpowers,
|
|
56
|
+
hardcores,
|
|
57
|
+
utilo.NumberedParameter(
|
|
58
|
+
shortcut='n',
|
|
59
|
+
longcut='newest',
|
|
60
|
+
message='n newest files',
|
|
61
|
+
default=0,
|
|
62
|
+
),
|
|
63
|
+
utilo.NumberedParameter(
|
|
64
|
+
shortcut='o',
|
|
65
|
+
longcut='oldest',
|
|
66
|
+
message='n oldest files',
|
|
67
|
+
default=0,
|
|
68
|
+
),
|
|
69
|
+
utilo.Parameter(
|
|
70
|
+
longcut='host',
|
|
71
|
+
message='server url to publish',
|
|
72
|
+
),
|
|
73
|
+
utilo.NumberedParameter(
|
|
74
|
+
longcut='port',
|
|
75
|
+
message='server port to publish',
|
|
76
|
+
default=train.DEV_SERVER_PORT,
|
|
77
|
+
),
|
|
78
|
+
]
|
|
79
|
+
parser = utilo.create_parser(
|
|
80
|
+
todo=commands,
|
|
81
|
+
config=utilo.ParserConfiguration.all_off(),
|
|
82
|
+
version=train.__version__,
|
|
83
|
+
)
|
|
84
|
+
parser.add_argument(
|
|
85
|
+
'regex',
|
|
86
|
+
nargs='?',
|
|
87
|
+
help='select matching files or a single file.',
|
|
88
|
+
)
|
|
89
|
+
args = utilo.parse(parser)
|
|
90
|
+
return args
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def select(args) -> list:
|
|
94
|
+
jobs = []
|
|
95
|
+
if hoverpowers := args.get('hoverpower'):
|
|
96
|
+
pages = utilo.parse_pages(hoverpowers, pagecount=len(hoverpower.PDF))
|
|
97
|
+
if pages is None:
|
|
98
|
+
jobs.extend(hoverpower.PDF)
|
|
99
|
+
else:
|
|
100
|
+
jobs.extend([hoverpower.PDF[page] for page in pages])
|
|
101
|
+
if hardcores := args.get('hardcore'):
|
|
102
|
+
pages = utilo.parse_pages(hardcores, pagecount=len(hardcore.ALL))
|
|
103
|
+
if pages is None:
|
|
104
|
+
jobs.extend(hardcore.ALL)
|
|
105
|
+
else:
|
|
106
|
+
jobs.extend([hardcore.ALL[page] for page in pages])
|
|
107
|
+
if number := args.get('newest'):
|
|
108
|
+
jobs.extend(newest(number))
|
|
109
|
+
if number := args.get('oldest'):
|
|
110
|
+
jobs.extend(oldest(number))
|
|
111
|
+
if regex := args.get('regex'):
|
|
112
|
+
if files := file_select(regex):
|
|
113
|
+
jobs.extend(files)
|
|
114
|
+
else:
|
|
115
|
+
jobs.extend(regex_select(regex=regex))
|
|
116
|
+
return jobs
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def regex_select(regex: str) -> list:
|
|
120
|
+
"""Select files via regex.
|
|
121
|
+
|
|
122
|
+
Use hoverpower and hardcore as source of selection.
|
|
123
|
+
|
|
124
|
+
>>> regex_select('bachelor032')
|
|
125
|
+
['...', '...',...]
|
|
126
|
+
"""
|
|
127
|
+
result = [
|
|
128
|
+
item for item in hoverpower.PDF # + hardcore.ALL # TODO: ENABLE LATER
|
|
129
|
+
if utilo.file_name(item).startswith(regex)
|
|
130
|
+
]
|
|
131
|
+
return result
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def file_select(path: str) -> list:
|
|
135
|
+
"""Select file which is not part of corpus(hoverpower, hardcore).
|
|
136
|
+
|
|
137
|
+
This file is mostly selected via cli.
|
|
138
|
+
|
|
139
|
+
>>> file_select(__file__)
|
|
140
|
+
['...pusher.py']
|
|
141
|
+
"""
|
|
142
|
+
if not utilo.isfilepath(path):
|
|
143
|
+
return None
|
|
144
|
+
return [utilo.make_absolute(path)]
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def upload(jobs, port, host):
|
|
148
|
+
utilo.log(f'publish to port: {port}')
|
|
149
|
+
utilo.log(f'publish to host: {host}')
|
|
150
|
+
for path in jobs:
|
|
151
|
+
utilo.log(f'upload: {path}')
|
|
152
|
+
train.performance.upload(
|
|
153
|
+
path,
|
|
154
|
+
server=host,
|
|
155
|
+
port=port,
|
|
156
|
+
public=True,
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def newest(count: int = 10) -> list:
|
|
161
|
+
"""\
|
|
162
|
+
>>> assert len(newest(10)) == 10
|
|
163
|
+
"""
|
|
164
|
+
files = sorted(hoverpower.PDF, key=utilo.file_age)
|
|
165
|
+
result = files[0:count]
|
|
166
|
+
return result
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def oldest(count: int = 10) -> list:
|
|
170
|
+
"""\
|
|
171
|
+
>>> assert len(oldest(5)) == 5
|
|
172
|
+
"""
|
|
173
|
+
files = sorted(hoverpower.PDF, key=utilo.file_age, reverse=True)
|
|
174
|
+
result = files[0:count]
|
|
175
|
+
return result
|
train/race.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# =============================================================================
|
|
2
|
+
# C O P Y R I G H T
|
|
3
|
+
# -----------------------------------------------------------------------------
|
|
4
|
+
# Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
|
|
5
|
+
# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
|
|
6
|
+
# use or distribution is an offensive act against international law and may
|
|
7
|
+
# be prosecuted under federal law. Its content is company confidential.
|
|
8
|
+
# =============================================================================
|
|
9
|
+
|
|
10
|
+
import sys
|
|
11
|
+
|
|
12
|
+
import configos
|
|
13
|
+
import hoverpower
|
|
14
|
+
import protoerror
|
|
15
|
+
import utilo
|
|
16
|
+
|
|
17
|
+
import train
|
|
18
|
+
import train.performance
|
|
19
|
+
|
|
20
|
+
FIRST = [
|
|
21
|
+
hoverpower.MASTER098_PDF,
|
|
22
|
+
hoverpower.MASTER116_PDF,
|
|
23
|
+
hoverpower.BACHELOR090_PDF,
|
|
24
|
+
hoverpower.HOME025_PDF,
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
SECOND = [
|
|
28
|
+
hoverpower.MASTER110_PDF,
|
|
29
|
+
hoverpower.BACHELOR128_PDF,
|
|
30
|
+
hoverpower.MASTER031_PDF,
|
|
31
|
+
hoverpower.BACHELOR051_PDF,
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
THIRD = [
|
|
35
|
+
hoverpower.MASTER072_PDF,
|
|
36
|
+
hoverpower.MASTER089_PDF,
|
|
37
|
+
hoverpower.MASTER098_PDF,
|
|
38
|
+
hoverpower.MASTER099_PDF,
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@utilo.saveme
|
|
43
|
+
def main():
|
|
44
|
+
commands = [
|
|
45
|
+
utilo.Flag(longcut='first'),
|
|
46
|
+
utilo.Flag(longcut='second'),
|
|
47
|
+
utilo.Flag(longcut='third'),
|
|
48
|
+
]
|
|
49
|
+
parser = utilo.create_parser(
|
|
50
|
+
todo=commands,
|
|
51
|
+
config=utilo.ParserConfiguration(
|
|
52
|
+
outputparameter=False,
|
|
53
|
+
inputparameter=False,
|
|
54
|
+
prefix=False,
|
|
55
|
+
verboseflag=True,
|
|
56
|
+
),
|
|
57
|
+
description='Determine performance for selected pdf package.',
|
|
58
|
+
version=train.__version__,
|
|
59
|
+
)
|
|
60
|
+
args = utilo.parse(parser)
|
|
61
|
+
|
|
62
|
+
first = args.get('first', False)
|
|
63
|
+
second = args.get('second', False)
|
|
64
|
+
third = args.get('third', False)
|
|
65
|
+
|
|
66
|
+
if first:
|
|
67
|
+
run(FIRST, start=4700)
|
|
68
|
+
|
|
69
|
+
if second:
|
|
70
|
+
run(SECOND, start=4710)
|
|
71
|
+
|
|
72
|
+
if third:
|
|
73
|
+
run(THIRD, start=4720)
|
|
74
|
+
|
|
75
|
+
if not any((first, second, third)):
|
|
76
|
+
parser.print_usage()
|
|
77
|
+
sys.exit(utilo.FAILURE)
|
|
78
|
+
sys.exit(utilo.SUCCESS)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def run(todo, start: int = 4700):
|
|
82
|
+
for port, path in enumerate(todo, start=start):
|
|
83
|
+
workspace = utilo.tmpdir(configos.tmp())
|
|
84
|
+
utilo.log(f'run performance for {path} in {workspace} on: {port}')
|
|
85
|
+
with utilo.profile(path):
|
|
86
|
+
ready = train.performance.run(workspace, path, port=port)
|
|
87
|
+
# load findings by page
|
|
88
|
+
findings = protoerror.findings_from_path(ready)
|
|
89
|
+
# global finding list
|
|
90
|
+
findings = utilo.flat([item.content for item in findings])
|
|
91
|
+
# TODO: REPLACE AFTER UPGRADING?
|
|
92
|
+
findings = utilo.unique(findings)
|
|
93
|
+
utilo.log(f'findings: {len(findings)}')
|