queuemo-quality 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,23 @@
1
+ Metadata-Version: 2.4
2
+ Name: queuemo-quality
3
+ Version: 1.0.0
4
+ Author-email: Helmut Konrad Schewe <helmutus@outlook.com>
5
+ License-Expression: MIT
6
+ Project-URL: Homepage, https://github.com/anaticulae/queuemo-quality
7
+ Project-URL: Repository, https://github.com/anaticulae/queuemo-quality
8
+ Classifier: Programming Language :: Python :: 3.12
9
+ Classifier: Programming Language :: Python :: 3.13
10
+ Classifier: Programming Language :: Python :: 3.14
11
+ Classifier: Programming Language :: Python :: 3.15
12
+ Requires-Python: >=3.12
13
+ Description-Content-Type: text/markdown
14
+ Requires-Dist: configos<2.0.0,>=1.0.4
15
+ Requires-Dist: protoerror<4.0.0,>=3.20.4
16
+ Requires-Dist: queuemo<2.0.0,>=1.28.1
17
+ Requires-Dist: utilo<3.0.0,>=2.112.3
18
+ Provides-Extra: dev
19
+ Requires-Dist: hoverpower==1.7.0; extra == "dev"
20
+ Requires-Dist: utilotest==1.1.1; extra == "dev"
21
+ Requires-Dist: viewo==2.31.1; extra == "dev"
22
+
23
+ # train
@@ -0,0 +1,11 @@
1
+ train/__init__.py,sha256=rxj1gtWlbzZa8SMmyNmC_mQzC_N2JN5TWbHA2g7MyXE,919
2
+ train/cli.py,sha256=fFoB_Prse0R_CuprfOo0r3Yv5EygxStIC-PSavd7kGk,1936
3
+ train/loco.py,sha256=_X8tPZU8vXGD9vMPkpJAjt98ut1xxjTXWTGOycR4rVo,3946
4
+ train/performance.py,sha256=sEaRQm8DcWseflY_REb3nU7qig9WlxDLepkL9eQ6Yr4,3069
5
+ train/pusher.py,sha256=HVHNIPOf60RLHFDp9shgiKiNyLDqVgtZ2qpRwYZNg24,4704
6
+ train/race.py,sha256=cGi8nkWNIcA1ljFpJNIYflYfn_ImLeQFvONFBHS7IRY,2640
7
+ queuemo_quality-1.0.0.dist-info/METADATA,sha256=QXtAgT0STI122dpjIZt5QzhEuEgmxdKzeo7eHKjHNak,874
8
+ queuemo_quality-1.0.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
9
+ queuemo_quality-1.0.0.dist-info/entry_points.txt,sha256=EyPOLJWA7piabhjaNlI7Tsqtul3pCoTFrwHzCcID9UU,91
10
+ queuemo_quality-1.0.0.dist-info/top_level.txt,sha256=9uEqXwMETbwSSbVD4SZtqKrNOB8f-gkLKceJw9bv7Xg,6
11
+ queuemo_quality-1.0.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,4 @@
1
+ [console_scripts]
2
+ pusher = train.pusher:main
3
+ race = train.race:main
4
+ train = train.cli:main
@@ -0,0 +1 @@
1
+ train
train/__init__.py ADDED
@@ -0,0 +1,22 @@
1
+ #==============================================================================
2
+ # C O P Y R I G H T
3
+ #------------------------------------------------------------------------------
4
+ # Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
5
+ # This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
6
+ # use or distribution is an offensive act against international law and may
7
+ # be prosecuted under federal law. Its content is company confidential.
8
+ #==============================================================================
9
+
10
+ import importlib.metadata
11
+ import os
12
+
13
+ import configos
14
+
15
+ __version__ = importlib.metadata.version('queuemo-quality')
16
+
17
+ ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
18
+ PROCESS = 'train'
19
+
20
+ DEV_SERVER_PORT = configos.env(name='DEV_SERVER_PORT', default=8080)
21
+ # 169.254.149.20
22
+ DEV_SERVER_URL = configos.env(name='HEAVEN', default='localhost')
train/cli.py ADDED
@@ -0,0 +1,63 @@
1
+ # =============================================================================
2
+ # C O P Y R I G H T
3
+ # -----------------------------------------------------------------------------
4
+ # Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
5
+ # This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
6
+ # use or distribution is an offensive act against international law and may
7
+ # be prosecuted under federal law. Its content is company confidential.
8
+ # =============================================================================
9
+
10
+ import os
11
+ import sys
12
+
13
+ import utilo
14
+
15
+ import train
16
+ import train.loco
17
+
18
+
19
+ @utilo.saveme
20
+ def main():
21
+ commands = [
22
+ utilo.NumberedParameter(
23
+ shortcut='p',
24
+ message='used port',
25
+ default=train.DEV_SERVER_PORT,
26
+ ),
27
+ utilo.NumberedParameter(shortcut='r', message='runtime', default=3600),
28
+ utilo.Parameter(longcut='workspace', message='workspace'),
29
+ ]
30
+ parser = utilo.cli.create_parser(
31
+ todo=commands,
32
+ config=utilo.ParserConfiguration(
33
+ multiprocessed=True,
34
+ inputparameter=False,
35
+ outputparameter=False,
36
+ prefix=False,
37
+ verboseflag=True,
38
+ ),
39
+ version=train.__version__,
40
+ )
41
+ args = utilo.parse(parser)
42
+
43
+ port = int(args['p'])
44
+ runtime = int(args['r'])
45
+ jobs = int(args['job'])
46
+ workspace = os.getcwd() if args['workspace'] is None else args['workspace']
47
+ if not os.path.exists(workspace):
48
+ utilo.error(f'workspace does not exists: {workspace}')
49
+ sys.exit(utilo.FAILURE)
50
+
51
+ loco = train.loco.Locomotive(
52
+ root=workspace,
53
+ port=port,
54
+ runtime=runtime,
55
+ jobs=jobs,
56
+ )
57
+ if loco.started():
58
+ utilo.error(f'server already started: {port} | {workspace}')
59
+ sys.exit(utilo.FAILURE)
60
+
61
+ loco.start()
62
+ loco.wait(runtime)
63
+ sys.exit(utilo.SUCCESS)
train/loco.py ADDED
@@ -0,0 +1,132 @@
1
+ # =============================================================================
2
+ # C O P Y R I G H T
3
+ # -----------------------------------------------------------------------------
4
+ # Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
5
+ # This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
6
+ # use or distribution is an offensive act against international law and may
7
+ # be prosecuted under federal law. Its content is company confidential.
8
+ # =============================================================================
9
+
10
+ import collections
11
+ import os
12
+ import subprocess # nosec
13
+
14
+ import utilo
15
+
16
+ import train
17
+
18
+ Status = collections.namedtuple('Status', 'viewo, queuemo')
19
+
20
+
21
+ class Locomotive:
22
+
23
+ def __init__(
24
+ self,
25
+ root: str,
26
+ port: int,
27
+ runtime: int = 1,
28
+ jobs: int = 1,
29
+ verbose: bool = False,
30
+ ):
31
+ self.root = root
32
+ self.port = port
33
+ self.runtime = runtime
34
+ self.jobs = jobs
35
+ self.viewo = None
36
+ self.queuemo = None
37
+ self.verbose = verbose
38
+
39
+ def state(self) -> Status:
40
+ viewo = check_process(self.viewo)
41
+ queuemo = None
42
+ if self.queuemo:
43
+ queuemo = check_process(self.queuemo)
44
+ return Status(viewo=viewo, queuemo=queuemo)
45
+
46
+ def started(self) -> bool:
47
+ completed = utilo.run(
48
+ f'curl http://{train.DEV_SERVER_URL}:{self.port}',
49
+ expect=None,
50
+ )
51
+ return not completed.returncode
52
+
53
+ def close(self) -> Status:
54
+ self.viewo.terminate()
55
+ if self.queuemo:
56
+ self.queuemo.terminate()
57
+
58
+ def start(self, queuemo_start: bool = True):
59
+ assert not self.started(), 'already running'
60
+ assert self.viewo is None and self.queuemo is None
61
+
62
+ tmp = os.path.join(self.root, 'tmp')
63
+ os.makedirs(tmp)
64
+ todo = os.path.join(self.root, 'todo')
65
+ os.makedirs(todo)
66
+ ready = os.path.join(self.root, 'ready')
67
+ os.makedirs(ready)
68
+
69
+ silent = subprocess.PIPE if not self.verbose else None
70
+
71
+ env = dict(os.environ)
72
+ env['SHARED_SPACE'] = str(self.root)
73
+ env['SHARED_TODO'] = str(todo)
74
+ env['SHARED_READY'] = str(ready)
75
+ env['SHARED_TMP'] = str(tmp)
76
+
77
+ cmd = f'viewo --port {self.port} -VVV'.split()
78
+ # HINT: DO NOT USE WITH TO AVOID WAITING PROBLEM
79
+ # pylint:disable=R1732
80
+ self.viewo = subprocess.Popen( # nosec
81
+ cmd,
82
+ env=env,
83
+ stdout=silent,
84
+ stderr=silent,
85
+ )
86
+ # ensure that process is started correctly
87
+ assert check_process(self.viewo) is None
88
+ if queuemo_start:
89
+ cmd = f'queuemo --runtime={self.runtime} --jobs={self.jobs}'.split()
90
+ self.queuemo = subprocess.Popen( # nosec
91
+ cmd,
92
+ env=env,
93
+ stdout=silent,
94
+ stderr=silent,
95
+ )
96
+ # ensure that process is started correctly
97
+ assert check_process(self.queuemo) is None # TODO: VERIFY THIS
98
+
99
+ def wait(self, seconds=10) -> Status:
100
+ result = None
101
+ if seconds is None:
102
+ seconds = 3600
103
+ for _ in range(seconds):
104
+ result = self.state()
105
+ if result.queuemo is not None:
106
+ assert result.viewo is None, str(result)
107
+ self.close()
108
+ # update starte
109
+ result = self.state()
110
+ break
111
+ return result
112
+
113
+
114
+ def create(root: str, port: int, runtime: int = 1):
115
+ return Locomotive(
116
+ root=root,
117
+ port=port,
118
+ runtime=runtime,
119
+ verbose=True,
120
+ )
121
+
122
+
123
+ def check_process(process) -> int:
124
+ try:
125
+ outs, errs = process.communicate(timeout=1.0)
126
+ if outs:
127
+ utilo.info(outs)
128
+ if errs:
129
+ utilo.info(errs)
130
+ except subprocess.TimeoutExpired:
131
+ pass
132
+ return process.returncode
train/performance.py ADDED
@@ -0,0 +1,101 @@
1
+ # =============================================================================
2
+ # C O P Y R I G H T
3
+ # -----------------------------------------------------------------------------
4
+ # Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
5
+ # This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
6
+ # use or distribution is an offensive act against international law and may
7
+ # be prosecuted under federal law. Its content is company confidential.
8
+ # =============================================================================
9
+
10
+ import os
11
+ import subprocess
12
+ import sys
13
+ import time
14
+
15
+ import utilo
16
+
17
+ import train
18
+ import train.loco
19
+
20
+
21
+ def wait(process):
22
+ completed = train.loco.check_process(process)
23
+ while completed is None: # pylint:disable=W0149
24
+ completed = train.loco.check_process(process)
25
+ time.sleep(1.0)
26
+ return completed
27
+
28
+
29
+ def upload(
30
+ file: str,
31
+ server: str = train.DEV_SERVER_URL,
32
+ port: int = train.DEV_SERVER_PORT,
33
+ public: bool = False,
34
+ verbose: bool = False,
35
+ ):
36
+ """\
37
+ Args:
38
+ file(str): file upload source
39
+ server(str): ip adress
40
+ port(int): server port
41
+ public(bool): make document accessible by public user
42
+ verbose(bool): if True log curl invocation
43
+ """
44
+ base, filename = os.path.split(file)
45
+ public = '?public=True' if public else ''
46
+ api = f'http://{server}:{port}/api/v0/documents{public}'
47
+ cmd = f'curl -F "file=@{filename}" {api}'
48
+ if verbose:
49
+ utilo.log(cmd)
50
+ completed = utilo.run(cmd, cwd=base, expect=None)
51
+ if completed.returncode:
52
+ utilo.error(cmd)
53
+ utilo.error(completed.stderr)
54
+ utilo.error('check connection')
55
+ sys.exit(utilo.FAILURE)
56
+
57
+
58
+ TWO_HOURS = 3600 * 2
59
+
60
+
61
+ def run(
62
+ workspace,
63
+ source,
64
+ port=train.DEV_SERVER_PORT,
65
+ runtime=TWO_HOURS,
66
+ ) -> str:
67
+ if not utilo.hasprog('train'):
68
+ utilo.error('install `train` please')
69
+ sys.exit(127)
70
+ cmd = f'train --workspace {workspace} -j=1 -p={port} -r={runtime}'.split()
71
+ utilo.log(cmd)
72
+ with subprocess.Popen( # nosec
73
+ cmd,
74
+ stdout=subprocess.PIPE,
75
+ stderr=subprocess.PIPE,
76
+ ) as process:
77
+ waitforstartup(port)
78
+ returncode = train.loco.check_process(process)
79
+ assert returncode is None, returncode
80
+ # upload test file
81
+ upload(file=source, port=port, public=True)
82
+ # wait for finished process
83
+ returncode = wait(process)
84
+ assert returncode == utilo.SUCCESS, returncode
85
+ ready = os.path.join(workspace, 'ready')
86
+ count = utilo.file_count(ready)
87
+ assert count >= 15, ready
88
+ return ready
89
+
90
+
91
+ def waitforstartup(port: int, server: str = train.DEV_SERVER_URL):
92
+ cmd = f'curl http://{server}:{port}'
93
+ # ensure correct startup
94
+ for _ in range(100):
95
+ # curl till server is online
96
+ completed = utilo.run(cmd, expect=None)
97
+ if completed.returncode == utilo.SUCCESS:
98
+ break
99
+ else:
100
+ utilo.error(f'timeout: {cmd}')
101
+ sys.exit(utilo.FAILURE)
train/pusher.py ADDED
@@ -0,0 +1,175 @@
1
+ # =============================================================================
2
+ # C O P Y R I G H T
3
+ # -----------------------------------------------------------------------------
4
+ # Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
5
+ # This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
6
+ # use or distribution is an offensive act against international law and may
7
+ # be prosecuted under federal law. Its content is company confidential.
8
+ # =============================================================================
9
+
10
+ import hoverpower
11
+ import utilo
12
+
13
+ import train
14
+ import train.performance
15
+
16
+ try:
17
+ import hardcore # pylint:disable=E0401
18
+ except ModuleNotFoundError:
19
+ hardcore = None # pylint:disable=C0103
20
+
21
+
22
+ @utilo.saveme
23
+ def main():
24
+ args = create()
25
+ jobs = select(args)
26
+ if not jobs:
27
+ utilo.error('nothing todo')
28
+ return utilo.FAILURE
29
+ port = train.DEV_SERVER_PORT
30
+ if args['port']:
31
+ port = int(args['port'])
32
+ host = train.DEV_SERVER_URL
33
+ if args['host']:
34
+ host = args['host']
35
+ upload(
36
+ jobs=jobs,
37
+ port=port,
38
+ host=host,
39
+ )
40
+ return utilo.SUCCESS
41
+
42
+
43
+ def create():
44
+ hoverpowers = utilo.Parameter(
45
+ shortcut='p',
46
+ longcut='hoverpower',
47
+ message='hoverpower files',
48
+ )
49
+ hardcores = utilo.Parameter(
50
+ shortcut='hc',
51
+ longcut='hardcore',
52
+ message='hardcore files',
53
+ )
54
+ commands = [
55
+ hoverpowers,
56
+ hardcores,
57
+ utilo.NumberedParameter(
58
+ shortcut='n',
59
+ longcut='newest',
60
+ message='n newest files',
61
+ default=0,
62
+ ),
63
+ utilo.NumberedParameter(
64
+ shortcut='o',
65
+ longcut='oldest',
66
+ message='n oldest files',
67
+ default=0,
68
+ ),
69
+ utilo.Parameter(
70
+ longcut='host',
71
+ message='server url to publish',
72
+ ),
73
+ utilo.NumberedParameter(
74
+ longcut='port',
75
+ message='server port to publish',
76
+ default=train.DEV_SERVER_PORT,
77
+ ),
78
+ ]
79
+ parser = utilo.create_parser(
80
+ todo=commands,
81
+ config=utilo.ParserConfiguration.all_off(),
82
+ version=train.__version__,
83
+ )
84
+ parser.add_argument(
85
+ 'regex',
86
+ nargs='?',
87
+ help='select matching files or a single file.',
88
+ )
89
+ args = utilo.parse(parser)
90
+ return args
91
+
92
+
93
+ def select(args) -> list:
94
+ jobs = []
95
+ if hoverpowers := args.get('hoverpower'):
96
+ pages = utilo.parse_pages(hoverpowers, pagecount=len(hoverpower.PDF))
97
+ if pages is None:
98
+ jobs.extend(hoverpower.PDF)
99
+ else:
100
+ jobs.extend([hoverpower.PDF[page] for page in pages])
101
+ if hardcores := args.get('hardcore'):
102
+ pages = utilo.parse_pages(hardcores, pagecount=len(hardcore.ALL))
103
+ if pages is None:
104
+ jobs.extend(hardcore.ALL)
105
+ else:
106
+ jobs.extend([hardcore.ALL[page] for page in pages])
107
+ if number := args.get('newest'):
108
+ jobs.extend(newest(number))
109
+ if number := args.get('oldest'):
110
+ jobs.extend(oldest(number))
111
+ if regex := args.get('regex'):
112
+ if files := file_select(regex):
113
+ jobs.extend(files)
114
+ else:
115
+ jobs.extend(regex_select(regex=regex))
116
+ return jobs
117
+
118
+
119
+ def regex_select(regex: str) -> list:
120
+ """Select files via regex.
121
+
122
+ Use hoverpower and hardcore as source of selection.
123
+
124
+ >>> regex_select('bachelor032')
125
+ ['...', '...',...]
126
+ """
127
+ result = [
128
+ item for item in hoverpower.PDF # + hardcore.ALL # TODO: ENABLE LATER
129
+ if utilo.file_name(item).startswith(regex)
130
+ ]
131
+ return result
132
+
133
+
134
+ def file_select(path: str) -> list:
135
+ """Select file which is not part of corpus(hoverpower, hardcore).
136
+
137
+ This file is mostly selected via cli.
138
+
139
+ >>> file_select(__file__)
140
+ ['...pusher.py']
141
+ """
142
+ if not utilo.isfilepath(path):
143
+ return None
144
+ return [utilo.make_absolute(path)]
145
+
146
+
147
+ def upload(jobs, port, host):
148
+ utilo.log(f'publish to port: {port}')
149
+ utilo.log(f'publish to host: {host}')
150
+ for path in jobs:
151
+ utilo.log(f'upload: {path}')
152
+ train.performance.upload(
153
+ path,
154
+ server=host,
155
+ port=port,
156
+ public=True,
157
+ )
158
+
159
+
160
+ def newest(count: int = 10) -> list:
161
+ """\
162
+ >>> assert len(newest(10)) == 10
163
+ """
164
+ files = sorted(hoverpower.PDF, key=utilo.file_age)
165
+ result = files[0:count]
166
+ return result
167
+
168
+
169
+ def oldest(count: int = 10) -> list:
170
+ """\
171
+ >>> assert len(oldest(5)) == 5
172
+ """
173
+ files = sorted(hoverpower.PDF, key=utilo.file_age, reverse=True)
174
+ result = files[0:count]
175
+ return result
train/race.py ADDED
@@ -0,0 +1,93 @@
1
+ # =============================================================================
2
+ # C O P Y R I G H T
3
+ # -----------------------------------------------------------------------------
4
+ # Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
5
+ # This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
6
+ # use or distribution is an offensive act against international law and may
7
+ # be prosecuted under federal law. Its content is company confidential.
8
+ # =============================================================================
9
+
10
+ import sys
11
+
12
+ import configos
13
+ import hoverpower
14
+ import protoerror
15
+ import utilo
16
+
17
+ import train
18
+ import train.performance
19
+
20
+ FIRST = [
21
+ hoverpower.MASTER098_PDF,
22
+ hoverpower.MASTER116_PDF,
23
+ hoverpower.BACHELOR090_PDF,
24
+ hoverpower.HOME025_PDF,
25
+ ]
26
+
27
+ SECOND = [
28
+ hoverpower.MASTER110_PDF,
29
+ hoverpower.BACHELOR128_PDF,
30
+ hoverpower.MASTER031_PDF,
31
+ hoverpower.BACHELOR051_PDF,
32
+ ]
33
+
34
+ THIRD = [
35
+ hoverpower.MASTER072_PDF,
36
+ hoverpower.MASTER089_PDF,
37
+ hoverpower.MASTER098_PDF,
38
+ hoverpower.MASTER099_PDF,
39
+ ]
40
+
41
+
42
+ @utilo.saveme
43
+ def main():
44
+ commands = [
45
+ utilo.Flag(longcut='first'),
46
+ utilo.Flag(longcut='second'),
47
+ utilo.Flag(longcut='third'),
48
+ ]
49
+ parser = utilo.create_parser(
50
+ todo=commands,
51
+ config=utilo.ParserConfiguration(
52
+ outputparameter=False,
53
+ inputparameter=False,
54
+ prefix=False,
55
+ verboseflag=True,
56
+ ),
57
+ description='Determine performance for selected pdf package.',
58
+ version=train.__version__,
59
+ )
60
+ args = utilo.parse(parser)
61
+
62
+ first = args.get('first', False)
63
+ second = args.get('second', False)
64
+ third = args.get('third', False)
65
+
66
+ if first:
67
+ run(FIRST, start=4700)
68
+
69
+ if second:
70
+ run(SECOND, start=4710)
71
+
72
+ if third:
73
+ run(THIRD, start=4720)
74
+
75
+ if not any((first, second, third)):
76
+ parser.print_usage()
77
+ sys.exit(utilo.FAILURE)
78
+ sys.exit(utilo.SUCCESS)
79
+
80
+
81
+ def run(todo, start: int = 4700):
82
+ for port, path in enumerate(todo, start=start):
83
+ workspace = utilo.tmpdir(configos.tmp())
84
+ utilo.log(f'run performance for {path} in {workspace} on: {port}')
85
+ with utilo.profile(path):
86
+ ready = train.performance.run(workspace, path, port=port)
87
+ # load findings by page
88
+ findings = protoerror.findings_from_path(ready)
89
+ # global finding list
90
+ findings = utilo.flat([item.content for item in findings])
91
+ # TODO: REPLACE AFTER UPGRADING?
92
+ findings = utilo.unique(findings)
93
+ utilo.log(f'findings: {len(findings)}')