idmtools-platform-general 0.0.0.dev0__py3-none-any.whl → 0.0.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- idmtools_platform_file/__init__.py +18 -0
- idmtools_platform_file/assets/__init__.py +77 -0
- idmtools_platform_file/assets/_run.sh.jinja2 +47 -0
- idmtools_platform_file/assets/batch.sh.jinja2 +24 -0
- idmtools_platform_file/assets/run_simulation.sh +8 -0
- idmtools_platform_file/cli/__init__.py +5 -0
- idmtools_platform_file/cli/file.py +185 -0
- idmtools_platform_file/file_operations/__init__.py +4 -0
- idmtools_platform_file/file_operations/file_operations.py +298 -0
- idmtools_platform_file/file_operations/operations_interface.py +74 -0
- idmtools_platform_file/file_platform.py +288 -0
- idmtools_platform_file/platform_operations/__init__.py +5 -0
- idmtools_platform_file/platform_operations/asset_collection_operations.py +172 -0
- idmtools_platform_file/platform_operations/experiment_operations.py +314 -0
- idmtools_platform_file/platform_operations/json_metadata_operations.py +320 -0
- idmtools_platform_file/platform_operations/simulation_operations.py +212 -0
- idmtools_platform_file/platform_operations/suite_operations.py +243 -0
- idmtools_platform_file/platform_operations/utils.py +461 -0
- idmtools_platform_file/plugin_info.py +82 -0
- idmtools_platform_file/tools/__init__.py +4 -0
- idmtools_platform_file/tools/job_history.py +334 -0
- idmtools_platform_file/tools/status_report/__init__.py +4 -0
- idmtools_platform_file/tools/status_report/status_report.py +222 -0
- idmtools_platform_file/tools/status_report/utils.py +159 -0
- idmtools_platform_general-0.0.3.dist-info/METADATA +81 -0
- idmtools_platform_general-0.0.3.dist-info/RECORD +35 -0
- idmtools_platform_general-0.0.3.dist-info/entry_points.txt +6 -0
- idmtools_platform_general-0.0.3.dist-info/licenses/LICENSE.TXT +3 -0
- idmtools_platform_general-0.0.3.dist-info/top_level.txt +2 -0
- idmtools_platform_process/__init__.py +17 -0
- idmtools_platform_process/platform_operations/__init__.py +5 -0
- idmtools_platform_process/platform_operations/experiment_operations.py +53 -0
- idmtools_platform_process/plugin_info.py +80 -0
- idmtools_platform_process/process_platform.py +52 -0
- idmtools_platform_general/__init__.py +0 -8
- idmtools_platform_general-0.0.0.dev0.dist-info/METADATA +0 -41
- idmtools_platform_general-0.0.0.dev0.dist-info/RECORD +0 -5
- idmtools_platform_general-0.0.0.dev0.dist-info/top_level.txt +0 -1
- {idmtools_platform_general-0.0.0.dev0.dist-info → idmtools_platform_general-0.0.3.dist-info}/WHEEL +0 -0
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Utility functions used by File Platform.
|
|
3
|
+
|
|
4
|
+
Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
|
|
5
|
+
"""
|
|
6
|
+
import os
|
|
7
|
+
import shutil
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from logging import getLogger
|
|
10
|
+
from typing import Dict, Tuple, TYPE_CHECKING
|
|
11
|
+
from idmtools.core import ItemType
|
|
12
|
+
|
|
13
|
+
if TYPE_CHECKING: # pragma: no cover
|
|
14
|
+
from idmtools.entities.iplatform import IPlatform
|
|
15
|
+
|
|
16
|
+
user_logger = getLogger('user')
|
|
17
|
+
|
|
18
|
+
EXPERIMENT_FILES = ['stdout.txt', 'stderr.txt']
|
|
19
|
+
SIMULATION_FILES = ['stdout.txt', 'stderr.txt', 'job_status.txt']
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def get_latest_experiment(platform: 'IPlatform') -> Dict:
|
|
23
|
+
"""
|
|
24
|
+
Find the latest experiment.
|
|
25
|
+
Args:
|
|
26
|
+
platform: Platform
|
|
27
|
+
Returns:
|
|
28
|
+
Dictionary with experiment info
|
|
29
|
+
"""
|
|
30
|
+
try:
|
|
31
|
+
last_dir = max(Path(platform.job_directory).glob('*/'), key=os.path.getmtime)
|
|
32
|
+
if last_dir.name.startswith("s_"):
|
|
33
|
+
batch_dir = max(Path(last_dir).glob('*/batch.sh'), key=os.path.getmtime)
|
|
34
|
+
elif last_dir.name.startswith("e_"):
|
|
35
|
+
batch_dir = max(Path(last_dir).glob('batch.sh'), key=os.path.getmtime)
|
|
36
|
+
exp_dir = Path(batch_dir).parent
|
|
37
|
+
exp_id = exp_dir.name
|
|
38
|
+
|
|
39
|
+
r = dict(experiment_id=exp_id, experiment_directory=str(exp_dir),
|
|
40
|
+
job_directory=str(platform.job_directory))
|
|
41
|
+
return r
|
|
42
|
+
except:
|
|
43
|
+
raise FileNotFoundError("Could not find the last Experiment")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def check_status(platform: 'IPlatform', exp_id: str = None, display: bool = False) -> None:
|
|
47
|
+
"""
|
|
48
|
+
List simulations status.
|
|
49
|
+
Args:
|
|
50
|
+
platform: Platform
|
|
51
|
+
exp_id: experiment id
|
|
52
|
+
display: True/False
|
|
53
|
+
Returns:
|
|
54
|
+
None
|
|
55
|
+
"""
|
|
56
|
+
if exp_id is None:
|
|
57
|
+
exp_dic = get_latest_experiment(platform)
|
|
58
|
+
exp_id = exp_dic['experiment_id']
|
|
59
|
+
|
|
60
|
+
_exp = platform.get_item(exp_id, ItemType.EXPERIMENT)
|
|
61
|
+
|
|
62
|
+
_pending = []
|
|
63
|
+
_running = []
|
|
64
|
+
_failed = []
|
|
65
|
+
_succeeded = []
|
|
66
|
+
_simulations = _exp.simulations
|
|
67
|
+
for sim in _simulations:
|
|
68
|
+
sim_dir = platform.get_directory(sim)
|
|
69
|
+
job_status_path = sim_dir.joinpath("job_status.txt")
|
|
70
|
+
if not job_status_path.exists():
|
|
71
|
+
_pending.append(f" {sim.id}")
|
|
72
|
+
else:
|
|
73
|
+
status = open(job_status_path).read().strip()
|
|
74
|
+
if status == '0':
|
|
75
|
+
_succeeded.append(f" {sim.id}")
|
|
76
|
+
elif status == '100':
|
|
77
|
+
_running.append(f" {sim.id}")
|
|
78
|
+
elif status == '-1':
|
|
79
|
+
_failed.append(f" {sim.id}")
|
|
80
|
+
else:
|
|
81
|
+
_running.append(f" {sim.id}")
|
|
82
|
+
|
|
83
|
+
user_logger.info(f'\nExperiment Directory: \n{str(platform.get_directory(_exp))}')
|
|
84
|
+
|
|
85
|
+
# Output report
|
|
86
|
+
user_logger.info(f"\n{'Simulation Count: '.ljust(20)} {len(_simulations)}\n")
|
|
87
|
+
|
|
88
|
+
user_logger.info(f'SUCCEEDED ({len(_succeeded)})')
|
|
89
|
+
if display:
|
|
90
|
+
user_logger.info('\n'.join(_succeeded))
|
|
91
|
+
|
|
92
|
+
user_logger.info(f'FAILED ({len(_failed)})')
|
|
93
|
+
if display:
|
|
94
|
+
user_logger.info('\n'.join(_failed))
|
|
95
|
+
|
|
96
|
+
user_logger.info(f'RUNNING ({len(_running)})')
|
|
97
|
+
if display:
|
|
98
|
+
user_logger.info('\n'.join(_running))
|
|
99
|
+
|
|
100
|
+
user_logger.info(f'PENDING ({len(_pending)})')
|
|
101
|
+
if display:
|
|
102
|
+
user_logger.info('\n'.join(_pending))
|
|
103
|
+
|
|
104
|
+
if _exp.status is None:
|
|
105
|
+
user_logger.info(f'\nExperiment Status: {None}')
|
|
106
|
+
else:
|
|
107
|
+
user_logger.info(f'\nExperiment Status: {_exp.status.name}\n')
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def clear_history(platform: 'IPlatform', exp_id: str = None, sim_id: Tuple = None, remove_list=None) -> None:
|
|
111
|
+
"""
|
|
112
|
+
Clear the history files generated from running experiment/simulation.
|
|
113
|
+
Args:
|
|
114
|
+
platform: Platform
|
|
115
|
+
exp_id: experiment id
|
|
116
|
+
sim_id: simulation id
|
|
117
|
+
remove_list: list of files/folders
|
|
118
|
+
Returns:
|
|
119
|
+
None
|
|
120
|
+
"""
|
|
121
|
+
|
|
122
|
+
def _clear_simulation(sim_id, remove_list):
|
|
123
|
+
"""
|
|
124
|
+
Delete generated output files for simulation.
|
|
125
|
+
Args:
|
|
126
|
+
sim_id: simulation id
|
|
127
|
+
remove_list: extra files to be deleted
|
|
128
|
+
Returns:
|
|
129
|
+
None
|
|
130
|
+
"""
|
|
131
|
+
sim_dir = platform.get_directory_by_id(sim_id, ItemType.SIMULATION)
|
|
132
|
+
for fi in SIMULATION_FILES + list(remove_list):
|
|
133
|
+
if sim_dir.joinpath(fi).exists():
|
|
134
|
+
if sim_dir.joinpath(fi).is_dir():
|
|
135
|
+
shutil.rmtree(sim_dir.joinpath(fi))
|
|
136
|
+
else:
|
|
137
|
+
sim_dir.joinpath(fi).unlink(missing_ok=True)
|
|
138
|
+
|
|
139
|
+
if sim_id is not None and len(sim_id) > 0:
|
|
140
|
+
for sid in sim_id:
|
|
141
|
+
_clear_simulation(sid, remove_list)
|
|
142
|
+
elif exp_id is not None:
|
|
143
|
+
exp = platform.get_item(exp_id, ItemType.EXPERIMENT)
|
|
144
|
+
exp_dir = platform.get_directory(exp)
|
|
145
|
+
|
|
146
|
+
# Delete generated files from experiment past run
|
|
147
|
+
for fi in EXPERIMENT_FILES:
|
|
148
|
+
if exp_dir.joinpath(fi).exists():
|
|
149
|
+
if exp_dir.joinpath(fi).is_dir():
|
|
150
|
+
shutil.rmtree(exp_dir.joinpath(fi))
|
|
151
|
+
else:
|
|
152
|
+
exp_dir.joinpath(fi).unlink(missing_ok=True)
|
|
153
|
+
|
|
154
|
+
# Delete generated files from simulations
|
|
155
|
+
for sim in exp.simulations:
|
|
156
|
+
_clear_simulation(sim.id, remove_list)
|
|
157
|
+
else:
|
|
158
|
+
user_logger.warning("Must provide exp_id or sim_id!")
|
|
159
|
+
exit(-1)
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: idmtools_platform_general
|
|
3
|
+
Version: 0.0.3
|
|
4
|
+
Summary: General platform for IDM-Tools
|
|
5
|
+
Author-email: Zhaowei Du <zdu@idmod.org>, Sharon Chen <shchen@idmod.org>, Clinton Collins <ccollins@idmod.org>, Jen Schripsema <jschripsema@idmod.org>, Ross Carter <rcarter@idmod.org>
|
|
6
|
+
Project-URL: Homepage, https://github.com/InstituteforDiseaseModeling/idmtools
|
|
7
|
+
Keywords: modeling,IDM
|
|
8
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Requires-Python: >=3.8
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE.TXT
|
|
16
|
+
Requires-Dist: idmtools<1.0.0,>=0.0.0
|
|
17
|
+
Requires-Dist: idmtools_cli<1.0.0,>=0.0.0
|
|
18
|
+
Provides-Extra: test
|
|
19
|
+
Requires-Dist: idmtools[test]; extra == "test"
|
|
20
|
+
Requires-Dist: idmtools_test; extra == "test"
|
|
21
|
+
Provides-Extra: packaging
|
|
22
|
+
Requires-Dist: flake8; extra == "packaging"
|
|
23
|
+
Requires-Dist: coverage; extra == "packaging"
|
|
24
|
+
Requires-Dist: bump2version; extra == "packaging"
|
|
25
|
+
Requires-Dist: twine; extra == "packaging"
|
|
26
|
+
Requires-Dist: natsort; extra == "packaging"
|
|
27
|
+
Dynamic: license-file
|
|
28
|
+
|
|
29
|
+

|
|
30
|
+
|
|
31
|
+
# idmtools-platform-general
|
|
32
|
+
|
|
33
|
+
<!-- START doctoc generated TOC please keep comment here to allow auto update -->
|
|
34
|
+
<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
|
|
35
|
+
**Table of Contents**
|
|
36
|
+
|
|
37
|
+
- [Installing](#installing)
|
|
38
|
+
- [Development Tips](#development-tips)
|
|
39
|
+
- [Use Cases](#use-cases)
|
|
40
|
+
- [Feature Roadmap](#feature-roadmap)
|
|
41
|
+
|
|
42
|
+
<!-- END doctoc generated TOC please keep comment here to allow auto update -->
|
|
43
|
+
|
|
44
|
+
## Installing
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install idmtools-platform-general --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
# Development Tips
|
|
51
|
+
|
|
52
|
+
There is a Makefile file available for most common development tasks. Here is a list of commands
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
clean - Clean up temproary files
|
|
56
|
+
lint - Lint package and tests
|
|
57
|
+
test - Run All tests
|
|
58
|
+
coverage - Run tests and generate coverage report that is shown in browser
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
On Windows, you can use `pymake` instead of `make`
|
|
62
|
+
|
|
63
|
+
# Use Cases
|
|
64
|
+
|
|
65
|
+
* Testing
|
|
66
|
+
* Test core functionality
|
|
67
|
+
* Performance Testing
|
|
68
|
+
* Integration with other systems
|
|
69
|
+
* Other HPC Systems
|
|
70
|
+
* Local Executions Systems
|
|
71
|
+
* Jupyter notebooks
|
|
72
|
+
* Basis for future local platforms
|
|
73
|
+
* Process
|
|
74
|
+
* Thread
|
|
75
|
+
* Dask
|
|
76
|
+
* Asyncio
|
|
77
|
+
|
|
78
|
+
# Feature Roadmap
|
|
79
|
+
|
|
80
|
+
* First Version
|
|
81
|
+
* Support for basic provisioning on a linux filesystem
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
idmtools_platform_file/__init__.py,sha256=XunlB1fSB_5Ethli_4sHp2u7Ow_TB6hZy1xwTmDMV34,496
|
|
2
|
+
idmtools_platform_file/file_platform.py,sha256=wRdxPj9gj7dDCpUpIT33y1jarCiZTX2tmdxYHPFVMT8,11472
|
|
3
|
+
idmtools_platform_file/plugin_info.py,sha256=nINJUoCO6kSJM4aMBTSCIQ0ErXxm5pF5hAoVi-BD1xM,2193
|
|
4
|
+
idmtools_platform_file/assets/__init__.py,sha256=WkwHwr8LMMKTHb6wgtvBtFPCK-QTsPvPsdovbhK6EXs,2748
|
|
5
|
+
idmtools_platform_file/assets/_run.sh.jinja2,sha256=Gq0qWqexM9nWXnjmNwNWyACxdgi-68Rv5BYY4_ms6yU,1386
|
|
6
|
+
idmtools_platform_file/assets/batch.sh.jinja2,sha256=yDS2gGPChN4VobCH_8Y8Fva0yb8fK25F368RW6SSF7U,1011
|
|
7
|
+
idmtools_platform_file/assets/run_simulation.sh,sha256=wpiGA6GaHxPBj65G79xoLe03Osm-F8IgzGGT5hmas9w,179
|
|
8
|
+
idmtools_platform_file/cli/__init__.py,sha256=KdQ3TSX0BvUAeTDGFDb6bZlQjVmBIBKR23LEJ-yD888,110
|
|
9
|
+
idmtools_platform_file/cli/file.py,sha256=CRcXEkE5D5u7CZ3GjiCCLrUek_oMlHKLSKjGUnr34dw,6485
|
|
10
|
+
idmtools_platform_file/file_operations/__init__.py,sha256=lSgyWUnNmdb5zkHhMqoJjTh3llWO4XWy0K7pPVdJQZM,85
|
|
11
|
+
idmtools_platform_file/file_operations/file_operations.py,sha256=vh9dAlnx44xc0BRiWtY4C1U5mUq0lJXHtbDk3BT6wlM,11127
|
|
12
|
+
idmtools_platform_file/file_operations/operations_interface.py,sha256=LuTIRIm9ANHbXDYmV3QK7m5QCZC5rCuzQKYRA_7ee3A,2217
|
|
13
|
+
idmtools_platform_file/platform_operations/__init__.py,sha256=puHgNM6soGldqQgeFZXfAT3vcze7aZ2F-RJgvhYyRIk,134
|
|
14
|
+
idmtools_platform_file/platform_operations/asset_collection_operations.py,sha256=3zsWi26jGVP0hegv8E5rXMA-U9Lpvyj5DPrE3ELNd9M,7052
|
|
15
|
+
idmtools_platform_file/platform_operations/experiment_operations.py,sha256=YxyHp2htLRsBWHFrCU858ICOFETsosDFc_tdzuGcjg8,11328
|
|
16
|
+
idmtools_platform_file/platform_operations/json_metadata_operations.py,sha256=CL14PbEQc9PL5jkrNTaYf5iKlY44hCOcbUnbQgKek_I,12922
|
|
17
|
+
idmtools_platform_file/platform_operations/simulation_operations.py,sha256=A760UF4nXe0S7sbhdjRwX2VfMf1rWOS_rTIy34lgWWE,7508
|
|
18
|
+
idmtools_platform_file/platform_operations/suite_operations.py,sha256=biIE3bwUKvEbjwovMGMmAYNTJOQS16JuAeOebnqCQuk,8039
|
|
19
|
+
idmtools_platform_file/platform_operations/utils.py,sha256=znxkk0lqvlwTxYM0wgZUdUv8sIMp0frTCSPr2xWtL-Y,14662
|
|
20
|
+
idmtools_platform_file/tools/__init__.py,sha256=QeEbFLLYcsehsfXfPbPqay_d96tLkPpql_Ahz9q226g,104
|
|
21
|
+
idmtools_platform_file/tools/job_history.py,sha256=zAeUXvuEEcje7eZ3Iyh_GXvCjv5tQfCodkby4Gn-ZXc,10473
|
|
22
|
+
idmtools_platform_file/tools/status_report/__init__.py,sha256=QeEbFLLYcsehsfXfPbPqay_d96tLkPpql_Ahz9q226g,104
|
|
23
|
+
idmtools_platform_file/tools/status_report/status_report.py,sha256=J5wYqm9QN_0rBwSm6p76ubq7GpfvUVLJxK693aJb558,9210
|
|
24
|
+
idmtools_platform_file/tools/status_report/utils.py,sha256=AwjF2GYXSsPZxD1nJ6EWdGlb6cZCDaB9tJU6_UtXkXY,5320
|
|
25
|
+
idmtools_platform_general-0.0.3.dist-info/licenses/LICENSE.TXT,sha256=l9S8Ydr_LcejxKoqK8191ZAOsmVX-nJLSPoLKZDUgcg,197
|
|
26
|
+
idmtools_platform_process/__init__.py,sha256=niwaySWSnj8OKTDnUW-4CGwnAhBo29wjg0JfsIB9pRU,498
|
|
27
|
+
idmtools_platform_process/plugin_info.py,sha256=gn0i8s8PDzQvLNvZV3ToxDyPQNZt88Uzxjzw_TbUs1o,2240
|
|
28
|
+
idmtools_platform_process/process_platform.py,sha256=nJhug0ktXz8Maw1AFsbePdkl25X4CO6WqbCv-Z0WxvU,1931
|
|
29
|
+
idmtools_platform_process/platform_operations/__init__.py,sha256=T9ejDi-dfrsubnDaY5q-hTUqnQ1xmmOJiTqiNhZolK8,137
|
|
30
|
+
idmtools_platform_process/platform_operations/experiment_operations.py,sha256=R1tHYkRBgwnj9XC8ynQSYvkGa9odr5SCTHTOv_akP6U,1802
|
|
31
|
+
idmtools_platform_general-0.0.3.dist-info/METADATA,sha256=K41iRgrorS4SLhV_kWXWk1SLoBdxqRRevLKxI2zzO1U,2645
|
|
32
|
+
idmtools_platform_general-0.0.3.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
33
|
+
idmtools_platform_general-0.0.3.dist-info/entry_points.txt,sha256=82B17eKZNcf94XgU-VV9hsJT_5boZUiA_6hJ9W3xky0,273
|
|
34
|
+
idmtools_platform_general-0.0.3.dist-info/top_level.txt,sha256=btIl858s0zexW_EYzxS3iDjq4tGn97QrZqDSdVK1LxU,49
|
|
35
|
+
idmtools_platform_general-0.0.3.dist-info/RECORD,,
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
[idmtools_cli.cli_plugins]
|
|
2
|
+
file = idmtools_platform_file.cli.file:file
|
|
3
|
+
|
|
4
|
+
[idmtools_platform]
|
|
5
|
+
idmtools_platform_file = idmtools_platform_file.plugin_info:FilePlatformSpecification
|
|
6
|
+
idmtools_platform_process = idmtools_platform_process.plugin_info:ProcessPlatformSpecification
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""
|
|
2
|
+
idmtools process platform.
|
|
3
|
+
|
|
4
|
+
Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
|
|
5
|
+
"""
|
|
6
|
+
# flake8: noqa F821
|
|
7
|
+
try:
|
|
8
|
+
from importlib.metadata import version, PackageNotFoundError
|
|
9
|
+
except ImportError:
|
|
10
|
+
# Python < 3.8
|
|
11
|
+
from importlib_metadata import version, PackageNotFoundError
|
|
12
|
+
|
|
13
|
+
try:
|
|
14
|
+
__version__ = version("idmtools-platform-general") # Use your actual package name
|
|
15
|
+
except PackageNotFoundError:
|
|
16
|
+
# Package not installed, use fallback
|
|
17
|
+
__version__ = "0.0.0+unknown"
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Here we implement the ProcessPlatform experiment operations.
|
|
3
|
+
|
|
4
|
+
Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
|
|
5
|
+
"""
|
|
6
|
+
import os
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from typing import TYPE_CHECKING
|
|
9
|
+
from idmtools.entities.experiment import Experiment
|
|
10
|
+
from idmtools_platform_file.platform_operations.experiment_operations import FilePlatformExperimentOperations
|
|
11
|
+
from logging import getLogger
|
|
12
|
+
|
|
13
|
+
user_logger = getLogger('user')
|
|
14
|
+
|
|
15
|
+
if TYPE_CHECKING:
|
|
16
|
+
from idmtools_platform_process.process_platform import ProcessPlatform
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class ProcessPlatformExperimentOperations(FilePlatformExperimentOperations):
|
|
21
|
+
"""
|
|
22
|
+
Experiment Operations for Process Platform.
|
|
23
|
+
"""
|
|
24
|
+
platform: 'ProcessPlatform'
|
|
25
|
+
|
|
26
|
+
def platform_run_item(self, experiment: Experiment, dry_run: bool = False, **kwargs):
|
|
27
|
+
"""
|
|
28
|
+
Run experiment.
|
|
29
|
+
Args:
|
|
30
|
+
experiment: idmtools Experiment
|
|
31
|
+
dry_run: True/False
|
|
32
|
+
kwargs: keyword arguments used to expand functionality
|
|
33
|
+
Returns:
|
|
34
|
+
None
|
|
35
|
+
"""
|
|
36
|
+
# Ensure parent
|
|
37
|
+
super().platform_run_item(experiment, **kwargs)
|
|
38
|
+
if not dry_run:
|
|
39
|
+
self.platform.submit_job(experiment, **kwargs)
|
|
40
|
+
|
|
41
|
+
def post_run_item(self, experiment: Experiment, **kwargs):
|
|
42
|
+
"""
|
|
43
|
+
Trigger right after commissioning experiment on platform.
|
|
44
|
+
|
|
45
|
+
Args:
|
|
46
|
+
experiment: Experiment just commissioned
|
|
47
|
+
kwargs: keyword arguments used to expand functionality
|
|
48
|
+
Returns:
|
|
49
|
+
None
|
|
50
|
+
"""
|
|
51
|
+
super().post_run_item(experiment, **kwargs)
|
|
52
|
+
user_logger.info(
|
|
53
|
+
f'\nYou may try the following command to check simulations running status: \n idmtools file {os.path.abspath(self.platform.job_directory)} status --exp-id {experiment.id}')
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""
|
|
2
|
+
idmtools process platform plugin definition.
|
|
3
|
+
|
|
4
|
+
Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
|
|
5
|
+
"""
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Type, Dict
|
|
8
|
+
from idmtools.entities.iplatform import IPlatform
|
|
9
|
+
from idmtools.registry.platform_specification import example_configuration_impl, get_platform_impl, \
|
|
10
|
+
get_platform_type_impl, PlatformSpecification
|
|
11
|
+
from idmtools.registry.plugin_specification import get_description_impl
|
|
12
|
+
|
|
13
|
+
PROCESS_EXAMPLE_CONFIG = """
|
|
14
|
+
[Process]
|
|
15
|
+
job_directory = /data
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class ProcessPlatformSpecification(PlatformSpecification):
|
|
20
|
+
"""
|
|
21
|
+
Process Platform Specification definition.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
@get_description_impl
|
|
25
|
+
def get_description(self) -> str:
|
|
26
|
+
"""
|
|
27
|
+
Retrieve description.
|
|
28
|
+
"""
|
|
29
|
+
return "Provides access to the Process Platform to IDM Tools"
|
|
30
|
+
|
|
31
|
+
@get_platform_impl
|
|
32
|
+
def get(self, **configuration) -> IPlatform:
|
|
33
|
+
"""
|
|
34
|
+
Build our process platform from the passed in configuration object.
|
|
35
|
+
|
|
36
|
+
We do our import of platform here to avoid any weirdness
|
|
37
|
+
Args:
|
|
38
|
+
configuration:
|
|
39
|
+
Returns:
|
|
40
|
+
IPlatform
|
|
41
|
+
"""
|
|
42
|
+
from process_platform import ProcessPlatform
|
|
43
|
+
return ProcessPlatform(**configuration)
|
|
44
|
+
|
|
45
|
+
@example_configuration_impl
|
|
46
|
+
def example_configuration(self):
|
|
47
|
+
"""
|
|
48
|
+
Retrieve example configuration.
|
|
49
|
+
"""
|
|
50
|
+
return PROCESS_EXAMPLE_CONFIG
|
|
51
|
+
|
|
52
|
+
@get_platform_type_impl
|
|
53
|
+
def get_type(self) -> Type['ProcessPlatform']: # noqa: F821
|
|
54
|
+
"""
|
|
55
|
+
Get type.
|
|
56
|
+
Returns:
|
|
57
|
+
Type
|
|
58
|
+
"""
|
|
59
|
+
from idmtools_platform_process.process_platform import ProcessPlatform
|
|
60
|
+
return ProcessPlatform
|
|
61
|
+
|
|
62
|
+
def get_version(self) -> str:
|
|
63
|
+
"""
|
|
64
|
+
Returns the version of the plugin.
|
|
65
|
+
Returns:
|
|
66
|
+
Plugin Version
|
|
67
|
+
"""
|
|
68
|
+
from idmtools_platform_process import __version__
|
|
69
|
+
return __version__
|
|
70
|
+
|
|
71
|
+
def get_configuration_aliases(self) -> Dict[str, Dict]:
|
|
72
|
+
"""
|
|
73
|
+
Provides configuration aliases that exist in PROCESS.
|
|
74
|
+
"""
|
|
75
|
+
config_aliases = dict(
|
|
76
|
+
PROCESS=dict(
|
|
77
|
+
job_directory=str(Path.home())
|
|
78
|
+
)
|
|
79
|
+
)
|
|
80
|
+
return config_aliases
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Here we implement the ProcessPlatform object.
|
|
3
|
+
|
|
4
|
+
Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
|
|
5
|
+
"""
|
|
6
|
+
import platform
|
|
7
|
+
import subprocess
|
|
8
|
+
from typing import Union, Any
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from idmtools.entities.experiment import Experiment
|
|
11
|
+
from idmtools.entities.simulation import Simulation
|
|
12
|
+
from idmtools_platform_file.file_platform import FilePlatform
|
|
13
|
+
from idmtools_platform_process.platform_operations.experiment_operations import ProcessPlatformExperimentOperations
|
|
14
|
+
from logging import getLogger
|
|
15
|
+
|
|
16
|
+
user_logger = getLogger('user')
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(repr=False)
|
|
20
|
+
class ProcessPlatform(FilePlatform):
|
|
21
|
+
"""
|
|
22
|
+
Process Platform definition.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def __post_init__(self):
|
|
26
|
+
super().__post_init__()
|
|
27
|
+
self._experiments = ProcessPlatformExperimentOperations(platform=self)
|
|
28
|
+
|
|
29
|
+
def submit_job(self, item: Union[Experiment, Simulation], **kwargs) -> Any:
|
|
30
|
+
"""
|
|
31
|
+
Submit a Process job.
|
|
32
|
+
Args:
|
|
33
|
+
item: idmtools Experiment or Simulation
|
|
34
|
+
kwargs: keyword arguments used to expand functionality
|
|
35
|
+
Returns:
|
|
36
|
+
Any
|
|
37
|
+
"""
|
|
38
|
+
if platform.system() in ["Windows"]:
|
|
39
|
+
user_logger.warning(
|
|
40
|
+
"\n/!\\ WARNING: The current ProcessPlatform only support running Experiment/Simulation on Linux!")
|
|
41
|
+
exit(-1)
|
|
42
|
+
|
|
43
|
+
if isinstance(item, Experiment):
|
|
44
|
+
working_directory = self.get_directory(item)
|
|
45
|
+
result = subprocess.run(['bash', 'batch.sh'], stdout=subprocess.PIPE, cwd=str(working_directory))
|
|
46
|
+
r = result.stdout.decode('utf-8').strip()
|
|
47
|
+
return r
|
|
48
|
+
elif isinstance(item, Simulation):
|
|
49
|
+
raise NotImplementedError("submit_job directly for simulation is not implemented on ProcessPlatform.")
|
|
50
|
+
else:
|
|
51
|
+
raise NotImplementedError(
|
|
52
|
+
f"Submit job is not implemented for {item.__class__.__name__} on ProcessPlatform.")
|
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: idmtools-platform-general
|
|
3
|
-
Version: 0.0.0.dev0
|
|
4
|
-
Summary: Placeholder package for idmtools-platform-general - reserved for future use
|
|
5
|
-
Author-email: IDM <idmtools@idmod.org>
|
|
6
|
-
License: Proprietary
|
|
7
|
-
Project-URL: Homepage, https://github.com/InstituteforDiseaseModeling/idmtools
|
|
8
|
-
Project-URL: Bug Tracker, https://github.com/InstituteforDiseaseModeling/idmtools/issues
|
|
9
|
-
Keywords: placeholder,reserved,idmtools
|
|
10
|
-
Classifier: Development Status :: 1 - Planning
|
|
11
|
-
Classifier: Intended Audience :: Developers
|
|
12
|
-
Classifier: Programming Language :: Python :: 3
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
-
Requires-Python: >=3.8
|
|
19
|
-
Description-Content-Type: text/markdown
|
|
20
|
-
|
|
21
|
-
# idmtools-platform-general
|
|
22
|
-
|
|
23
|
-
**PLACEHOLDER PACKAGE - NOT FOR USE**
|
|
24
|
-
|
|
25
|
-
This package is a placeholder to reserve the name `idmtools-platform-general` on PyPI.
|
|
26
|
-
|
|
27
|
-
The actual package implementation will be released in the future.
|
|
28
|
-
|
|
29
|
-
## Purpose
|
|
30
|
-
|
|
31
|
-
This placeholder ensures the package name is reserved and prevents name squatting.
|
|
32
|
-
|
|
33
|
-
## Status
|
|
34
|
-
|
|
35
|
-
This is version 0.0.0.dev0 - a minimal placeholder release.
|
|
36
|
-
|
|
37
|
-
## Future Release
|
|
38
|
-
|
|
39
|
-
The full implementation of idmtools-platform-general will be published when ready.
|
|
40
|
-
|
|
41
|
-
For more information, visit: https://github.com/InstituteforDiseaseModeling/idmtools
|
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
idmtools_platform_general/__init__.py,sha256=VlY2cLEaJwVvzxNuyNQ-nEKZXHuyPQzlWB9EtjLvafM,196
|
|
2
|
-
idmtools_platform_general-0.0.0.dev0.dist-info/METADATA,sha256=7w5TcccWsEEYWv2iQBumUWnPgKrrC7ml6NjVZ05Pv_o,1503
|
|
3
|
-
idmtools_platform_general-0.0.0.dev0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
4
|
-
idmtools_platform_general-0.0.0.dev0.dist-info/top_level.txt,sha256=VkbH4fEYe23vDnQhvmq6o_ecE658Cr59Oahz5GMSmMI,26
|
|
5
|
-
idmtools_platform_general-0.0.0.dev0.dist-info/RECORD,,
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
idmtools_platform_general
|
{idmtools_platform_general-0.0.0.dev0.dist-info → idmtools_platform_general-0.0.3.dist-info}/WHEEL
RENAMED
|
File without changes
|