cloudlab 1.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
cloudlab/__init__.py ADDED
@@ -0,0 +1 @@
1
+ # this file is intentionally empty
@@ -0,0 +1,294 @@
1
+ import argparse
2
+ import importlib.resources
3
+ import json
4
+ import logging
5
+ import os
6
+ import os.path
7
+ import re
8
+ import shutil
9
+ import stat
10
+ import subprocess
11
+ import sys
12
+ import time
13
+
14
+ import jinja2
15
+ import yaml
16
+
17
+ CLOUDFORMATION_TEMPLATE_NAME = 'cf.yaml'
18
+ CONFIG_FILE_NAME = 'cloudlab_config.yaml'
19
+ config = None # config is global
20
+
21
+
22
+ def print_sample():
23
+ sample = importlib.resources.files('cloudlab').joinpath('resources/cloudlab_config.yaml').read_text()
24
+ print(sample)
25
+
26
+
27
+ def run():
28
+ global config
29
+
30
+ parser = argparse.ArgumentParser(prog='cloudlab', description='Declarative lab environment builder for AWS')
31
+ parser.add_argument("command", choices=['mkenv', 'rmenv', 'update', 'sample'])
32
+ parser.add_argument("environment", nargs="?", default="cloudlab", help='A unique name for the environment')
33
+ parser.add_argument("--plan", default='aws_with_subnets', help='The name of the template to use when creating this environment')
34
+ parser.add_argument("--no-provision", dest='provision', action='store_false', help='Generate the CloudFormation template but do not provision the environment')
35
+
36
+ args = parser.parse_args()
37
+
38
+ if args.command == 'sample':
39
+ print_sample()
40
+ sys.exit(0)
41
+
42
+ if not os.path.isfile(CONFIG_FILE_NAME):
43
+ sys.exit(f'Exiting because the configuration file is not present. For a starter configuration file, run "cloudlab sample > {CONFIG_FILE_NAME}"')
44
+
45
+ with open(CONFIG_FILE_NAME, 'r') as config_file:
46
+ config = yaml.safe_load(config_file)
47
+
48
+ envdir = args.environment
49
+ command = args.command
50
+ template = f'{args.plan}.yaml.j2'
51
+ provision = args.provision
52
+
53
+ logging.basicConfig(format='%(asctime)s:%(message)s', level=logging.DEBUG)
54
+
55
+ # validate that the environment directory, which is also used as an AWS CloudFormation stack name, is valid
56
+ valid_stack_name_re = r"[a-zA-Z][-a-zA-Z0-9]*"
57
+ if re.fullmatch(valid_stack_name_re, envdir) is None:
58
+ sys.exit(f'"{envdir}" is not a valid environment name. Because it used as an AWS CloudFormation stack name, '
59
+ f'it must match the following regular expression: "{valid_stack_name_re}"')
60
+
61
+ if command == 'mkenv':
62
+ mkenv(envdir, False, template, provision)
63
+ elif command == 'rmenv':
64
+ rmenv(envdir)
65
+ elif command == 'update':
66
+ mkenv(envdir, True, template, provision)
67
+
68
+
69
+ def rmenv(envdir):
70
+ if envdir.endswith('/'):
71
+ envdir = envdir[0:-1]
72
+
73
+ envname = os.path.basename(envdir)
74
+
75
+ runaws('aws cloudformation delete-stack --stack-name={}'.format(envname))
76
+ logging.info("Deleted cloud formation stack: %s", envname)
77
+
78
+ runaws('aws ec2 delete-key-pair --key-name={}'.format(envname))
79
+ logging.info('Deleted the AWS key pair named %s.', envname + '.pem')
80
+
81
+ shutil.rmtree(envdir, ignore_errors=True)
82
+ logging.info('Removed cloudlab environment: %s.', envdir)
83
+
84
+
85
+ def mkenv(envdir, update, template, provision):
86
+ global config
87
+
88
+ # retrieve the template file
89
+ j2loader = jinja2.PackageLoader('cloudlab', 'plans')
90
+ j2env = jinja2.Environment(loader=j2loader, lstrip_blocks=True, trim_blocks=True)
91
+ try:
92
+ j2_template = j2env.get_template(template)
93
+ except jinja2.exceptions.TemplateNotFound:
94
+ sys.exit(f'cloudlab template not found: {template}')
95
+
96
+ # convert private_ip_suffixes to private_ip_addresses for each server
97
+ for subnet in config['subnets']:
98
+ for group in subnet['servers']:
99
+ private_ip_addresses = []
100
+ for suffix in group['private_ip_suffixes']:
101
+ private_ip_addresses.append( make_ip(subnet['cidr'], suffix))
102
+
103
+ group['private_ip_addresses'] = private_ip_addresses
104
+
105
+ # create the directory
106
+ if envdir.endswith('/'):
107
+ envdir = envdir[0:-1]
108
+
109
+ envname = os.path.basename(envdir)
110
+
111
+ if update:
112
+ if not os.path.exists(envdir):
113
+ sys.exit('Environment directory does not exist. Exiting.')
114
+ # else:
115
+ # shutil.rmtree(envdir, ignore_errors=True)
116
+ # logging.info('Removed cloudlab environment directory: %s.', envdir)
117
+
118
+ else:
119
+ if os.path.exists(envdir):
120
+ sys.exit('Environment directory already exists. Exiting.')
121
+
122
+ os.makedirs(envdir)
123
+ logging.info('Created directory %s', envdir)
124
+
125
+ # generate the yaml file and save it to the target environment
126
+ config['key_pair_name'] = envname
127
+
128
+ # render the template
129
+ cf_template_file = os.path.join(envdir, CLOUDFORMATION_TEMPLATE_NAME)
130
+ with open(cf_template_file, 'w') as f:
131
+ j2_template.stream(config=config).dump(f)
132
+
133
+ logging.info(f'Generated Cloud Formation Template: {cf_template_file}')
134
+ if not provision:
135
+ return
136
+
137
+ # create the key pair
138
+ keyfile_name = os.path.join(envdir, envname + '.pem')
139
+ if not update:
140
+ result = runaws('aws ec2 create-key-pair --key-name={}'.format(envname))
141
+
142
+ with open(keyfile_name, 'w') as f:
143
+ f.write(result['KeyMaterial'])
144
+
145
+ os.chmod(keyfile_name, stat.S_IRUSR | stat.S_IWUSR)
146
+
147
+ logging.info('Created a new key pair and saved the private key to {}.'.format(keyfile_name))
148
+
149
+ # deploy to AWS using Cloud Formation
150
+ nowait = False
151
+ if update:
152
+ # first we need to figure out the latest event related to this stack so we can start watching events after that
153
+ result = runaws(f'aws cloudformation describe-stack-events --stack-name={envname}')
154
+ last_event_timestamp = result['StackEvents'][0]['Timestamp']
155
+
156
+ result = runaws_result('aws cloudformation update-stack '
157
+ f'--stack-name={envname} '
158
+ f'--template-body=file://{cf_template_file}')
159
+
160
+ if result.returncode != 0:
161
+ if 'No updates are to be performed' in str(result.stdout):
162
+ nowait = True
163
+ logging.info("The Cloudformation stack is already up to date")
164
+ else:
165
+ sys.exit(f'An error occurred while updating the Cloudformation stack: {result.stdout}')
166
+
167
+ else:
168
+ logging.info('Waiting for stack update to complete.')
169
+ else:
170
+ runaws('aws cloudformation create-stack '
171
+ f'--stack-name={envname} '
172
+ f'--template-body=file://{cf_template_file}')
173
+
174
+ logging.info('Cloud Formation stack created. Waiting for provisioning to complete.')
175
+
176
+ # use aws cloudformation describe-stack-events to follow the progress
177
+ done = nowait
178
+ previously_seen = dict()
179
+ while not done:
180
+ time.sleep(5.0)
181
+ result = runaws(f'aws cloudformation describe-stack-events --stack-name={envname}')
182
+ for event in reversed(result['StackEvents']):
183
+ if update and event['Timestamp'] <= last_event_timestamp:
184
+ continue
185
+
186
+ if event['EventId'] in previously_seen:
187
+ continue
188
+ else:
189
+ if not event['ResourceStatus'].endswith('IN_PROGRESS'):
190
+ logging.info('Provisioning event: %s %s',
191
+ event['LogicalResourceId'] if event['ResourceType'] != 'AWS::CloudFormation::Stack' else f'CloudLab {envname}',
192
+ event['ResourceStatus'])
193
+
194
+ previously_seen[event['EventId']] = event
195
+ if event['ResourceType'] == 'AWS::CloudFormation::Stack' and event['ResourceStatus'].find('COMPLETE') >= 0:
196
+ done = True
197
+
198
+ result = runaws(f'aws cloudformation describe-stacks --stack-name={envname}')
199
+ status = result['Stacks'][0]['StackStatus']
200
+
201
+ if update:
202
+ if not status.startswith('UPDATE_COMPLETE'):
203
+ sys.exit('There was a failure while updating the CloudFormation stack.')
204
+ else:
205
+ logging.info('Cloud formation stack updated.')
206
+
207
+ else:
208
+ if status != 'CREATE_COMPLETE':
209
+ sys.exit('There was a failure while creating the CloudFormation stack.')
210
+ else:
211
+ logging.info('Cloud formation stack created.')
212
+
213
+ # build the inventory
214
+ inventory = {
215
+ "all": {
216
+ "vars": {
217
+ "ansible_ssh_private_key_file": envname + ".pem"
218
+ }
219
+ }
220
+ }
221
+
222
+ for subnet in config['subnets']:
223
+ for group in subnet['servers']:
224
+ role = group['role']
225
+ for ip_suffix in group['private_ip_suffixes']:
226
+ server_lookup_key = f"Instance{subnet['az'].upper()}{ip_suffix}Attributes"
227
+ attributes = get_cf_output(result, server_lookup_key).split('|')
228
+ public_ip = attributes[0]
229
+ private_ip = attributes[1]
230
+ # dns_name = attributes[2]
231
+
232
+ inv_role = inventory.setdefault(role, {
233
+ 'vars': {
234
+ 'ansible_user': config['roles'][role]['ssh_user']
235
+ },
236
+ 'hosts': {}
237
+ })
238
+
239
+ inv_role['hosts'][public_ip] = {
240
+ 'private_ip': private_ip
241
+ }
242
+
243
+ invfile = os.path.join(envdir, 'inventory.yaml')
244
+ with open(invfile, 'w') as f:
245
+ yaml.dump(inventory, f)
246
+
247
+ logging.info('Wrote inventory to {}'.format(invfile))
248
+
249
+ # returns public_ip, private_ip
250
+ def get_cf_output(cf_cmd_result, key):
251
+ result = None
252
+ for output in cf_cmd_result['Stacks'][0]['Outputs']:
253
+ if output['OutputKey'] == key:
254
+ result = output['OutputValue']
255
+ break
256
+
257
+ return result
258
+
259
+ # automatically appends --region= --output=json
260
+ def runaws(commands):
261
+ cmdarray = commands.split()
262
+ cmdarray += ['--region={}'.format(config['region']), '--output=json']
263
+
264
+ result = subprocess.run(cmdarray, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
265
+
266
+ if result.returncode != 0:
267
+ sys.exit(
268
+ 'Exiting because an error occurred while running: {}. The output was: {}'.format(' '.join(cmdarray),
269
+ result.stdout))
270
+ if len(result.stdout) > 0:
271
+ return json.loads(result.stdout)
272
+ else:
273
+ return None
274
+
275
+ # automatically appends --region= --output=json
276
+ def runaws_result(commands):
277
+ cmdarray = commands.split()
278
+ cmdarray += ['--region={}'.format(config['region']), '--output=json']
279
+
280
+ return subprocess.run(cmdarray, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
281
+
282
+ # Parses cidr, extracts the first 3 octets, and appends suffix to then yielding a 4 octet IP
283
+ def make_ip(cidr, suffix):
284
+ cidr_re = r'(\d{1,3}\.\d{1,3}\.\d{1,3})\.\d{1,3}/\d{2}'
285
+ suffix_re = r'\d{1,3}'
286
+ match = re.fullmatch(cidr_re, cidr)
287
+ if match is None:
288
+ raise SyntaxError(f'cidr "{cidr}" does not have the expected format.')
289
+
290
+ suffix = str(suffix)
291
+ if re.fullmatch(suffix_re, suffix) is None:
292
+ raise SyntaxError(f'suffix "{suffix}" does not have the expected format.')
293
+
294
+ return f'{match.group(1)}.{suffix}'
@@ -0,0 +1,147 @@
1
+ ---
2
+ AWSTemplateFormatVersion: 2010-09-09
3
+ Description: "CloudLab {{ config.key_pair_name }}"
4
+
5
+ Resources:
6
+ VPC:
7
+ Type: AWS::EC2::VPC
8
+ Properties:
9
+ CidrBlock: {{ config.cidr }}
10
+ EnableDnsSupport: true
11
+ EnableDnsHostnames: true
12
+ InstanceTenancy: default
13
+
14
+ InternetGateway:
15
+ Type: AWS::EC2::InternetGateway
16
+
17
+ GatewayAttachment:
18
+ Type: AWS::EC2::VPCGatewayAttachment
19
+ Properties:
20
+ VpcId: !Ref VPC
21
+ InternetGatewayId: !Ref InternetGateway
22
+
23
+ MainRouteTable:
24
+ Type: AWS::EC2::RouteTable
25
+ Properties:
26
+ VpcId: !Ref VPC
27
+
28
+ NonLocalRoute:
29
+ Type: AWS::EC2::Route
30
+ Properties:
31
+ DestinationCidrBlock: 0.0.0.0/0
32
+ RouteTableId: !Ref MainRouteTable
33
+ GatewayId: !Ref InternetGateway
34
+ DependsOn: InternetGateway
35
+
36
+ {% for subnet in config.subnets %}
37
+ Subnet{{ subnet.az|upper }}:
38
+ Type: AWS::EC2::Subnet
39
+ Properties:
40
+ CidrBlock: {{ subnet.cidr }}
41
+ VpcId: !Ref VPC
42
+
43
+ Subnet{{ subnet.az|upper }}RouteTableAssociation:
44
+ Type: AWS::EC2::SubnetRouteTableAssociation
45
+ Properties:
46
+ RouteTableId: !Ref MainRouteTable
47
+ SubnetId: !Ref Subnet{{ subnet.az|upper }}
48
+ {% endfor %}
49
+
50
+ BaseSecurityGroup:
51
+ Type: AWS::EC2::SecurityGroup
52
+ Properties:
53
+ GroupDescription: Allow SSH
54
+ VpcId: !Ref VPC
55
+
56
+ SSHIngressRule:
57
+ Type: AWS::EC2::SecurityGroupIngress
58
+ Properties:
59
+ GroupId: !Ref BaseSecurityGroup
60
+ IpProtocol: tcp
61
+ FromPort: 22
62
+ ToPort: 22
63
+ CidrIp: 0.0.0.0/0
64
+
65
+ VPCIngressRule:
66
+ Type: AWS::EC2::SecurityGroupIngress
67
+ Properties:
68
+ GroupId: !Ref BaseSecurityGroup
69
+ IpProtocol: -1
70
+ CidrIp: {{ config.cidr }}
71
+
72
+ AllEgressRule:
73
+ Type: AWS::EC2::SecurityGroupEgress
74
+ Properties:
75
+ GroupId: !Ref BaseSecurityGroup
76
+ IpProtocol: -1
77
+ CidrIp: 0.0.0.0/0
78
+
79
+ {% for role_name, role in config.roles.items() %}
80
+ {{ role_name }}SecurityGroup:
81
+ Type: AWS::EC2::SecurityGroup
82
+ Properties:
83
+ GroupDescription: Ingress rules for {{ role_name }}
84
+ VpcId: !Ref VPC
85
+
86
+ {% for port in role.open_ports %}
87
+ {{ role_name }}Port{{ port }}IngressRule:
88
+ Type: AWS::EC2::SecurityGroupIngress
89
+ Properties:
90
+ GroupId: !Ref {{ role_name }}SecurityGroup
91
+ IpProtocol: tcp
92
+ FromPort: {{ port }}
93
+ ToPort: {{ port }}
94
+ CidrIp: 0.0.0.0/0
95
+ {% endfor %}
96
+ {% endfor %}
97
+
98
+ {% for subnet in config.subnets %}
99
+ {% for group in subnet.servers %}
100
+ {% for private_ip_suffix in group.private_ip_suffixes %}
101
+ Instance{{ subnet.az|upper }}{{ private_ip_suffix }}:
102
+ Type: AWS::EC2::Instance
103
+ Properties:
104
+ DisableApiTermination: false
105
+ InstanceInitiatedShutdownBehavior: stop
106
+ {% if config.roles[group.role].ami_id %}
107
+ ImageId: {{ config.roles[group.role].ami_id }}
108
+ {% else %}
109
+ ImageId: '{% raw %}{{resolve:ssm:{% endraw %}{{ config.roles[group.role].ami_name }}{% raw %}}}{% endraw %}'
110
+ {% endif %}
111
+ InstanceType: {{ config.roles[group.role].instance_type }}
112
+ KeyName: {{ config.key_pair_name }}
113
+ Monitoring: false
114
+ NetworkInterfaces:
115
+ - DeleteOnTermination: true
116
+ Description: primary network interface
117
+ DeviceIndex: 0
118
+ SubnetId: !Ref Subnet{{ subnet.az|upper }}
119
+ PrivateIpAddress: {{ group.private_ip_addresses[loop.index0] }}
120
+ GroupSet:
121
+ - !Ref BaseSecurityGroup
122
+ - !Ref {{ group.role }}SecurityGroup
123
+ AssociatePublicIpAddress: true
124
+ {% endfor %}
125
+ {% endfor %}
126
+ {% endfor %}
127
+
128
+ Outputs:
129
+ {% for subnet in config.subnets %}
130
+ {% for group in subnet.servers %}
131
+ {% for private_ip in group.private_ip_suffixes %}
132
+ Instance{{ subnet.az|upper }}{{ private_ip }}Attributes:
133
+ Description: The attributes of instance {{ subnet.az|upper }}{{ private_ip }}.
134
+ Value: !Join
135
+ - '|'
136
+ - - !GetAtt
137
+ - Instance{{ subnet.az|upper }}{{ private_ip }}
138
+ - PublicIp
139
+ - !GetAtt
140
+ - Instance{{ subnet.az|upper }}{{ private_ip }}
141
+ - PrivateIp
142
+ - !GetAtt
143
+ - Instance{{ subnet.az|upper }}{{ private_ip }}
144
+ - PublicDnsName
145
+ {% endfor %}
146
+ {% endfor %}
147
+ {% endfor %}
@@ -0,0 +1,45 @@
1
+ # This example create a vpc with 2 subnets, one in us-east-2a and one in us-east-2b. There are
2
+ # 2 types of machines, which are listed under "roles": a ClusterMember and a LoadGenerator. Roles can have
3
+ # any name you choose so long as they do not contain spaces or special characters. The instance type, ami and
4
+ # other details are specified under the role.
5
+ #
6
+ # AMIs can be specified with either ami_id or ami_name. The ami_id is the Amazon image id, which is *region specific*.
7
+ # A region agnostic approach is supported via ssm parameters. If ami_id is not provided, then the ami_name will be
8
+ # used to look up the image id for the region you are in.
9
+ #
10
+ # See https://medium.com/@josiahsicad/list-of-ec2-image-public-ssm-parameters-3338f83d4ec8 for an explanation of
11
+ # how to look up SSM parameters for various operating systems. This sample illustrates both approaches.
12
+ #
13
+ # Number of instances of each type, and placement within subnets is specified under "subnets". In the example
14
+ # below, 2 instances of the ClusterMember type will be placed in us-east-2a and will have IP addresses
15
+ # 10.0.1.101 and 10.0.1.102. The us-east-2b region will contain 2 more ClusterMember instances at
16
+ # 10.0.2.101 and 10.0.2.102, and a LoadGenerator instance at 10.0.2.201
17
+ #
18
+ ---
19
+ region: us-east-2
20
+ cidr: 10.0.0.0/16 # must be a /16 subnet, must be a non-routable IP (e.g. 10.*.*.*, 192.168.*.*)
21
+ roles:
22
+ ClusterMember: # role names are copied directly into the CloudFormation template and should not contain special characters
23
+ instance_type: m5.xlarge
24
+ ami_name: /aws/service/canonical/ubuntu/server/xenial/stable/current/amd64/hvm/ebs-gp2/ami-id
25
+ ssh_user: ubuntu
26
+ open_ports: [5701, 5702]
27
+ LoadGenerator:
28
+ instance_type: m5.xlarge
29
+ ami_name: /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64 # wont be used when ami_id is present
30
+ ami_id: ami-05efc83cb5512477c # note this is region specific
31
+ ssh_user: ec2-user
32
+ open_ports: [8080]
33
+ subnets:
34
+ - az: a # will be appended to region, e.g. us-east-2a
35
+ cidr: 10.0.1.0/24 # must be a /24
36
+ servers:
37
+ - private_ip_suffixes: [101, 102] # only need to give the last octet, becomes 10.0.1.101
38
+ role: ClusterMember
39
+ - az: b
40
+ cidr: 10.0.2.0/24
41
+ servers:
42
+ - private_ip_suffixes: [101, 102] # creates 2 servers, 10.0.2.101 and 10.0.2.102
43
+ role: ClusterMember
44
+ - private_ip_suffixes: [201]
45
+ role: LoadGenerator
@@ -0,0 +1,193 @@
1
+ Metadata-Version: 2.4
2
+ Name: cloudlab
3
+ Version: 1.2.2
4
+ Summary: A tool for provisioning lab environments on AWS
5
+ Author: Randy May
6
+ Author-email: randy@mathysphere.com
7
+ Requires-Python: >=3.10
8
+ Classifier: Programming Language :: Python :: 3
9
+ Classifier: Programming Language :: Python :: 3.10
10
+ Classifier: Programming Language :: Python :: 3.11
11
+ Classifier: Programming Language :: Python :: 3.12
12
+ Classifier: Programming Language :: Python :: 3.13
13
+ Classifier: Programming Language :: Python :: 3.14
14
+ Requires-Dist: jinja2 (>=3.1.4)
15
+ Requires-Dist: pyyaml (>=6.0.2)
16
+ Description-Content-Type: text/markdown
17
+
18
+ # Cloudlab Overview
19
+
20
+ Cloudlab allows you to quickly provision a lab environment on AWS. It attempts to strike a good balance between
21
+ configurability and ease of use by adopting a somewhat opinionated approach.
22
+
23
+ Cloudlab provisions a single VPC on AWS with a /16 private address space. Within the VPC, 1 or more subnets can be
24
+ configured. All subnets are public and all have their own /24 address space inside the VPC. All machines are assigned
25
+ a role. The machine type, image and a list of open ports are configured at the role level to avoid redundant
26
+ configuration.
27
+
28
+ After successful deployment, cloudlab writes an `inventory.yaml` file, which list the public and private ips of all
29
+ servers, grouped by role. The inventory file is suitable for use with Ansible.
30
+
31
+ A sample configuration for deploying 5 servers to 2 subnets is shown below.
32
+ ```yaml
33
+ ---
34
+ region: us-east-2
35
+ cidr: 10.0.0.0/16 # must be a /16 subnet, must be a non-routable IP (e.g. 10.*.*.*, 192.168.*.*)
36
+ roles:
37
+ ClusterMember: # role names are copied directly into the CloudFormation template and should not contain special characters
38
+ instance_type: m5.xlarge
39
+ ami_name: /aws/service/canonical/ubuntu/server/xenial/stable/current/amd64/hvm/ebs-gp2/ami-id
40
+ ssh_user: ubuntu
41
+ open_ports: [5701, 5702]
42
+ LoadGenerator:
43
+ instance_type: m5.xlarge
44
+ ami_name: /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64 # wont be used when ami_id is present
45
+ ami_id: ami-05efc83cb5512477c # note this is region specific
46
+ ssh_user: ec2-user
47
+ open_ports: [8080]
48
+ subnets:
49
+ - az: a # will be appended to region, e.g. us-east-2a
50
+ cidr: 10.0.1.0/24 # must be a /24
51
+ servers:
52
+ - private_ip_suffixes: [101, 102] # only need to give the last octet, becomes 10.0.1.101
53
+ role: ClusterMember
54
+ - az: b
55
+ cidr: 10.0.2.0/24
56
+ servers:
57
+ - private_ip_suffixes: [101, 102] # creates 2 servers, 10.0.2.101 and 10.0.2.102
58
+ role: ClusterMember
59
+ - private_ip_suffixes: [201]
60
+ role: LoadGenerator
61
+ ```
62
+
63
+ Other Notes:
64
+ - All instances (hosts) will have an Amazon assigned public IP address and DNS name.
65
+ - All hosts can initiate outbound connections to any other server either in or outside of the VPC
66
+ - All hosts can receive connections on any port, from any other server in the VPC
67
+ - All hosts accept inbound connections on port 22
68
+ - Other than port 22, hosts will only accept inbound connections on specific ports which can be configured per role
69
+ - Each environment provisioned with cloudlab will have its own ssh key which will be shared by all the hosts.
70
+ - AMIs can be specified either with "ami_id", which is the region specific image id, or by "ami_name", which is
71
+ actually a public ssm parameter which is mapped to the correct id in each environment. For example:
72
+ "/aws/service/canonical/ubuntu/server/xenial/stable/current/amd64/hvm/ebs-gp2/ami-id". For help finding the
73
+ SSM parameter for a certain image, see [this article](https://medium.com/@josiahsicad/list-of-ec2-image-public-ssm-parameters-3338f83d4ec8).
74
+
75
+ # Setup
76
+
77
+ Cloudlab requires python 3.10+.
78
+
79
+ It can be installed using pip.
80
+
81
+ ```
82
+ pip install cloudlab
83
+ ```
84
+
85
+ Cloudlab requires the aws cli to be present. See https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
86
+ for installation instructions.
87
+
88
+ The AWS CLI will need to be configured with your credentials so it can access your AWS account.
89
+ If your aws cli is not already configured, provide your credentials using the `aws configure` command or,
90
+ if sso is configured, `aws sso login`
91
+
92
+ Cloudlab will have whatever privileges your cli has. The snippet below is an IAM policy describing the permissions
93
+ needed by cloudlab.
94
+
95
+ ```json
96
+ {
97
+ "Version": "2012-10-17",
98
+ "Statement": [
99
+ {
100
+ "Action": [
101
+ "ec2:*",
102
+ "cloudformation:*",
103
+ "elasticloudbalancing:*"
104
+ ],
105
+ "Effect": "Allow",
106
+ "Resource": "*"
107
+ }
108
+ ]
109
+ }
110
+ ```
111
+
112
+ # Usage
113
+
114
+ ## Define your environment
115
+ Create a file called "cloudlab_config.yaml" in the current directory and edit it to describe the environment you
116
+ would like to provision. See the example above. You can also create a sample configuration file by running
117
+ `cloudlab sample > cloudlab_config.yaml`. See the comments in the sample file for details.
118
+
119
+ ## Create a new environment
120
+
121
+ ```
122
+ cloudlab mkenv <envdir>
123
+ ```
124
+
125
+ "envdir" should be an absolute or relative path. The name of the directory, without any path, will be used as the
126
+ name of the environment and the name must be unique. The "envdir" directory will be created. The process
127
+ will fail if the directory exists.
128
+
129
+ You can generate a CloudFormation template and skip the provisioning step by adding the `--no-provision` flag as
130
+ shown below.
131
+
132
+ ```
133
+ cloudlab mkenv <envdir> --no-provision
134
+ ```
135
+ ## Destroy an environment
136
+
137
+ ```
138
+ cloudlab rmenv <envdir>
139
+ ```
140
+
141
+ This command is idempotent. It will not fail if the environment does not exist.
142
+
143
+ ## Update an environment
144
+
145
+ Update `cloudlab_config.yaml` with your changes and run ...
146
+
147
+ ```
148
+ cloudlab update <envdir>
149
+ ```
150
+
151
+ # Release Notes
152
+
153
+ ## v1.2.2
154
+ - Converted to Poetry build system
155
+ - Use SSM parameters for AMI ids.
156
+
157
+ ## v1.2.1 - includes an important fix to the inventory file
158
+ - Fix to correct the format used for variables in the Ansible inventory file
159
+ - Added a validation that the environment name is also a valid CloudFormation stack name
160
+
161
+ ## v1.2.0 is a major update
162
+ - default plan now includes multiple subnets
163
+ - configuration format has changed to allow the specification of subnets
164
+ - the `--plan` option allows alternate templates to be used although there currently are none.
165
+ - the `cloudlab sample` will output a sample configuration file
166
+ - removed dependency on methods in setuptools
167
+
168
+ ## v1.1.9
169
+
170
+ - updated required verision of PyYAML to >= 5.4 to avoid known vulnerability in earlier versions.
171
+
172
+ ## v1.1.8
173
+
174
+ - tplate updates broke cloudlab. The tplate version is now pinned to 1.0.3.
175
+
176
+ ## v1.1.7
177
+
178
+ - Due to a limitation of 60 outputs in an AWS CloudFormation template, it was
179
+ not possible to provision more than 20 servers using cloudlab. With this
180
+ update, the limit has been raised to 60 servers.
181
+
182
+ ## v1.1.6
183
+
184
+ - fix for failing "update" command
185
+
186
+ ## v1.1.5
187
+
188
+ - fixed a bug that caused mkenv to fail
189
+
190
+ ## v1.1.4
191
+
192
+ - added the "update" command
193
+
@@ -0,0 +1,8 @@
1
+ cloudlab/__init__.py,sha256=z2CnkAUf9davrLieLZYsV237Ns0FNG-L5UAknrvfMGs,35
2
+ cloudlab/commandline.py,sha256=uWFEgOx9PLPCutUH5dtZ6lac5-ZiPeLrnCkWfJ6094U,10916
3
+ cloudlab/plans/aws_with_subnets.yaml.j2,sha256=rJkzfW8NWTqqaszRdny3IJ_MEOi_rFybI05vMYmun3I,4429
4
+ cloudlab/resources/cloudlab_config.yaml,sha256=dLCXGmtpAhUgBNyrKQ_9JvR6AUki5stLjAtOJw0VDpA,2467
5
+ cloudlab-1.2.2.dist-info/METADATA,sha256=0Yc_KjRaKswK1CdXBM0lNAEKMAdbYiok7bwm9sxBAg0,6970
6
+ cloudlab-1.2.2.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
7
+ cloudlab-1.2.2.dist-info/entry_points.txt,sha256=xC28NOEc2iIilrCyfCjovTtGtDgSjl01N8BNXglx-aI,53
8
+ cloudlab-1.2.2.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: poetry-core 2.2.1
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,3 @@
1
+ [console_scripts]
2
+ cloudlab=cloudlab.commandline:run
3
+