cloud-submit 0.5.1__tar.gz → 0.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. {cloud_submit-0.5.1 → cloud_submit-0.6}/PKG-INFO +1 -1
  2. {cloud_submit-0.5.1 → cloud_submit-0.6}/pyproject.toml +1 -1
  3. {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/cli.py +51 -2
  4. {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/controller.py +39 -0
  5. {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/environment_handler.py +8 -0
  6. {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/envs/aws/remote_environment_handler.py +163 -16
  7. {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/envs/local/environment_handler.py +9 -0
  8. {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/execution/config.py +6 -2
  9. {cloud_submit-0.5.1 → cloud_submit-0.6}/.gitignore +0 -0
  10. {cloud_submit-0.5.1 → cloud_submit-0.6}/LICENSE +0 -0
  11. {cloud_submit-0.5.1 → cloud_submit-0.6}/README.md +0 -0
  12. {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/aws/src/csub/build_config.py +0 -0
  13. {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/aws/src/fit.py +0 -0
  14. {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/aws/src/generate.py +0 -0
  15. {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/aws/src/parameters.py +0 -0
  16. {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/aws/userconfig/template.yaml +0 -0
  17. {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/basic/src/csub/build_config.py +0 -0
  18. {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/basic/src/fit.py +0 -0
  19. {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/basic/src/generate.py +0 -0
  20. {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/basic/src/parameters.py +0 -0
  21. {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/basic/userconfig/template.yaml +0 -0
  22. {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/multi-worker/src/csub/build_config.py +0 -0
  23. {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/multi-worker/src/fit.py +0 -0
  24. {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/multi-worker/src/generate.py +0 -0
  25. {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/multi-worker/src/parameters.py +0 -0
  26. {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/multi-worker/userconfig/template.yaml +0 -0
  27. {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/__init__.py +0 -0
  28. {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/config.py +0 -0
  29. {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/envs/aws/__init__.py +0 -0
  30. {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/envs/aws/local_environment_handler.py +0 -0
  31. {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/envs/aws/local_execution_handler.py +0 -0
  32. {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/envs/aws/remote_execution_handler.py +0 -0
  33. {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/envs/aws/s3_tools.py +0 -0
  34. {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/envs/local/execution_handler.py +0 -0
  35. {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/execution/base_execution_handler.py +0 -0
  36. {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/execution/execute.py +0 -0
  37. {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/execution/utils.py +0 -0
  38. {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/images.py +0 -0
  39. {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: cloud-submit
3
- Version: 0.5.1
3
+ Version: 0.6
4
4
  Summary: A tool for building processing pipelines in Python and executing them locally or in the cloud.
5
5
  Project-URL: Homepage, https://github.com/mwiebusch78/cloud-submit
6
6
  Project-URL: Bug Tracker, https://github.com/mwiebusch78/cloud-submit/issues
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "cloud-submit"
7
- version = "0.5.1"
7
+ version = "0.6"
8
8
  authors = [
9
9
  { name="Martin Wiebusch" },
10
10
  ]
@@ -178,8 +178,7 @@ def list_pipelines(ctx, steps, pipelines):
178
178
  Run the pipeline PIPELINE. Specify STEPS to select a subset of steps to run.
179
179
  STEPS should be a comma-separated list of step names or step ranges of the form
180
180
  start:end where start and end are step names. If steps is not specified all
181
- steps are executed. By default, all *execution* images needed for the run are
182
- re-built (but you can use --no-rebuild to prevent that).
181
+ steps are executed.
183
182
  """
184
183
  )
185
184
  @click.option(
@@ -273,6 +272,56 @@ def run_pipeline(
273
272
  abort(str(e))
274
273
 
275
274
 
275
+ @pipelines.command(
276
+ name='deploy',
277
+ help="""Deploy a pipeline.
278
+
279
+ Deploy the pipeline PIPELINE.
280
+ """
281
+ )
282
+ @click.option(
283
+ '--env', '-e',
284
+ type=str,
285
+ default=None,
286
+ help='The name of the environment to use for the deployment.',
287
+ )
288
+ @click.option(
289
+ '--build-env',
290
+ type=str,
291
+ default=None,
292
+ help='The name of the environment to use for building images.',
293
+ )
294
+ @click.argument(
295
+ 'pipeline',
296
+ type=str,
297
+ required=True,
298
+ )
299
+ @click.pass_context
300
+ def deploy_pipeline(
301
+ ctx,
302
+ env,
303
+ build_env,
304
+ pipeline,
305
+ ):
306
+ init(ctx)
307
+ config = ctx.obj['config']
308
+ controller = ctx.obj['controller']
309
+
310
+ try:
311
+ pipeline = config.pipelines[pipeline]
312
+ except KeyError:
313
+ abort(f'Pipeline not found: {pipeline}')
314
+
315
+ try:
316
+ deployment_data = controller.deploy_pipeline(
317
+ pipeline.name,
318
+ env=env,
319
+ build_env=build_env,
320
+ )
321
+ except CloudSubmitError as e:
322
+ abort(str(e))
323
+
324
+
276
325
  # images subgroup
277
326
 
278
327
 
@@ -444,6 +444,45 @@ class Controller:
444
444
  'start_timestamp': now,
445
445
  }
446
446
 
447
+ def deploy_pipeline(
448
+ self,
449
+ pipeline,
450
+ env=None,
451
+ build_env=None,
452
+ ):
453
+ try:
454
+ pipeline = self._config.pipelines[pipeline]
455
+ except KeyError:
456
+ raise CloudSubmitError(f'Pipeline not found: {pipeline}')
457
+
458
+ if build_env is None:
459
+ build_env = env
460
+
461
+ env_handler = self._config.get_run_env(env)
462
+ now = dt.datetime.now(tz=dt.UTC)
463
+ build_id = env_handler.generate_build_id(now, None)
464
+
465
+ with self._config.in_project_root():
466
+ images = [step.image for step in pipeline.steps]
467
+ self.build_images(images, build_id=build_id, env=build_env)
468
+ refs = {}
469
+ for step in pipeline.steps:
470
+ image = self._config.images.get(step.image)
471
+ if image is None:
472
+ raise CloudSubmitError(
473
+ f'Cannot find declaration for image {step.image}.')
474
+ repo = env_handler.get_image_repo(image)
475
+ refs[step.name] = ':'.join([repo, build_id])
476
+
477
+ temp_path = 'temp/deploy'
478
+ ensure_path(temp_path, clear=True)
479
+ env_handler.deploy_pipeline(
480
+ pipeline=pipeline,
481
+ image_refs=refs,
482
+ temp_path=temp_path,
483
+ )
484
+
485
+
447
486
  def print_logs(
448
487
  self,
449
488
  run_id,
@@ -387,3 +387,11 @@ class EnvironmentHandler:
387
387
 
388
388
  def print_logs(self, run_id, start_timestamp, stream=False):
389
389
  raise NotImplementedError
390
+
391
+ def deploy_pipeline(
392
+ self,
393
+ pipeline,
394
+ image_refs,
395
+ temp_path
396
+ ):
397
+ raise NotImplementedError
@@ -35,6 +35,8 @@ class RemoteAWSEnv(LocalAWSEnv):
35
35
  aws_command='aws',
36
36
  container_aws_command='aws',
37
37
  log_group_prefix='/ecs/csub/',
38
+ scheduler_timezone=None,
39
+ scheduler_role_arn=None,
38
40
  ):
39
41
  super().__init__(
40
42
  name=name,
@@ -59,6 +61,8 @@ class RemoteAWSEnv(LocalAWSEnv):
59
61
  self._ecs_task_role_arn = str(ecs_task_role_arn)
60
62
  self._stepfunctions_role_arn = str(stepfunctions_role_arn)
61
63
  self._log_group_prefix = str(log_group_prefix)
64
+ self._scheduler_timezone = scheduler_timezone
65
+ self._scheduler_role_arn = scheduler_role_arn
62
66
 
63
67
  del self._envvars['CSUB_AWS_PROFILE']
64
68
  del self._envvars['CSUB_AWS_REGION']
@@ -91,8 +95,16 @@ class RemoteAWSEnv(LocalAWSEnv):
91
95
  repo_name = self.get_image_repo(image)
92
96
  return self._list_remote_image_tags(repo_name)
93
97
 
94
- def _make_workflow_name(self, run_id):
95
- return '--'.join([self._project, self._user, run_id])
98
+ def _make_workflow_name(self, pipeline_name, run_id):
99
+ if run_id is None:
100
+ parts = ['deployed', self._project, self._user, pipeline_name]
101
+ else:
102
+ parts = ['adhoc', self._project, self._user, run_id]
103
+ return '--'.join(parts)
104
+
105
+ def _make_schedule_name(self, pipeline_name):
106
+ parts = [self._project, self._user, pipeline_name]
107
+ return '--'.join(parts)
96
108
 
97
109
  def _make_workflow_arn(self, workflow_name):
98
110
  return ':'.join([
@@ -101,6 +113,13 @@ class RemoteAWSEnv(LocalAWSEnv):
101
113
  'stateMachine', workflow_name,
102
114
  ])
103
115
 
116
+ def _make_schedule_arn(self, schedule_name):
117
+ return ':'.join([
118
+ 'arn', 'aws', 'scheduler',
119
+ self._aws_region, self._aws_account_id,
120
+ f'schedule/default/{schedule_name}',
121
+ ])
122
+
104
123
  def _check_workflow_exists(self, workflow_name):
105
124
  workflow_arn = self._make_workflow_arn(workflow_name)
106
125
  result = run_command(
@@ -119,6 +138,24 @@ class RemoteAWSEnv(LocalAWSEnv):
119
138
  arns = set(s['stateMachineArn'] for s in result['stateMachines'])
120
139
  return workflow_arn in arns
121
140
 
141
+ def _check_schedule_exists(self, schedule_name):
142
+ schedule_arn = self._make_schedule_arn(schedule_name)
143
+ result = run_command(
144
+ [
145
+ self._aws_command,
146
+ 'scheduler',
147
+ 'list-schedules',
148
+ '--profile', self._aws_profile,
149
+ '--region', self._aws_region,
150
+ ],
151
+ stdout=subprocess.PIPE,
152
+ text=True,
153
+ hide_stderr=True,
154
+ )
155
+ result = json.loads(result.stdout)
156
+ arns = set(s['Arn'] for s in result['Schedules'])
157
+ return schedule_arn in arns
158
+
122
159
  def _group_steps(self, pipeline, image_refs):
123
160
  groups = []
124
161
  last_spec = None
@@ -157,11 +194,20 @@ class RemoteAWSEnv(LocalAWSEnv):
157
194
  for name, value in self._secrets.items()
158
195
  if value is not None
159
196
  ]
197
+ if timestamp is None or run_id is None:
198
+ timestamp = "{% $states.context.Execution.StartTime %}"
199
+ run_id = (
200
+ "{% $fromMillis($toMillis($states.context.Execution.StartTime), "
201
+ "'[Y0001][M01][D01]-[H01][m01][s01]') %}"
202
+ )
203
+ else:
204
+ timestamp = timestamp.isoformat()
205
+
160
206
  return {
161
207
  f"RegisterTask{task_index}": {
162
208
  "Type": "Task",
163
209
  "Resource": "arn:aws:states:::aws-sdk:ecs:registerTaskDefinition",
164
- "Parameters": {
210
+ "Arguments": {
165
211
  "Family": f"{workflow_name}--{task_index}",
166
212
  "RequiresCompatibilities": ["EC2"],
167
213
  "NetworkMode": "host",
@@ -184,7 +230,7 @@ class RemoteAWSEnv(LocalAWSEnv):
184
230
  "Environment": [
185
231
  {
186
232
  "Name": "CSUB_TIMESTAMP",
187
- "Value": timestamp.isoformat(),
233
+ "Value": timestamp,
188
234
  },
189
235
  {
190
236
  "Name": "CSUB_RUN_ID",
@@ -220,15 +266,16 @@ class RemoteAWSEnv(LocalAWSEnv):
220
266
  }
221
267
  ]
222
268
  },
223
- "ResultPath": "$.RegisteredTask",
269
+ "Output": "{% {'RegisteredTask': $states.result} %}",
224
270
  "Next": f"RunTask{task_index}"
225
271
  },
226
272
  f"RunTask{task_index}": {
227
273
  "Type": "Task",
228
274
  "Resource": "arn:aws:states:::ecs:runTask.sync",
229
- "Parameters": {
275
+ "Arguments": {
230
276
  "Cluster": self._ecs_cluster_arn,
231
- "TaskDefinition.$": "$.RegisteredTask.TaskDefinition.TaskDefinitionArn",
277
+ "TaskDefinition":
278
+ "{% $states.input.RegisteredTask.TaskDefinition.TaskDefinitionArn %}",
232
279
  "CapacityProviderStrategy": [
233
280
  {
234
281
  "CapacityProvider": self._ecs_capacity_provider,
@@ -252,11 +299,11 @@ class RemoteAWSEnv(LocalAWSEnv):
252
299
  }
253
300
  ]
254
301
  },
255
- "ResultPath": "$.ExecutionResult",
302
+ "Output": "{% $merge([$states.input, {'ExecutionResult': $states.result}]) %}",
256
303
  "Catch": [
257
304
  {
258
305
  "ErrorEquals": ["States.ALL"],
259
- "ResultPath": "$.ExecutionError",
306
+ "Output": "{% $merge([$states.input, {'ExecutionError': $states.errorOutput}]) %}",
260
307
  "Next": f"DeregisterTask{task_index}OnFailure"
261
308
  }
262
309
  ],
@@ -265,8 +312,8 @@ class RemoteAWSEnv(LocalAWSEnv):
265
312
  f"DeregisterTask{task_index}OnSuccess": {
266
313
  "Type": "Task",
267
314
  "Resource": "arn:aws:states:::aws-sdk:ecs:deregisterTaskDefinition",
268
- "Parameters": {
269
- "TaskDefinition.$": "$.RegisteredTask.TaskDefinition.TaskDefinitionArn"
315
+ "Arguments": {
316
+ "TaskDefinition": "{% $states.input.RegisteredTask.TaskDefinition.TaskDefinitionArn %}"
270
317
  },
271
318
  ("End" if is_last else "Next"): (
272
319
  True if is_last else f"RegisterTask{task_index + 1}"),
@@ -274,8 +321,8 @@ class RemoteAWSEnv(LocalAWSEnv):
274
321
  f"DeregisterTask{task_index}OnFailure": {
275
322
  "Type": "Task",
276
323
  "Resource": "arn:aws:states:::aws-sdk:ecs:deregisterTaskDefinition",
277
- "Parameters": {
278
- "TaskDefinition.$": "$.RegisteredTask.TaskDefinition.TaskDefinitionArn"
324
+ "Arguments": {
325
+ "TaskDefinition": "{% $states.input.RegisteredTask.TaskDefinition.TaskDefinitionArn %}"
279
326
  },
280
327
  "Next": "WorkflowFailed"
281
328
  },
@@ -325,6 +372,7 @@ class RemoteAWSEnv(LocalAWSEnv):
325
372
  }
326
373
 
327
374
  return {
375
+ "QueryLanguage": "JSONata",
328
376
  "StartAt": "RegisterTask0",
329
377
  "States": states,
330
378
  }
@@ -338,7 +386,7 @@ class RemoteAWSEnv(LocalAWSEnv):
338
386
  run_id,
339
387
  temp_path
340
388
  ):
341
- workflow_name = self._make_workflow_name(run_id)
389
+ workflow_name = self._make_workflow_name(None, run_id)
342
390
  workflow = self._build_workflow(
343
391
  pipeline=pipeline,
344
392
  image_refs=image_refs,
@@ -349,7 +397,7 @@ class RemoteAWSEnv(LocalAWSEnv):
349
397
 
350
398
  workflow_path = os.path.join(temp_path, f'{workflow_name}.json')
351
399
  with open(workflow_path, 'w') as stream:
352
- json.dump(workflow, stream)
400
+ json.dump(workflow, stream, indent=4)
353
401
 
354
402
  workflow_arn = self._make_workflow_arn(workflow_name)
355
403
  if self._check_workflow_exists(workflow_name):
@@ -404,7 +452,7 @@ class RemoteAWSEnv(LocalAWSEnv):
404
452
 
405
453
  def print_logs(self, run_id, start_timestamp, stream=False):
406
454
  print('\nStreaming logs...')
407
- workflow_name = self._make_workflow_name(run_id)
455
+ workflow_name = self._make_workflow_name(None, run_id)
408
456
  command = [
409
457
  self._aws_command,
410
458
  'logs',
@@ -418,3 +466,102 @@ class RemoteAWSEnv(LocalAWSEnv):
418
466
  command.append('--follow')
419
467
  command.append(self._log_group_prefix + workflow_name)
420
468
  run_command(command)
469
+
470
+ def deploy_pipeline(
471
+ self,
472
+ pipeline,
473
+ image_refs,
474
+ temp_path,
475
+ schedule=True,
476
+ ):
477
+ workflow_name = self._make_workflow_name(pipeline.name, None)
478
+ workflow = self._build_workflow(
479
+ pipeline=pipeline,
480
+ image_refs=image_refs,
481
+ workflow_name=workflow_name,
482
+ timestamp=None,
483
+ run_id=None,
484
+ )
485
+
486
+ workflow_path = os.path.join(temp_path, f'{workflow_name}.json')
487
+ with open(workflow_path, 'w') as stream:
488
+ json.dump(workflow, stream, indent=4)
489
+
490
+ workflow_arn = self._make_workflow_arn(workflow_name)
491
+ if self._check_workflow_exists(workflow_name):
492
+ run_command(
493
+ [
494
+ self._aws_command,
495
+ 'stepfunctions',
496
+ 'update-state-machine',
497
+ '--profile', self._aws_profile,
498
+ '--region', self._aws_region,
499
+ '--state-machine-arn', workflow_arn,
500
+ '--role-arn', self._stepfunctions_role_arn,
501
+ '--definition', f'file://{workflow_path}',
502
+ ],
503
+ stdout=subprocess.DEVNULL,
504
+ )
505
+ else:
506
+ run_command(
507
+ [
508
+ self._aws_command,
509
+ 'stepfunctions',
510
+ 'create-state-machine',
511
+ '--profile', self._aws_profile,
512
+ '--region', self._aws_region,
513
+ '--name', workflow_name,
514
+ '--role-arn', self._stepfunctions_role_arn,
515
+ '--definition', f'file://{workflow_path}',
516
+ ],
517
+ stdout=subprocess.DEVNULL,
518
+ )
519
+ print(f'Created step function {workflow_arn}.')
520
+
521
+ if schedule and pipeline.schedule is not None:
522
+ if self._scheduler_timezone is None:
523
+ raise CloudSubmitError(
524
+ 'You must specify the schedule timezone '
525
+ 'to use automatic scheduling.'
526
+ )
527
+ if self._scheduler_role_arn is None:
528
+ raise CloudSubmitError(
529
+ 'You must specify the scheduler role ARN '
530
+ 'to use automatic scheduling.'
531
+ )
532
+ schedule_name = self._make_schedule_name(pipeline.name)
533
+ target = json.dumps({
534
+ 'Arn': workflow_arn,
535
+ 'RoleArn': self._scheduler_role_arn,
536
+ })
537
+ options = [
538
+ '--profile', self._aws_profile,
539
+ '--region', self._aws_region,
540
+ '--name', schedule_name,
541
+ '--schedule-expression', f'cron({pipeline.schedule})',
542
+ '--schedule-expression-timezone', self._scheduler_timezone,
543
+ '--flexible-time-window', '{"Mode": "OFF"}',
544
+ '--target', target,
545
+ ]
546
+ if self._check_schedule_exists(schedule_name):
547
+ run_command(
548
+ [
549
+ self._aws_command,
550
+ 'scheduler',
551
+ 'update-schedule',
552
+ *options
553
+ ],
554
+ stdout=subprocess.DEVNULL,
555
+ )
556
+ else:
557
+ run_command(
558
+ [
559
+ self._aws_command,
560
+ 'scheduler',
561
+ 'create-schedule',
562
+ *options,
563
+ ],
564
+ stdout=subprocess.DEVNULL,
565
+ )
566
+ print(f'Created schedule {schedule_name}.')
567
+
@@ -170,3 +170,12 @@ class LocalEnv(EnvironmentHandler):
170
170
  def print_logs(self, run_id, start_timestamp, stream=False):
171
171
  raise CloudSubmitError(
172
172
  'Log retrieval is not supported by this environment.')
173
+
174
+ def deploy_pipeline(
175
+ self,
176
+ pipeline,
177
+ image_refs,
178
+ temp_path
179
+ ):
180
+ raise CloudSubmitError(
181
+ 'Deployment is not supported by this environment.')
@@ -194,10 +194,12 @@ class Pipeline:
194
194
  name,
195
195
  steps,
196
196
  default_submit_timestamp=None,
197
+ schedule=None,
197
198
  ):
198
199
  self.name = name
199
200
  self.steps = list(steps)
200
201
  self.default_submit_timestamp = to_utc(default_submit_timestamp)
202
+ self.schedule = schedule
201
203
 
202
204
  step_names = set()
203
205
  for step in self.steps:
@@ -212,7 +214,8 @@ class Pipeline:
212
214
  return {
213
215
  'name': self.name,
214
216
  'steps': [s.to_dict() for s in self.steps],
215
- 'default_submit_timestamp': self.default_submit_timestamp
217
+ 'default_submit_timestamp': self.default_submit_timestamp,
218
+ 'schedule': self.schedule,
216
219
  }
217
220
 
218
221
  @staticmethod
@@ -220,7 +223,8 @@ class Pipeline:
220
223
  return Pipeline(
221
224
  name=obj['name'],
222
225
  steps=[Step.from_dict(s) for s in obj['steps']],
223
- default_submit_timestamp=obj['default_submit_timestamp']
226
+ default_submit_timestamp=obj['default_submit_timestamp'],
227
+ schedule=obj['schedule'],
224
228
  )
225
229
 
226
230
 
File without changes
File without changes
File without changes