outerbounds 0.3.57rc1__py3-none-any.whl → 0.3.58__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
@@ -1,12 +1,10 @@
1
1
  import click
2
2
  from . import local_setup_cli
3
3
  from . import workstations_cli
4
- from . import perimeters_cli
5
4
 
6
5
 
7
6
  @click.command(
8
- cls=click.CommandCollection,
9
- sources=[local_setup_cli.cli, workstations_cli.cli, perimeters_cli.cli],
7
+ cls=click.CommandCollection, sources=[local_setup_cli.cli, workstations_cli.cli]
10
8
  )
11
9
  def cli(**kwargs):
12
10
  pass
@@ -43,8 +43,6 @@ BAD_EXTENSION_MESSAGE = (
43
43
  "Mis-installation of the Outerbounds Platform extension package has been detected."
44
44
  )
45
45
 
46
- PERIMETER_CONFIG_URL_KEY = "OB_CURRENT_PERIMETER_MF_CONFIG_URL"
47
-
48
46
 
49
47
  class Narrator:
50
48
  def __init__(self, verbose):
@@ -237,21 +235,20 @@ class ConfigEntrySpec:
237
235
  def get_config_specs():
238
236
  return [
239
237
  ConfigEntrySpec(
240
- "METAFLOW_DATASTORE_SYSROOT_S3",
241
- r"s3://[a-z0-9\-]+/metaflow(-[a-z0-9\-]+)[/]?",
242
- ),
243
- ConfigEntrySpec(
244
- "METAFLOW_DATATOOLS_S3ROOT", r"s3://[a-z0-9\-]+/data(-[a-z0-9\-]+)[/]?"
238
+ "METAFLOW_DATASTORE_SYSROOT_S3", "s3://[a-z0-9\-]+/metaflow[/]?"
245
239
  ),
240
+ ConfigEntrySpec("METAFLOW_DATATOOLS_S3ROOT", "s3://[a-z0-9\-]+/data[/]?"),
246
241
  ConfigEntrySpec("METAFLOW_DEFAULT_AWS_CLIENT_PROVIDER", "obp", expected="obp"),
247
242
  ConfigEntrySpec("METAFLOW_DEFAULT_DATASTORE", "s3", expected="s3"),
248
243
  ConfigEntrySpec("METAFLOW_DEFAULT_METADATA", "service", expected="service"),
249
- ConfigEntrySpec("METAFLOW_KUBERNETES_NAMESPACE", r"jobs-.*"),
250
- ConfigEntrySpec("METAFLOW_KUBERNETES_SANDBOX_INIT_SCRIPT", r"eval \$\(.*"),
251
- ConfigEntrySpec("METAFLOW_SERVICE_AUTH_KEY", r"[a-zA-Z0-9!_\-\.]+"),
252
- ConfigEntrySpec("METAFLOW_SERVICE_URL", r"https://metadata\..*"),
253
- ConfigEntrySpec("METAFLOW_UI_URL", r"https://ui\..*"),
254
- ConfigEntrySpec("OBP_AUTH_SERVER", r"auth\..*"),
244
+ ConfigEntrySpec(
245
+ "METAFLOW_KUBERNETES_NAMESPACE", "jobs\-default", expected="jobs-default"
246
+ ),
247
+ ConfigEntrySpec("METAFLOW_KUBERNETES_SANDBOX_INIT_SCRIPT", "eval \$\(.*"),
248
+ ConfigEntrySpec("METAFLOW_SERVICE_AUTH_KEY", "[a-zA-Z0-9!_\-\.]+"),
249
+ ConfigEntrySpec("METAFLOW_SERVICE_URL", "https://metadata\..*"),
250
+ ConfigEntrySpec("METAFLOW_UI_URL", "https://ui\..*"),
251
+ ConfigEntrySpec("OBP_AUTH_SERVER", "auth\..*"),
255
252
  ]
256
253
 
257
254
 
@@ -264,12 +261,7 @@ def check_metaflow_config(narrator: Narrator) -> CommandStatus:
264
261
  mitigation="",
265
262
  )
266
263
 
267
- profile = os.environ.get("METAFLOW_PROFILE")
268
- config_dir = os.path.expanduser(
269
- os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
270
- )
271
-
272
- config = metaflowconfig.init_config(config_dir, profile)
264
+ config = metaflowconfig.init_config()
273
265
  for spec in get_config_specs():
274
266
  narrator.announce_check("config entry " + spec.name)
275
267
  if spec.name not in config:
@@ -312,12 +304,7 @@ def check_metaflow_token(narrator: Narrator) -> CommandStatus:
312
304
  mitigation="",
313
305
  )
314
306
 
315
- profile = os.environ.get("METAFLOW_PROFILE")
316
- config_dir = os.path.expanduser(
317
- os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
318
- )
319
-
320
- config = metaflowconfig.init_config(config_dir, profile)
307
+ config = metaflowconfig.init_config()
321
308
  try:
322
309
  if "OBP_AUTH_SERVER" in config:
323
310
  k8s_response = requests.get(
@@ -376,13 +363,7 @@ def check_workstation_api_accessible(narrator: Narrator) -> CommandStatus:
376
363
  )
377
364
 
378
365
  try:
379
- profile = os.environ.get("METAFLOW_PROFILE")
380
- config_dir = os.path.expanduser(
381
- os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
382
- )
383
-
384
- config = metaflowconfig.init_config(config_dir, profile)
385
-
366
+ config = metaflowconfig.init_config()
386
367
  missing_keys = []
387
368
  if "METAFLOW_SERVICE_AUTH_KEY" not in config:
388
369
  missing_keys.append("METAFLOW_SERVICE_AUTH_KEY")
@@ -441,13 +422,7 @@ def check_kubeconfig_valid_for_workstations(narrator: Narrator) -> CommandStatus
441
422
  )
442
423
 
443
424
  try:
444
- profile = os.environ.get("METAFLOW_PROFILE")
445
- config_dir = os.path.expanduser(
446
- os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
447
- )
448
-
449
- config = metaflowconfig.init_config(config_dir, profile)
450
-
425
+ config = metaflowconfig.init_config()
451
426
  missing_keys = []
452
427
  if "METAFLOW_SERVICE_AUTH_KEY" not in config:
453
428
  missing_keys.append("METAFLOW_SERVICE_AUTH_KEY")
@@ -610,13 +585,6 @@ class ConfigurationWriter:
610
585
  self.decoded_config = None
611
586
  self.out_dir = out_dir
612
587
  self.profile = profile
613
- self.selected_perimeter = None
614
-
615
- ob_config_dir = path.expanduser(os.getenv("OBP_CONFIG_DIR", out_dir))
616
- self.ob_config_path = path.join(
617
- ob_config_dir,
618
- "ob_config_{}.json".format(profile) if profile else "ob_config.json",
619
- )
620
588
 
621
589
  def decode(self):
622
590
  self.decoded_config = deserialize(self.encoded_config)
@@ -624,9 +592,6 @@ class ConfigurationWriter:
624
592
  def process_decoded_config(self):
625
593
  config_type = self.decoded_config.get("OB_CONFIG_TYPE", "inline")
626
594
  if config_type == "inline":
627
- if "OBP_PERIMETER" in self.decoded_config:
628
- self.selected_perimeter = self.decoded_config["OBP_PERIMETER"]
629
-
630
595
  if "OBP_METAFLOW_CONFIG_URL" in self.decoded_config:
631
596
  self.decoded_config = {
632
597
  "OBP_METAFLOW_CONFIG_URL": self.decoded_config[
@@ -683,18 +648,6 @@ class ConfigurationWriter:
683
648
  with open(config_path, "w") as fd:
684
649
  json.dump(self.existing, fd, indent=4)
685
650
 
686
- # Every time a config is initialized, we should also reset the corresponding ob_config[_profile].json
687
- remote_config = metaflowconfig.init_config(self.out_dir, self.profile)
688
- if self.selected_perimeter and "OBP_METAFLOW_CONFIG_URL" in self.decoded_config:
689
- with open(self.ob_config_path, "w") as fd:
690
- ob_config_dict = {
691
- "OB_CURRENT_PERIMETER": self.selected_perimeter,
692
- PERIMETER_CONFIG_URL_KEY: self.decoded_config[
693
- "OBP_METAFLOW_CONFIG_URL"
694
- ],
695
- }
696
- json.dump(ob_config_dict, fd, indent=4)
697
-
698
651
  def confirm_overwrite_config(self, config_path):
699
652
  if os.path.exists(config_path):
700
653
  if not click.confirm(
@@ -19,10 +19,6 @@ from ..utils.schema import (
19
19
  OuterboundsCommandStatus,
20
20
  )
21
21
  from tempfile import NamedTemporaryFile
22
- from .perimeters_cli import (
23
- get_perimeters_from_api_or_fail_command,
24
- confirm_user_has_access_to_perimeter_or_fail,
25
- )
26
22
 
27
23
  KUBECTL_INSTALL_MITIGATION = "Please install kubectl manually from https://kubernetes.io/docs/tasks/tools/#kubectl"
28
24
 
@@ -93,7 +89,7 @@ def generate_workstation_token(config_dir=None, profile=None):
93
89
  @click.option(
94
90
  "-p",
95
91
  "--profile",
96
- default=os.environ.get("METAFLOW_PROFILE", ""),
92
+ default="",
97
93
  help="The named metaflow profile in which your workstation exists",
98
94
  )
99
95
  @click.option(
@@ -114,6 +110,7 @@ def configure_cloud_workstation(config_dir=None, profile=None, binary=None, outp
114
110
  kubeconfig_configure_step = CommandStatus(
115
111
  "ConfigureKubeConfig", OuterboundsCommandStatus.OK, "Kubeconfig is configured"
116
112
  )
113
+
117
114
  try:
118
115
  metaflow_token = metaflowconfig.get_metaflow_token_from_config(
119
116
  config_dir, profile
@@ -194,25 +191,10 @@ def configure_cloud_workstation(config_dir=None, profile=None, binary=None, outp
194
191
  @click.option(
195
192
  "-p",
196
193
  "--profile",
197
- default=os.environ.get("METAFLOW_PROFILE", ""),
194
+ default="",
198
195
  help="The named metaflow profile in which your workstation exists",
199
196
  )
200
- @click.option(
201
- "-o",
202
- "--output",
203
- default="json",
204
- help="Show output in the specified format.",
205
- type=click.Choice(["json"]),
206
- )
207
- def list_workstations(config_dir=None, profile=None, output="json"):
208
- list_response = OuterboundsCommandResponse()
209
- list_step = CommandStatus(
210
- "listWorkstations",
211
- OuterboundsCommandStatus.OK,
212
- "Workstation list successfully fetched!",
213
- )
214
- list_response.add_or_update_data("workstations", [])
215
-
197
+ def list_workstations(config_dir=None, profile=None):
216
198
  try:
217
199
  metaflow_token = metaflowconfig.get_metaflow_token_from_config(
218
200
  config_dir, profile
@@ -223,23 +205,17 @@ def list_workstations(config_dir=None, profile=None, output="json"):
223
205
  workstations_response = requests.get(
224
206
  f"{api_url}/v1/workstations", headers={"x-api-key": metaflow_token}
225
207
  )
226
- workstations_response.raise_for_status()
227
- list_response.add_or_update_data(
228
- "workstations", workstations_response.json()["workstations"]
229
- )
230
- if output == "json":
231
- click.echo(json.dumps(list_response.as_dict(), indent=4))
208
+ try:
209
+ workstations_response.raise_for_status()
210
+ click.echo(json.dumps(workstations_response.json(), indent=4))
211
+ except HTTPError:
212
+ click.secho("Failed to generate workstation token.", fg="red")
213
+ click.secho(
214
+ "Error: {}".format(json.dumps(workstations_response.json(), indent=4))
215
+ )
232
216
  except Exception as e:
233
- list_step.update(
234
- OuterboundsCommandStatus.FAIL, "Failed to list workstations", ""
235
- )
236
- list_response.add_step(list_step)
237
- if output == "json":
238
- list_response.add_or_update_data("error", str(e))
239
- click.echo(json.dumps(list_response.as_dict(), indent=4))
240
- else:
241
- click.secho("Failed to list workstations", fg="red", err=True)
242
- click.secho("Error: {}".format(str(e)), fg="red", err=True)
217
+ click.secho("Failed to list workstations", fg="red")
218
+ click.secho("Error: {}".format(str(e)))
243
219
 
244
220
 
245
221
  @cli.command(help="Hibernate workstation", hidden=True)
@@ -259,7 +235,7 @@ def list_workstations(config_dir=None, profile=None, output="json"):
259
235
  @click.option(
260
236
  "-w",
261
237
  "--workstation",
262
- default=os.environ.get("METAFLOW_PROFILE", ""),
238
+ default="",
263
239
  help="The ID of the workstation to hibernate",
264
240
  )
265
241
  def hibernate_workstation(config_dir=None, profile=None, workstation=None):
@@ -267,8 +243,6 @@ def hibernate_workstation(config_dir=None, profile=None, workstation=None):
267
243
  click.secho("Please specify a workstation ID", fg="red")
268
244
  return
269
245
  try:
270
- if not profile:
271
- profile = metaflowconfig.get_metaflow_profile()
272
246
  metaflow_token = metaflowconfig.get_metaflow_token_from_config(
273
247
  config_dir, profile
274
248
  )
@@ -293,7 +267,7 @@ def hibernate_workstation(config_dir=None, profile=None, workstation=None):
293
267
  )
294
268
  except Exception as e:
295
269
  click.secho("Failed to hibernate workstation", fg="red")
296
- click.secho("Error: {}".format(str(e)), fg="red")
270
+ click.secho("Error: {}".format(str(e)))
297
271
 
298
272
 
299
273
  @cli.command(help="Restart workstation to the int", hidden=True)
@@ -307,7 +281,7 @@ def hibernate_workstation(config_dir=None, profile=None, workstation=None):
307
281
  @click.option(
308
282
  "-p",
309
283
  "--profile",
310
- default=os.environ.get("METAFLOW_PROFILE", ""),
284
+ default="",
311
285
  help="The named metaflow profile in which your workstation exists",
312
286
  )
313
287
  @click.option(
@@ -345,7 +319,7 @@ def restart_workstation(config_dir=None, profile=None, workstation=None):
345
319
  )
346
320
  except Exception as e:
347
321
  click.secho("Failed to restart workstation", fg="red")
348
- click.secho("Error: {}".format(str(e)), fg="red")
322
+ click.secho("Error: {}".format(str(e)))
349
323
 
350
324
 
351
325
  @cli.command(help="Install dependencies needed by workstations", hidden=True)
@@ -512,85 +486,8 @@ def add_to_path(program_path, platform):
512
486
  with open(path_to_rc_file, "a+") as f: # Open bashrc file
513
487
  if program_path not in f.read():
514
488
  f.write("\n# Added by Outerbounds\n")
515
- f.write(f"export PATH=$PATH:{program_path}")
489
+ f.write(program_path)
516
490
 
517
491
 
518
492
  def to_windows_path(path):
519
493
  return os.path.normpath(path).replace(os.sep, "\\")
520
-
521
-
522
- @cli.command(help="Show relevant links for a deployment & perimeter", hidden=True)
523
- @click.option(
524
- "-d",
525
- "--config-dir",
526
- default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
527
- help="Path to Metaflow configuration directory",
528
- show_default=True,
529
- )
530
- @click.option(
531
- "-p",
532
- "--profile",
533
- default="",
534
- help="The named metaflow profile in which your workstation exists",
535
- )
536
- @click.option(
537
- "--perimeter-id",
538
- default="",
539
- help="The id of the perimeter to use",
540
- )
541
- @click.option(
542
- "-o",
543
- "--output",
544
- default="",
545
- help="Show output in the specified format.",
546
- type=click.Choice(["json", ""]),
547
- )
548
- def show_relevant_links(config_dir=None, profile=None, perimeter_id="", output=""):
549
- show_links_response = OuterboundsCommandResponse()
550
- show_links_step = CommandStatus(
551
- "showRelevantLinks",
552
- OuterboundsCommandStatus.OK,
553
- "Relevant links successfully fetched!",
554
- )
555
- show_links_response.add_or_update_data("links", [])
556
- links = []
557
- try:
558
- if not perimeter_id:
559
- metaflow_config = metaflowconfig.init_config(config_dir, profile)
560
- else:
561
- perimeters_dict = get_perimeters_from_api_or_fail_command(
562
- config_dir, profile, output, show_links_response, show_links_step
563
- )
564
- confirm_user_has_access_to_perimeter_or_fail(
565
- perimeter_id,
566
- perimeters_dict,
567
- output,
568
- show_links_response,
569
- show_links_step,
570
- )
571
-
572
- metaflow_config = metaflowconfig.init_config_from_url(
573
- config_dir, profile, perimeters_dict[perimeter_id]["remote_config_url"]
574
- )
575
-
576
- links.append(
577
- {
578
- "id": "metaflow-ui-url",
579
- "url": metaflow_config["METAFLOW_UI_URL"],
580
- "label": "Metaflow UI URL",
581
- }
582
- )
583
- show_links_response.add_or_update_data("links", links)
584
- if output == "json":
585
- click.echo(json.dumps(show_links_response.as_dict(), indent=4))
586
- except Exception as e:
587
- show_links_step.update(
588
- OuterboundsCommandStatus.FAIL, "Failed to show relevant links", ""
589
- )
590
- show_links_response.add_step(show_links_step)
591
- if output == "json":
592
- show_links_response.add_or_update_data("error", str(e))
593
- click.echo(json.dumps(show_links_response.as_dict(), indent=4))
594
- else:
595
- click.secho("Failed to show relevant links", fg="red", err=True)
596
- click.secho("Error: {}".format(str(e)), fg="red", err=True)
@@ -1,43 +1,15 @@
1
1
  import json
2
2
  import os
3
3
  import requests
4
- from os import path
5
- import requests
6
- from typing import Dict
7
-
8
4
 
9
- def init_config(config_dir, profile) -> Dict[str, str]:
10
- config = read_metaflow_config_from_filesystem(config_dir, profile)
11
5
 
12
- # This is new remote-metaflow config; fetch it from the URL
13
- if "OBP_METAFLOW_CONFIG_URL" in config:
14
- remote_config = init_config_from_url(
15
- config_dir, profile, config["OBP_METAFLOW_CONFIG_URL"]
16
- )
17
- remote_config["OBP_METAFLOW_CONFIG_URL"] = config["OBP_METAFLOW_CONFIG_URL"]
18
- return remote_config
19
- # Legacy config, use from filesystem
20
- return config
21
-
22
-
23
- def init_config_from_url(config_dir, profile, url) -> Dict[str, str]:
24
- config = read_metaflow_config_from_filesystem(config_dir, profile)
25
-
26
- if config is None or "METAFLOW_SERVICE_AUTH_KEY" not in config:
27
- raise Exception("METAFLOW_SERVICE_AUTH_KEY not found in config file")
28
-
29
- config_response = requests.get(
30
- url,
31
- headers={"x-api-key": f'{config["METAFLOW_SERVICE_AUTH_KEY"]}'},
6
+ def init_config() -> dict:
7
+ profile = os.environ.get("METAFLOW_PROFILE")
8
+ config_dir = os.path.expanduser(
9
+ os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
32
10
  )
33
- config_response.raise_for_status()
34
- remote_config = config_response.json()["config"]
35
- return remote_config
36
-
37
11
 
38
- def read_metaflow_config_from_filesystem(config_dir, profile) -> Dict[str, str]:
39
12
  config_filename = f"config_{profile}.json" if profile else "config.json"
40
-
41
13
  path_to_config = os.path.join(config_dir, config_filename)
42
14
 
43
15
  if os.path.exists(path_to_config):
@@ -45,6 +17,22 @@ def read_metaflow_config_from_filesystem(config_dir, profile) -> Dict[str, str]:
45
17
  config = json.load(json_file)
46
18
  else:
47
19
  raise Exception("Unable to locate metaflow config at '%s')" % (path_to_config))
20
+
21
+ # This is new remote-metaflow config; fetch it from the URL
22
+ if "OBP_METAFLOW_CONFIG_URL" in config:
23
+ if config is None or "METAFLOW_SERVICE_AUTH_KEY" not in config:
24
+ raise Exception("METAFLOW_SERVICE_AUTH_KEY not found in config file")
25
+
26
+ config_response = requests.get(
27
+ config["OBP_METAFLOW_CONFIG_URL"],
28
+ headers={"x-api-key": f'{config["METAFLOW_SERVICE_AUTH_KEY"]}'},
29
+ )
30
+ config_response.raise_for_status()
31
+ remote_config = config_response.json()["config"]
32
+ remote_config["METAFLOW_SERVICE_AUTH_KEY"] = config["METAFLOW_SERVICE_AUTH_KEY"]
33
+ return remote_config
34
+
35
+ # Legacy config, use from filesystem
48
36
  return config
49
37
 
50
38
 
@@ -56,10 +44,13 @@ def get_metaflow_token_from_config(config_dir: str, profile: str) -> str:
56
44
  config_dir (str): Path to the config directory
57
45
  profile (str): The named metaflow profile
58
46
  """
59
- config = init_config(config_dir, profile)
60
- if config is None or "METAFLOW_SERVICE_AUTH_KEY" not in config:
61
- raise Exception("METAFLOW_SERVICE_AUTH_KEY not found in config file")
62
- return config["METAFLOW_SERVICE_AUTH_KEY"]
47
+ config_filename = f"config_{profile}.json" if profile else "config.json"
48
+ config_path = os.path.join(config_dir, config_filename)
49
+ with open(config_path) as json_file:
50
+ config = json.load(json_file)
51
+ if config is None or "METAFLOW_SERVICE_AUTH_KEY" not in config:
52
+ raise Exception("METAFLOW_SERVICE_AUTH_KEY not found in config file")
53
+ return config["METAFLOW_SERVICE_AUTH_KEY"]
63
54
 
64
55
 
65
56
  def get_sanitized_url_from_config(config_dir: str, profile: str, key: str) -> str:
@@ -71,12 +62,16 @@ def get_sanitized_url_from_config(config_dir: str, profile: str, key: str) -> st
71
62
  profile (str): The named metaflow profile
72
63
  key (str): The key to look up in the config file
73
64
  """
74
- config = init_config(config_dir, profile)
75
- if key not in config:
76
- raise Exception(f"Key {key} not found in config")
77
- url_in_config = config[key]
78
- if not url_in_config.startswith("https://"):
79
- url_in_config = f"https://{url_in_config}"
80
-
81
- url_in_config = url_in_config.rstrip("/")
82
- return url_in_config
65
+ config_filename = f"config_{profile}.json" if profile else "config.json"
66
+ config_path = os.path.join(config_dir, config_filename)
67
+
68
+ with open(config_path) as json_file:
69
+ config = json.load(json_file)
70
+ if key not in config:
71
+ raise Exception(f"Key {key} not found in config file {config_path}")
72
+ url_in_config = config[key]
73
+ if not url_in_config.startswith("https://"):
74
+ url_in_config = f"https://{url_in_config}"
75
+
76
+ url_in_config = url_in_config.rstrip("/")
77
+ return url_in_config
@@ -5,7 +5,6 @@ class OuterboundsCommandStatus(Enum):
5
5
  OK = "OK"
6
6
  FAIL = "FAIL"
7
7
  WARN = "WARN"
8
- NOT_SUPPORTED = "NOT_SUPPORTED"
9
8
 
10
9
 
11
10
  class CommandStatus:
@@ -38,19 +37,10 @@ class OuterboundsCommandResponse:
38
37
  self._message = ""
39
38
  self._steps = []
40
39
  self.metadata = {}
41
- self._data = {}
42
-
43
- def update(self, status, code, message):
44
- self.status = status
45
- self._code = code
46
- self._message = message
47
40
 
48
41
  def add_or_update_metadata(self, key, value):
49
42
  self.metadata[key] = value
50
43
 
51
- def add_or_update_data(self, key, value):
52
- self._data[key] = value
53
-
54
44
  def add_step(self, step: CommandStatus):
55
45
  self._steps.append(step)
56
46
  self._process_step_status(step)
@@ -69,11 +59,10 @@ class OuterboundsCommandResponse:
69
59
  self._message = "We found one or more warnings with your installation."
70
60
 
71
61
  def as_dict(self):
72
- self._data["steps"] = [step.as_dict() for step in self._steps]
73
62
  return {
74
63
  "status": self.status.value,
75
64
  "code": self._code,
76
65
  "message": self._message,
66
+ "steps": [step.as_dict() for step in self._steps],
77
67
  "metadata": self.metadata,
78
- "data": self._data,
79
68
  }
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: outerbounds
3
- Version: 0.3.57rc1
3
+ Version: 0.3.58
4
4
  Summary: More Data Science, Less Administration
5
5
  License: Proprietary
6
6
  Keywords: data science,machine learning,MLOps
@@ -23,9 +23,9 @@ Requires-Dist: click (>=8.1.3,<9.0.0)
23
23
  Requires-Dist: google-api-core (>=2.16.1,<3.0.0) ; extra == "gcp"
24
24
  Requires-Dist: google-auth (>=2.27.0,<3.0.0) ; extra == "gcp"
25
25
  Requires-Dist: google-cloud-storage (>=2.14.0,<3.0.0) ; extra == "gcp"
26
- Requires-Dist: ob-metaflow (==2.11.4.8)
27
- Requires-Dist: ob-metaflow-extensions (==1.1.48)
28
- Requires-Dist: ob-metaflow-stubs (==2.11.4.8)
26
+ Requires-Dist: ob-metaflow (==2.11.4.9)
27
+ Requires-Dist: ob-metaflow-extensions (==1.1.49)
28
+ Requires-Dist: ob-metaflow-stubs (==2.11.4.9)
29
29
  Requires-Dist: opentelemetry-distro (==0.41b0)
30
30
  Requires-Dist: opentelemetry-exporter-otlp-proto-http (==1.20.0)
31
31
  Requires-Dist: opentelemetry-instrumentation-requests (==0.41b0)
@@ -0,0 +1,14 @@
1
+ outerbounds/__init__.py,sha256=GPdaubvAYF8pOFWJ3b-sPMKCpyfpteWVMZWkmaYhxRw,32
2
+ outerbounds/cli_main.py,sha256=e9UMnPysmc7gbrimq2I4KfltggyU7pw59Cn9aEguVcU,74
3
+ outerbounds/command_groups/__init__.py,sha256=QPWtj5wDRTINDxVUL7XPqG3HoxHNvYOg08EnuSZB2Hc,21
4
+ outerbounds/command_groups/cli.py,sha256=61VsBlPG2ykP_786eCyllqeM8DMhPAOfj2FhktrSd7k,207
5
+ outerbounds/command_groups/local_setup_cli.py,sha256=g_kkrlDGzYvZTm184pW6QwotpkcqBamB14kH_Kv8TbM,28685
6
+ outerbounds/command_groups/workstations_cli.py,sha256=VgydQzCas3mlAFyzZuanjl1E8Zh7pBrbKbbP6t6N2WU,18237
7
+ outerbounds/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
+ outerbounds/utils/kubeconfig.py,sha256=l1mUP1j9VIq3fsffi5bJ1Nk-hYlwd1dIqkpj7DvVS1E,7936
9
+ outerbounds/utils/metaflowconfig.py,sha256=6u9D4x-pQVCPKnmGkTg9uSSHrq4mGnWQl7TurwyV2e8,2945
10
+ outerbounds/utils/schema.py,sha256=nBuarFbdZu0LGhG0YkJ6pEIvdglfM_TO_W_Db2vksb0,2017
11
+ outerbounds-0.3.58.dist-info/METADATA,sha256=ADmikJlmX_lGTzMKbqEBXBSi_X_pBFTE6iZiZYLCEUw,1407
12
+ outerbounds-0.3.58.dist-info/WHEEL,sha256=vVCvjcmxuUltf8cYhJ0sJMRDLr1XsPuxEId8YDzbyCY,88
13
+ outerbounds-0.3.58.dist-info/entry_points.txt,sha256=7ye0281PKlvqxu15rjw60zKg2pMsXI49_A8BmGqIqBw,47
14
+ outerbounds-0.3.58.dist-info/RECORD,,
@@ -1,392 +0,0 @@
1
- import base64
2
- import hashlib
3
- import json
4
- import os
5
- import re
6
- import subprocess
7
- import sys
8
- import zlib
9
- from base64 import b64decode, b64encode
10
- from importlib.machinery import PathFinder
11
- from os import path
12
- from pathlib import Path
13
- from typing import Any, Callable, Dict, List
14
-
15
- import boto3
16
- import click
17
- import requests
18
- from requests.exceptions import HTTPError
19
-
20
- from ..utils import kubeconfig, metaflowconfig
21
- from ..utils.schema import (
22
- CommandStatus,
23
- OuterboundsCommandResponse,
24
- OuterboundsCommandStatus,
25
- )
26
-
27
- from .local_setup_cli import PERIMETER_CONFIG_URL_KEY
28
-
29
-
30
- @click.group()
31
- def cli(**kwargs):
32
- pass
33
-
34
-
35
- @cli.command(help="Switch current perimeter")
36
- @click.option(
37
- "-d",
38
- "--config-dir",
39
- default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
40
- help="Path to Metaflow configuration directory",
41
- show_default=True,
42
- )
43
- @click.option(
44
- "-p",
45
- "--profile",
46
- default=os.environ.get("METAFLOW_PROFILE", ""),
47
- help="The named metaflow profile in which your workstation exists",
48
- )
49
- @click.option(
50
- "-o",
51
- "--output",
52
- default="",
53
- help="Show output in the specified format.",
54
- type=click.Choice(["json", ""]),
55
- )
56
- @click.option("--id", default="", type=str, help="Perimeter name to switch to")
57
- @click.option(
58
- "-f",
59
- "--force",
60
- is_flag=True,
61
- help="Force change the existing perimeter",
62
- default=False,
63
- )
64
- def switch_perimeter(config_dir=None, profile=None, output="", id=None, force=False):
65
- switch_perimeter_response = OuterboundsCommandResponse()
66
-
67
- switch_perimeter_step = CommandStatus(
68
- "SwitchPerimeter",
69
- OuterboundsCommandStatus.OK,
70
- "Perimeter was successfully switched!",
71
- )
72
-
73
- perimeters = get_perimeters_from_api_or_fail_command(
74
- config_dir, profile, output, switch_perimeter_response, switch_perimeter_step
75
- )
76
- confirm_user_has_access_to_perimeter_or_fail(
77
- id, perimeters, output, switch_perimeter_response, switch_perimeter_step
78
- )
79
-
80
- path_to_config = get_ob_config_file_path(config_dir, profile)
81
-
82
- import fcntl
83
-
84
- try:
85
- if os.path.exists(path_to_config):
86
- if not force:
87
- fd = os.open(path_to_config, os.O_WRONLY)
88
- # Try to acquire an exclusive lock
89
- fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
90
- else:
91
- click.secho(
92
- "Force flag is set. Perimeter will be switched, but can have unintended consequences on other running processes.",
93
- fg="yellow",
94
- err=True,
95
- )
96
-
97
- ob_config_dict = {
98
- "OB_CURRENT_PERIMETER": str(id),
99
- PERIMETER_CONFIG_URL_KEY: perimeters[id]["remote_config_url"],
100
- }
101
-
102
- # Now that we have the lock, we can safely write to the file
103
- with open(path_to_config, "w") as file:
104
- json.dump(ob_config_dict, file, indent=4)
105
-
106
- click.secho("Perimeter switched to {}".format(id), fg="green", err=True)
107
- except BlockingIOError:
108
- # This exception is raised if the file is already locked (non-blocking mode)
109
- # Note that its the metaflow package (the extension actually) that acquires a shared read lock
110
- # on the file whenever a process imports metaflow.
111
- # In the future we might want to get smarter about it and show which process is holding the lock.
112
- click.secho(
113
- "Can't switch perimeter while Metaflow is in use. Please make sure there are no running python processes or notebooks using metaflow.",
114
- fg="red",
115
- err=True,
116
- )
117
- switch_perimeter_step.update(
118
- status=OuterboundsCommandStatus.FAIL,
119
- reason="Can't switch perimeter while Metaflow is in use.",
120
- mitigation="Please make sure there are no running python processes or notebooks using metaflow.",
121
- )
122
-
123
- switch_perimeter_response.add_step(switch_perimeter_step)
124
- if output == "json":
125
- click.echo(json.dumps(switch_perimeter_response.as_dict(), indent=4))
126
- return
127
-
128
-
129
- @cli.command(help="Show current perimeter")
130
- @click.option(
131
- "-d",
132
- "--config-dir",
133
- default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
134
- help="Path to Metaflow configuration directory",
135
- show_default=True,
136
- )
137
- @click.option(
138
- "-p",
139
- "--profile",
140
- default=os.environ.get("METAFLOW_PROFILE", ""),
141
- help="Configure a named profile. Activate the profile by setting "
142
- "`METAFLOW_PROFILE` environment variable.",
143
- )
144
- @click.option(
145
- "-o",
146
- "--output",
147
- default="",
148
- help="Show output in the specified format.",
149
- type=click.Choice(["json", ""]),
150
- )
151
- def show_current_perimeter(config_dir=None, profile=None, output=""):
152
- show_current_perimeter_response = OuterboundsCommandResponse()
153
-
154
- show_current_perimeter_step = CommandStatus(
155
- "ShowCurrentPerimeter",
156
- OuterboundsCommandStatus.OK,
157
- "Current Perimeter Fetch Successful.",
158
- )
159
-
160
- ob_config_dict = get_ob_config_or_fail_command(
161
- config_dir,
162
- profile,
163
- output,
164
- show_current_perimeter_response,
165
- show_current_perimeter_step,
166
- )
167
-
168
- perimeters = get_perimeters_from_api_or_fail_command(
169
- config_dir,
170
- profile,
171
- output,
172
- show_current_perimeter_response,
173
- show_current_perimeter_step,
174
- )
175
- confirm_user_has_access_to_perimeter_or_fail(
176
- ob_config_dict["OB_CURRENT_PERIMETER"],
177
- perimeters,
178
- output,
179
- show_current_perimeter_response,
180
- show_current_perimeter_step,
181
- )
182
-
183
- click.secho(
184
- "Current Perimeter: {}".format(ob_config_dict["OB_CURRENT_PERIMETER"]),
185
- fg="green",
186
- err=True,
187
- )
188
-
189
- show_current_perimeter_response.add_or_update_data(
190
- "current_perimeter", ob_config_dict["OB_CURRENT_PERIMETER"]
191
- )
192
-
193
- if output == "json":
194
- click.echo(json.dumps(show_current_perimeter_response.as_dict(), indent=4))
195
-
196
-
197
- @cli.command(help="List all available perimeters")
198
- @click.option(
199
- "-d",
200
- "--config-dir",
201
- default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
202
- help="Path to Metaflow configuration directory",
203
- show_default=True,
204
- )
205
- @click.option(
206
- "-p",
207
- "--profile",
208
- default=os.environ.get("METAFLOW_PROFILE", ""),
209
- help="The named metaflow profile in which your workstation exists",
210
- )
211
- @click.option(
212
- "-o",
213
- "--output",
214
- default="",
215
- help="Show output in the specified format.",
216
- type=click.Choice(["json", ""]),
217
- )
218
- def list_perimeters(config_dir=None, profile=None, output=""):
219
- list_perimeters_response = OuterboundsCommandResponse()
220
-
221
- list_perimeters_step = CommandStatus(
222
- "ListPerimeters", OuterboundsCommandStatus.OK, "Perimeter Fetch Successful."
223
- )
224
-
225
- if "WORKSTATION_ID" in os.environ and (
226
- "OBP_DEFAULT_PERIMETER" not in os.environ
227
- or "OBP_DEFAULT_PERIMETER_URL" not in os.environ
228
- ):
229
- list_perimeters_response.update(
230
- OuterboundsCommandStatus.NOT_SUPPORTED,
231
- 500,
232
- "Perimeters are not supported on old workstations.",
233
- )
234
- click.secho(
235
- "Perimeters are not supported on old workstations.", err=True, fg="red"
236
- )
237
- if output == "json":
238
- click.echo(json.dumps(list_perimeters_response.as_dict(), indent=4))
239
- return
240
-
241
- ob_config_dict = get_ob_config_or_fail_command(
242
- config_dir, profile, output, list_perimeters_response, list_perimeters_step
243
- )
244
- active_perimeter = ob_config_dict["OB_CURRENT_PERIMETER"]
245
-
246
- perimeters = get_perimeters_from_api_or_fail_command(
247
- config_dir, profile, output, list_perimeters_response, list_perimeters_step
248
- )
249
-
250
- perimeter_list = []
251
- for perimeter in perimeters.values():
252
- status = "OK"
253
- perimeter_list.append(
254
- {
255
- "id": perimeter["perimeter"],
256
- "active": perimeter["perimeter"] == active_perimeter,
257
- "status": status,
258
- }
259
- )
260
- if perimeter["perimeter"] != active_perimeter:
261
- click.secho("Perimeter: {}".format(perimeter["perimeter"]), err=True)
262
- else:
263
- click.secho(
264
- "Perimeter: {} (active)".format(perimeter["perimeter"]),
265
- fg="green",
266
- err=True,
267
- )
268
-
269
- list_perimeters_response.add_or_update_data("perimeters", perimeter_list)
270
-
271
- if output == "json":
272
- click.echo(json.dumps(list_perimeters_response.as_dict(), indent=4))
273
-
274
-
275
- def get_list_perimeters_api_response(config_dir, profile):
276
- metaflow_token = metaflowconfig.get_metaflow_token_from_config(config_dir, profile)
277
- api_url = metaflowconfig.get_sanitized_url_from_config(
278
- config_dir, profile, "OBP_API_SERVER"
279
- )
280
- perimeters_response = requests.get(
281
- f"{api_url}/v1/me/perimeters?privilege=Execute",
282
- headers={"x-api-key": metaflow_token},
283
- )
284
- perimeters_response.raise_for_status()
285
- return perimeters_response.json()["perimeters"]
286
-
287
-
288
- def get_ob_config_file_path(config_dir: str, profile: str) -> str:
289
- # If OBP_CONFIG_DIR is set, use that, otherwise use METAFLOW_HOME
290
- # If neither are set, use ~/.metaflowconfig
291
- obp_config_dir = path.expanduser(os.environ.get("OBP_CONFIG_DIR", config_dir))
292
-
293
- ob_config_filename = f"ob_config_{profile}.json" if profile else "ob_config.json"
294
- return os.path.expanduser(os.path.join(obp_config_dir, ob_config_filename))
295
-
296
-
297
- def get_perimeters_from_api_or_fail_command(
298
- config_dir: str,
299
- profile: str,
300
- output: str,
301
- command_response: OuterboundsCommandResponse,
302
- command_step: CommandStatus,
303
- ) -> Dict[str, Dict[str, str]]:
304
- try:
305
- perimeters = get_list_perimeters_api_response(config_dir, profile)
306
- except:
307
- click.secho(
308
- "Failed to fetch perimeters from API.",
309
- fg="red",
310
- err=True,
311
- )
312
- command_step.update(
313
- status=OuterboundsCommandStatus.FAIL,
314
- reason="Failed to fetch perimeters from API",
315
- mitigation="",
316
- )
317
- command_response.add_step(command_step)
318
- if output == "json":
319
- click.echo(json.dumps(command_response.as_dict(), indent=4))
320
- sys.exit(1)
321
- return {p["perimeter"]: p for p in perimeters}
322
-
323
-
324
- def get_ob_config_or_fail_command(
325
- config_dir: str,
326
- profile: str,
327
- output: str,
328
- command_response: OuterboundsCommandResponse,
329
- command_step: CommandStatus,
330
- ) -> Dict[str, str]:
331
- path_to_config = get_ob_config_file_path(config_dir, profile)
332
-
333
- if not os.path.exists(path_to_config):
334
- click.secho(
335
- "Config file not found at {}".format(path_to_config), fg="red", err=True
336
- )
337
- command_step.update(
338
- status=OuterboundsCommandStatus.FAIL,
339
- reason="Config file not found",
340
- mitigation="Please make sure the config file exists at {}".format(
341
- path_to_config
342
- ),
343
- )
344
- command_response.add_step(command_step)
345
- if output == "json":
346
- click.echo(json.dumps(command_response.as_dict(), indent=4))
347
- sys.exit(1)
348
-
349
- with open(path_to_config, "r") as file:
350
- ob_config_dict = json.load(file)
351
-
352
- if "OB_CURRENT_PERIMETER" not in ob_config_dict:
353
- click.secho(
354
- "OB_CURRENT_PERIMETER not found in Config file: {}".format(path_to_config),
355
- fg="red",
356
- err=True,
357
- )
358
- command_step.update(
359
- status=OuterboundsCommandStatus.FAIL,
360
- reason="OB_CURRENT_PERIMETER not found in Config file: {}",
361
- mitigation="",
362
- )
363
- command_response.add_step(command_step)
364
- if output == "json":
365
- click.echo(json.dumps(command_response.as_dict(), indent=4))
366
- sys.exit(1)
367
-
368
- return ob_config_dict
369
-
370
-
371
- def confirm_user_has_access_to_perimeter_or_fail(
372
- perimeter_id: str,
373
- perimeters: Dict[str, Any],
374
- output: str,
375
- command_response: OuterboundsCommandResponse,
376
- command_step: CommandStatus,
377
- ):
378
- if perimeter_id not in perimeters:
379
- click.secho(
380
- f"You do not have access to perimeter {perimeter_id} or it does not exist.",
381
- fg="red",
382
- err=True,
383
- )
384
- command_step.update(
385
- status=OuterboundsCommandStatus.FAIL,
386
- reason=f"You do not have access to perimeter {perimeter_id} or it does not exist.",
387
- mitigation="",
388
- )
389
- command_response.add_step(command_step)
390
- if output == "json":
391
- click.echo(json.dumps(command_response.as_dict(), indent=4))
392
- sys.exit(1)
@@ -1,15 +0,0 @@
1
- outerbounds/__init__.py,sha256=GPdaubvAYF8pOFWJ3b-sPMKCpyfpteWVMZWkmaYhxRw,32
2
- outerbounds/cli_main.py,sha256=e9UMnPysmc7gbrimq2I4KfltggyU7pw59Cn9aEguVcU,74
3
- outerbounds/command_groups/__init__.py,sha256=QPWtj5wDRTINDxVUL7XPqG3HoxHNvYOg08EnuSZB2Hc,21
4
- outerbounds/command_groups/cli.py,sha256=H4LxcYTmsY9DQUrReSRLjvbg9s9Ro7s-eUrcMqEJ_9A,261
5
- outerbounds/command_groups/local_setup_cli.py,sha256=5xMBWZ3Py_Oint40Vq9YaQ69BJXRz27dt9ZJvs5fhvU,30559
6
- outerbounds/command_groups/perimeters_cli.py,sha256=HfmPa0LSu1YwauQrZ1sPI7kdIkqtlCxWQcHfBmmq3E8,12661
7
- outerbounds/command_groups/workstations_cli.py,sha256=b5lt8_g2B0zCoUoNriTRv32IPB6E4mI2sUhubDT7Yjo,21966
8
- outerbounds/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
9
- outerbounds/utils/kubeconfig.py,sha256=l1mUP1j9VIq3fsffi5bJ1Nk-hYlwd1dIqkpj7DvVS1E,7936
10
- outerbounds/utils/metaflowconfig.py,sha256=xiMl8TXSmPdQSJFfVXLUeRI2pPBgn3n7tPn-1fhmDm0,2878
11
- outerbounds/utils/schema.py,sha256=cNlgjmteLPbDzSEUSQDsq8txdhMGyezSmM83jU3aa0w,2329
12
- outerbounds-0.3.57rc1.dist-info/METADATA,sha256=0Lj3TQPpF2LTyS08l2XjJ0vBJYk5Tk3AEFZP6cRhMbI,1410
13
- outerbounds-0.3.57rc1.dist-info/WHEEL,sha256=vVCvjcmxuUltf8cYhJ0sJMRDLr1XsPuxEId8YDzbyCY,88
14
- outerbounds-0.3.57rc1.dist-info/entry_points.txt,sha256=7ye0281PKlvqxu15rjw60zKg2pMsXI49_A8BmGqIqBw,47
15
- outerbounds-0.3.57rc1.dist-info/RECORD,,