outerbounds 0.3.55rc8__py3-none-any.whl → 0.3.133__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. outerbounds/_vendor/PyYAML.LICENSE +20 -0
  2. outerbounds/_vendor/__init__.py +0 -0
  3. outerbounds/_vendor/_yaml/__init__.py +34 -0
  4. outerbounds/_vendor/click/__init__.py +73 -0
  5. outerbounds/_vendor/click/_compat.py +626 -0
  6. outerbounds/_vendor/click/_termui_impl.py +717 -0
  7. outerbounds/_vendor/click/_textwrap.py +49 -0
  8. outerbounds/_vendor/click/_winconsole.py +279 -0
  9. outerbounds/_vendor/click/core.py +2998 -0
  10. outerbounds/_vendor/click/decorators.py +497 -0
  11. outerbounds/_vendor/click/exceptions.py +287 -0
  12. outerbounds/_vendor/click/formatting.py +301 -0
  13. outerbounds/_vendor/click/globals.py +68 -0
  14. outerbounds/_vendor/click/parser.py +529 -0
  15. outerbounds/_vendor/click/py.typed +0 -0
  16. outerbounds/_vendor/click/shell_completion.py +580 -0
  17. outerbounds/_vendor/click/termui.py +787 -0
  18. outerbounds/_vendor/click/testing.py +479 -0
  19. outerbounds/_vendor/click/types.py +1073 -0
  20. outerbounds/_vendor/click/utils.py +580 -0
  21. outerbounds/_vendor/click.LICENSE +28 -0
  22. outerbounds/_vendor/vendor_any.txt +2 -0
  23. outerbounds/_vendor/yaml/__init__.py +471 -0
  24. outerbounds/_vendor/yaml/_yaml.cpython-311-darwin.so +0 -0
  25. outerbounds/_vendor/yaml/composer.py +146 -0
  26. outerbounds/_vendor/yaml/constructor.py +862 -0
  27. outerbounds/_vendor/yaml/cyaml.py +177 -0
  28. outerbounds/_vendor/yaml/dumper.py +138 -0
  29. outerbounds/_vendor/yaml/emitter.py +1239 -0
  30. outerbounds/_vendor/yaml/error.py +94 -0
  31. outerbounds/_vendor/yaml/events.py +104 -0
  32. outerbounds/_vendor/yaml/loader.py +62 -0
  33. outerbounds/_vendor/yaml/nodes.py +51 -0
  34. outerbounds/_vendor/yaml/parser.py +629 -0
  35. outerbounds/_vendor/yaml/reader.py +208 -0
  36. outerbounds/_vendor/yaml/representer.py +378 -0
  37. outerbounds/_vendor/yaml/resolver.py +245 -0
  38. outerbounds/_vendor/yaml/scanner.py +1555 -0
  39. outerbounds/_vendor/yaml/serializer.py +127 -0
  40. outerbounds/_vendor/yaml/tokens.py +129 -0
  41. outerbounds/command_groups/apps_cli.py +450 -0
  42. outerbounds/command_groups/cli.py +9 -5
  43. outerbounds/command_groups/local_setup_cli.py +247 -36
  44. outerbounds/command_groups/perimeters_cli.py +212 -32
  45. outerbounds/command_groups/tutorials_cli.py +111 -0
  46. outerbounds/command_groups/workstations_cli.py +2 -2
  47. outerbounds/utils/kubeconfig.py +2 -2
  48. outerbounds/utils/metaflowconfig.py +93 -16
  49. outerbounds/utils/schema.py +2 -2
  50. outerbounds/utils/utils.py +19 -0
  51. outerbounds/vendor.py +159 -0
  52. {outerbounds-0.3.55rc8.dist-info → outerbounds-0.3.133.dist-info}/METADATA +17 -6
  53. outerbounds-0.3.133.dist-info/RECORD +59 -0
  54. {outerbounds-0.3.55rc8.dist-info → outerbounds-0.3.133.dist-info}/WHEEL +1 -1
  55. outerbounds-0.3.55rc8.dist-info/RECORD +0 -15
  56. {outerbounds-0.3.55rc8.dist-info → outerbounds-0.3.133.dist-info}/entry_points.txt +0 -0
@@ -1,23 +1,15 @@
1
- import base64
2
- import hashlib
3
1
  import json
4
2
  import os
5
- import re
6
- import subprocess
7
3
  import sys
8
- import zlib
9
- from base64 import b64decode, b64encode
10
- from importlib.machinery import PathFinder
4
+ from io import StringIO
11
5
  from os import path
12
- from pathlib import Path
13
- from typing import Any, Callable, Dict, List
14
-
15
- import boto3
16
- import click
6
+ from typing import Any, Dict
7
+ from outerbounds._vendor import click
17
8
  import requests
18
- from requests.exceptions import HTTPError
9
+ import configparser
19
10
 
20
- from ..utils import kubeconfig, metaflowconfig
11
+ from ..utils import metaflowconfig
12
+ from ..utils.utils import safe_write_to_disk
21
13
  from ..utils.schema import (
22
14
  CommandStatus,
23
15
  OuterboundsCommandResponse,
@@ -32,7 +24,12 @@ def cli(**kwargs):
32
24
  pass
33
25
 
34
26
 
35
- @cli.command(help="Switch current perimeter", hidden=True)
27
+ @click.group(help="Manage perimeters")
28
+ def perimeter(**kwargs):
29
+ pass
30
+
31
+
32
+ @perimeter.command(help="Switch current perimeter")
36
33
  @click.option(
37
34
  "-d",
38
35
  "--config-dir",
@@ -61,7 +58,7 @@ def cli(**kwargs):
61
58
  help="Force change the existing perimeter",
62
59
  default=False,
63
60
  )
64
- def switch_perimeter(config_dir=None, profile=None, output="", id=None, force=False):
61
+ def switch(config_dir=None, profile=None, output="", id=None, force=False):
65
62
  switch_perimeter_response = OuterboundsCommandResponse()
66
63
 
67
64
  switch_perimeter_step = CommandStatus(
@@ -77,7 +74,7 @@ def switch_perimeter(config_dir=None, profile=None, output="", id=None, force=Fa
77
74
  id, perimeters, output, switch_perimeter_response, switch_perimeter_step
78
75
  )
79
76
 
80
- path_to_config = get_ob_config_file_path(config_dir, profile)
77
+ path_to_config = metaflowconfig.get_ob_config_file_path(config_dir, profile)
81
78
 
82
79
  import fcntl
83
80
 
@@ -121,12 +118,34 @@ def switch_perimeter(config_dir=None, profile=None, output="", id=None, force=Fa
121
118
  )
122
119
 
123
120
  switch_perimeter_response.add_step(switch_perimeter_step)
121
+
122
+ ensure_cloud_creds_step = CommandStatus(
123
+ "EnsureCloudCredentials",
124
+ OuterboundsCommandStatus.OK,
125
+ "Cloud credentials were successfully updated.",
126
+ )
127
+
128
+ try:
129
+ ensure_cloud_credentials_for_shell(config_dir, profile)
130
+ except:
131
+ click.secho(
132
+ "Failed to update cloud credentials.",
133
+ fg="red",
134
+ err=True,
135
+ )
136
+ ensure_cloud_creds_step.update(
137
+ status=OuterboundsCommandStatus.FAIL,
138
+ reason="Failed to update cloud credentials.",
139
+ mitigation="",
140
+ )
141
+
142
+ switch_perimeter_response.add_step(ensure_cloud_creds_step)
143
+
124
144
  if output == "json":
125
145
  click.echo(json.dumps(switch_perimeter_response.as_dict(), indent=4))
126
- return
127
146
 
128
147
 
129
- @cli.command(help="Show current perimeter", hidden=True)
148
+ @perimeter.command(help="Show current perimeter")
130
149
  @click.option(
131
150
  "-d",
132
151
  "--config-dir",
@@ -148,7 +167,7 @@ def switch_perimeter(config_dir=None, profile=None, output="", id=None, force=Fa
148
167
  help="Show output in the specified format.",
149
168
  type=click.Choice(["json", ""]),
150
169
  )
151
- def show_current_perimeter(config_dir=None, profile=None, output=""):
170
+ def show_current(config_dir=None, profile=None, output=""):
152
171
  show_current_perimeter_response = OuterboundsCommandResponse()
153
172
 
154
173
  show_current_perimeter_step = CommandStatus(
@@ -194,7 +213,7 @@ def show_current_perimeter(config_dir=None, profile=None, output=""):
194
213
  click.echo(json.dumps(show_current_perimeter_response.as_dict(), indent=4))
195
214
 
196
215
 
197
- @cli.command(help="List all available perimeters", hidden=True)
216
+ @perimeter.command(help="List all available perimeters")
198
217
  @click.option(
199
218
  "-d",
200
219
  "--config-dir",
@@ -215,7 +234,7 @@ def show_current_perimeter(config_dir=None, profile=None, output=""):
215
234
  help="Show output in the specified format.",
216
235
  type=click.Choice(["json", ""]),
217
236
  )
218
- def list_perimeters(config_dir=None, profile=None, output=""):
237
+ def list(config_dir=None, profile=None, output=""):
219
238
  list_perimeters_response = OuterboundsCommandResponse()
220
239
 
221
240
  list_perimeters_step = CommandStatus(
@@ -272,6 +291,58 @@ def list_perimeters(config_dir=None, profile=None, output=""):
272
291
  click.echo(json.dumps(list_perimeters_response.as_dict(), indent=4))
273
292
 
274
293
 
294
+ @perimeter.command(
295
+ help="Ensure credentials for cloud are synced with perimeter", hidden=True
296
+ )
297
+ @click.option(
298
+ "-d",
299
+ "--config-dir",
300
+ default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
301
+ help="Path to Metaflow configuration directory",
302
+ show_default=True,
303
+ )
304
+ @click.option(
305
+ "-p",
306
+ "--profile",
307
+ default=os.environ.get("METAFLOW_PROFILE", ""),
308
+ help="The named metaflow profile in which your workstation exists",
309
+ )
310
+ @click.option(
311
+ "-o",
312
+ "--output",
313
+ default="",
314
+ help="Show output in the specified format.",
315
+ type=click.Choice(["json", ""]),
316
+ )
317
+ def ensure_cloud_creds(config_dir=None, profile=None, output=""):
318
+ ensure_cloud_creds_step = CommandStatus(
319
+ "EnsureCloudCredentials",
320
+ OuterboundsCommandStatus.OK,
321
+ "Cloud credentials were successfully updated.",
322
+ )
323
+
324
+ ensure_cloud_creds_response = OuterboundsCommandResponse()
325
+
326
+ try:
327
+ ensure_cloud_credentials_for_shell(config_dir, profile)
328
+ click.secho("Cloud credentials updated successfully.", fg="green", err=True)
329
+ except:
330
+ click.secho(
331
+ "Failed to update cloud credentials.",
332
+ fg="red",
333
+ err=True,
334
+ )
335
+ ensure_cloud_creds_step.update(
336
+ status=OuterboundsCommandStatus.FAIL,
337
+ reason="Failed to update cloud credentials.",
338
+ mitigation="",
339
+ )
340
+
341
+ ensure_cloud_creds_response.add_step(ensure_cloud_creds_step)
342
+ if output == "json":
343
+ click.echo(json.dumps(ensure_cloud_creds_response.as_dict(), indent=4))
344
+
345
+
275
346
  def get_list_perimeters_api_response(config_dir, profile):
276
347
  metaflow_token = metaflowconfig.get_metaflow_token_from_config(config_dir, profile)
277
348
  api_url = metaflowconfig.get_sanitized_url_from_config(
@@ -285,15 +356,6 @@ def get_list_perimeters_api_response(config_dir, profile):
285
356
  return perimeters_response.json()["perimeters"]
286
357
 
287
358
 
288
- def get_ob_config_file_path(config_dir: str, profile: str) -> str:
289
- # If OBP_CONFIG_DIR is set, use that, otherwise use METAFLOW_HOME
290
- # If neither are set, use ~/.metaflowconfig
291
- obp_config_dir = path.expanduser(os.environ.get("OBP_CONFIG_DIR", config_dir))
292
-
293
- ob_config_filename = f"ob_config_{profile}.json" if profile else "ob_config.json"
294
- return os.path.expanduser(os.path.join(obp_config_dir, ob_config_filename))
295
-
296
-
297
359
  def get_perimeters_from_api_or_fail_command(
298
360
  config_dir: str,
299
361
  profile: str,
@@ -328,7 +390,7 @@ def get_ob_config_or_fail_command(
328
390
  command_response: OuterboundsCommandResponse,
329
391
  command_step: CommandStatus,
330
392
  ) -> Dict[str, str]:
331
- path_to_config = get_ob_config_file_path(config_dir, profile)
393
+ path_to_config = metaflowconfig.get_ob_config_file_path(config_dir, profile)
332
394
 
333
395
  if not os.path.exists(path_to_config):
334
396
  click.secho(
@@ -368,6 +430,23 @@ def get_ob_config_or_fail_command(
368
430
  return ob_config_dict
369
431
 
370
432
 
433
+ def ensure_cloud_credentials_for_shell(config_dir, profile):
434
+ if "WORKSTATION_ID" not in os.environ:
435
+ # Naive check to see if we're running in workstation. No need to ensure anything
436
+ # if this is not a workstation.
437
+ return
438
+
439
+ mf_config = metaflowconfig.init_config(config_dir, profile)
440
+
441
+ # Currently we only support GCP. TODO: utkarsh to add support for AWS and Azure
442
+ if "METAFLOW_DEFAULT_GCP_CLIENT_PROVIDER" in mf_config:
443
+ # This is a GCP deployment.
444
+ ensure_gcp_cloud_creds(config_dir, profile)
445
+ elif "METAFLOW_DEFAULT_AWS_CLIENT_PROVIDER" in mf_config:
446
+ # This is an AWS deployment.
447
+ ensure_aws_cloud_creds(config_dir, profile)
448
+
449
+
371
450
  def confirm_user_has_access_to_perimeter_or_fail(
372
451
  perimeter_id: str,
373
452
  perimeters: Dict[str, Any],
@@ -390,3 +469,104 @@ def confirm_user_has_access_to_perimeter_or_fail(
390
469
  if output == "json":
391
470
  click.echo(json.dumps(command_response.as_dict(), indent=4))
392
471
  sys.exit(1)
472
+
473
+
474
+ def ensure_gcp_cloud_creds(config_dir, profile):
475
+ token_info = get_gcp_auth_credentials(config_dir, profile)
476
+ auth_url = metaflowconfig.get_sanitized_url_from_config(
477
+ config_dir, profile, "OBP_AUTH_SERVER"
478
+ )
479
+ metaflow_token = metaflowconfig.get_metaflow_token_from_config(config_dir, profile)
480
+
481
+ try:
482
+ # GOOGLE_APPLICATION_CREDENTIALS is a well known gcloud environment variable
483
+ credentials_file_loc = os.environ["GOOGLE_APPLICATION_CREDENTIALS"]
484
+ except KeyError:
485
+ # This is most likely an old workstation when these params weren't set. Do nothing.
486
+ # Alternatively, user might have deliberately unset it to use their own auth.
487
+ return
488
+
489
+ credentials_json = {
490
+ "type": "external_account",
491
+ "audience": f"//iam.googleapis.com/projects/{token_info['gcpProjectNumber']}/locations/global/workloadIdentityPools/{token_info['gcpWorkloadIdentityPool']}/providers/{token_info['gcpWorkloadIdentityPoolProvider']}",
492
+ "subject_token_type": "urn:ietf:params:oauth:token-type:jwt",
493
+ "token_url": "https://sts.googleapis.com/v1/token",
494
+ "service_account_impersonation_url": f"https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/{token_info['gcpServiceAccountEmail']}:generateAccessToken",
495
+ "credential_source": {
496
+ "url": f"{auth_url}/generate/gcp",
497
+ "headers": {"x-api-key": metaflow_token},
498
+ "format": {"type": "json", "subject_token_field_name": "token"},
499
+ },
500
+ }
501
+
502
+ safe_write_to_disk(credentials_file_loc, json.dumps(credentials_json))
503
+
504
+
505
+ def ensure_aws_cloud_creds(config_dir, profile):
506
+ token_info = get_aws_auth_credentials(config_dir, profile)
507
+
508
+ try:
509
+ token_file_loc = os.environ["OBP_AWS_WEB_IDENTITY_TOKEN_FILE"]
510
+
511
+ # AWS_CONFIG_FILE is a well known aws cli environment variable
512
+ config_file_loc = os.environ["AWS_CONFIG_FILE"]
513
+ except KeyError:
514
+ # This is most likely an old workstation when these params weren't set. Do nothing.
515
+ # Alternatively, user might have deliberately unset it to use their own auth.
516
+ return
517
+
518
+ aws_config = configparser.ConfigParser()
519
+ aws_config.read(config_file_loc)
520
+
521
+ aws_config["profile task"] = {
522
+ "role_arn": token_info["role_arn"],
523
+ "web_identity_token_file": token_file_loc,
524
+ }
525
+
526
+ if token_info.get("cspr_role_arn"):
527
+ # If CSPR role is present, then we need to use the task role (in the task profile)
528
+ # to assume the CSPR role.
529
+ aws_config["profile outerbounds"] = {
530
+ "role_arn": token_info["cspr_role_arn"],
531
+ "source_profile": "task",
532
+ }
533
+ else:
534
+ # If no CSPR role is present, just use the task profile as the outerbounds profile.
535
+ aws_config["profile outerbounds"] = aws_config["profile task"]
536
+
537
+ aws_config_string = StringIO()
538
+ aws_config.write(aws_config_string)
539
+
540
+ safe_write_to_disk(token_file_loc, token_info["token"])
541
+ safe_write_to_disk(config_file_loc, aws_config_string.getvalue())
542
+
543
+
544
+ def get_aws_auth_credentials(config_dir, profile): # pragma: no cover
545
+ token = metaflowconfig.get_metaflow_token_from_config(config_dir, profile)
546
+ auth_server_url = metaflowconfig.get_sanitized_url_from_config(
547
+ config_dir, profile, "OBP_AUTH_SERVER"
548
+ )
549
+
550
+ response = requests.get(
551
+ "{}/generate/aws".format(auth_server_url), headers={"x-api-key": token}
552
+ )
553
+ response.raise_for_status()
554
+
555
+ return response.json()
556
+
557
+
558
+ def get_gcp_auth_credentials(config_dir, profile):
559
+ token = metaflowconfig.get_metaflow_token_from_config(config_dir, profile)
560
+ auth_server_url = metaflowconfig.get_sanitized_url_from_config(
561
+ config_dir, profile, "OBP_AUTH_SERVER"
562
+ )
563
+
564
+ response = requests.get(
565
+ "{}/generate/gcp".format(auth_server_url), headers={"x-api-key": token}
566
+ )
567
+ response.raise_for_status()
568
+
569
+ return response.json()
570
+
571
+
572
+ cli.add_command(perimeter, name="perimeter")
@@ -0,0 +1,111 @@
1
+ import os
2
+ from outerbounds._vendor import click
3
+ import requests
4
+
5
+ import tarfile
6
+ import hashlib
7
+ import tempfile
8
+
9
+
10
+ @click.group()
11
+ def cli(**kwargs):
12
+ pass
13
+
14
+
15
+ @click.group(help="Manage tutorials curated by Outerbounds.", hidden=True)
16
+ def tutorials(**kwargs):
17
+ pass
18
+
19
+
20
+ @tutorials.command(help="Pull Outerbounds tutorials.")
21
+ @click.option(
22
+ "--url",
23
+ required=True,
24
+ help="URL to pull the tutorials from.",
25
+ type=str,
26
+ )
27
+ @click.option(
28
+ "--destination-dir",
29
+ help="Show output in the specified format.",
30
+ type=str,
31
+ required=True,
32
+ )
33
+ @click.option(
34
+ "--force-overwrite",
35
+ is_flag=True,
36
+ help="Overwrite all existing files across all tutorials.",
37
+ type=bool,
38
+ required=False,
39
+ default=False,
40
+ )
41
+ def pull(url="", destination_dir="", force_overwrite=False):
42
+ try:
43
+ secure_download_and_extract(
44
+ url, destination_dir, force_overwrite=force_overwrite
45
+ )
46
+ click.secho("Tutorials pulled successfully.", fg="green", err=True)
47
+ except Exception as e:
48
+ print(e)
49
+ click.secho(f"Failed to pull tutorials: {e}", fg="red", err=True)
50
+
51
+
52
+ def secure_download_and_extract(
53
+ url, dest_dir, expected_hash=None, force_overwrite=False
54
+ ):
55
+ """
56
+ Download a tar.gz file from a URL, verify its integrity, and extract its contents.
57
+
58
+ :param url: URL of the tar.gz file to download
59
+ :param dest_dir: Destination directory to extract the contents
60
+ :param expected_hash: Expected SHA256 hash of the file (optional)
61
+ """
62
+
63
+ with tempfile.TemporaryDirectory() as temp_dir:
64
+ temp_file = os.path.join(
65
+ temp_dir, hashlib.md5(url.encode()).hexdigest() + ".tar.gz"
66
+ )
67
+
68
+ # Download the file
69
+ try:
70
+ response = requests.get(url, stream=True, verify=True)
71
+ response.raise_for_status()
72
+
73
+ with open(temp_file, "wb") as f:
74
+ for chunk in response.iter_content(chunk_size=8192):
75
+ f.write(chunk)
76
+ except requests.exceptions.RequestException as e:
77
+ raise Exception(f"Failed to download file: {e}")
78
+
79
+ if expected_hash:
80
+ with open(temp_file, "rb") as f:
81
+ file_hash = hashlib.sha256(f.read()).hexdigest()
82
+ if file_hash != expected_hash:
83
+ raise Exception("File integrity check failed")
84
+
85
+ os.makedirs(dest_dir, exist_ok=True)
86
+
87
+ try:
88
+ with tarfile.open(temp_file, "r:gz") as tar:
89
+ # Keep track of new journeys to extract.
90
+ to_extract = []
91
+ members = tar.getmembers()
92
+ for member in members:
93
+ member_path = os.path.join(dest_dir, member.name)
94
+ # Check for any files trying to write outside the destination
95
+ if not os.path.abspath(member_path).startswith(
96
+ os.path.abspath(dest_dir)
97
+ ):
98
+ raise Exception("Attempted path traversal in tar file")
99
+ if not os.path.exists(member_path):
100
+ # The user might have modified the existing files, leave them untouched.
101
+ to_extract.append(member)
102
+
103
+ if force_overwrite:
104
+ tar.extractall(path=dest_dir)
105
+ else:
106
+ tar.extractall(path=dest_dir, members=to_extract)
107
+ except tarfile.TarError as e:
108
+ raise Exception(f"Failed to extract tar file: {e}")
109
+
110
+
111
+ cli.add_command(tutorials, name="tutorials")
@@ -1,5 +1,5 @@
1
- import click
2
- import yaml
1
+ from outerbounds._vendor import click
2
+ from outerbounds._vendor import yaml
3
3
  import requests
4
4
  import base64
5
5
  import datetime
@@ -1,6 +1,6 @@
1
1
  import os
2
- import yaml
3
- from yaml.scanner import ScannerError
2
+ from outerbounds._vendor import yaml
3
+ from outerbounds._vendor.yaml.scanner import ScannerError
4
4
  import subprocess
5
5
  from os import path
6
6
  import platform
@@ -1,25 +1,49 @@
1
+ from outerbounds._vendor import click
1
2
  import json
2
3
  import os
3
4
  import requests
4
5
  from os import path
5
- import requests
6
+ from typing import Dict, Union
7
+ import sys
8
+
9
+ """
10
+ key: perimeter specific URL to fetch the remote metaflow config from
11
+ value: the remote metaflow config
12
+ """
13
+ CACHED_REMOTE_METAFLOW_CONFIG: Dict[str, Dict[str, str]] = {}
14
+
6
15
 
16
+ CURRENT_PERIMETER_KEY = "OB_CURRENT_PERIMETER"
17
+ CURRENT_PERIMETER_URL = "OB_CURRENT_PERIMETER_MF_CONFIG_URL"
18
+ CURRENT_PERIMETER_URL_LEGACY_KEY = (
19
+ "OB_CURRENT_PERIMETER_URL" # For backwards compatibility with workstations.
20
+ )
7
21
 
8
- def init_config(config_dir="", profile="") -> dict:
22
+
23
+ def init_config(config_dir, profile) -> Dict[str, str]:
24
+ global CACHED_REMOTE_METAFLOW_CONFIG
9
25
  config = read_metaflow_config_from_filesystem(config_dir, profile)
10
26
 
11
- # This is new remote-metaflow config; fetch it from the URL
12
- if "OBP_METAFLOW_CONFIG_URL" in config:
13
- remote_config = init_config_from_url(
14
- config_dir, profile, config["OBP_METAFLOW_CONFIG_URL"]
15
- )
16
- remote_config["OBP_METAFLOW_CONFIG_URL"] = config["OBP_METAFLOW_CONFIG_URL"]
27
+ # Either user has an ob_config.json file with the perimeter URL
28
+ # or the default config on the filesystem has the config URL in it.
29
+ perimeter_specifc_url = get_perimeter_config_url_if_set_in_ob_config(
30
+ config_dir, profile
31
+ ) or config.get("OBP_METAFLOW_CONFIG_URL", "")
32
+
33
+ if perimeter_specifc_url != "":
34
+ if perimeter_specifc_url in CACHED_REMOTE_METAFLOW_CONFIG:
35
+ return CACHED_REMOTE_METAFLOW_CONFIG[perimeter_specifc_url]
36
+
37
+ remote_config = init_config_from_url(config_dir, profile, perimeter_specifc_url)
38
+ remote_config["OBP_METAFLOW_CONFIG_URL"] = perimeter_specifc_url
39
+
40
+ CACHED_REMOTE_METAFLOW_CONFIG[perimeter_specifc_url] = remote_config
17
41
  return remote_config
18
- # Legacy config, use from filesystem
42
+
19
43
  return config
20
44
 
21
45
 
22
- def init_config_from_url(config_dir, profile, url) -> dict:
46
+ def init_config_from_url(config_dir, profile, url) -> Dict[str, str]:
23
47
  config = read_metaflow_config_from_filesystem(config_dir, profile)
24
48
 
25
49
  if config is None or "METAFLOW_SERVICE_AUTH_KEY" not in config:
@@ -34,13 +58,9 @@ def init_config_from_url(config_dir, profile, url) -> dict:
34
58
  return remote_config
35
59
 
36
60
 
37
- def read_metaflow_config_from_filesystem(config_dir="", profile="") -> dict:
38
- profile = profile or os.environ.get("METAFLOW_PROFILE")
39
- config_dir = config_dir or os.path.expanduser(
40
- os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
41
- )
42
-
61
+ def read_metaflow_config_from_filesystem(config_dir, profile) -> Dict[str, str]:
43
62
  config_filename = f"config_{profile}.json" if profile else "config.json"
63
+
44
64
  path_to_config = os.path.join(config_dir, config_filename)
45
65
 
46
66
  if os.path.exists(path_to_config):
@@ -83,3 +103,60 @@ def get_sanitized_url_from_config(config_dir: str, profile: str, key: str) -> st
83
103
 
84
104
  url_in_config = url_in_config.rstrip("/")
85
105
  return url_in_config
106
+
107
+
108
+ def get_remote_metaflow_config_for_perimeter(
109
+ origin_token: str, perimeter: str, api_server: str
110
+ ):
111
+ try:
112
+ response = requests.get(
113
+ f"{api_server}/v1/perimeters/{perimeter}/metaflowconfigs/default",
114
+ headers={"x-api-key": origin_token},
115
+ )
116
+ response.raise_for_status()
117
+ config = response.json()["config"]
118
+ config["METAFLOW_SERVICE_AUTH_KEY"] = origin_token
119
+ return config
120
+ except Exception as e:
121
+ click.secho(
122
+ f"Failed to get metaflow config from {api_server}. Error: {str(e)}",
123
+ fg="red",
124
+ )
125
+ sys.exit(1)
126
+
127
+
128
+ def get_ob_config_file_path(config_dir: str, profile: str) -> str:
129
+ # If OBP_CONFIG_DIR is set, use that, otherwise use METAFLOW_HOME
130
+ # If neither are set, use ~/.metaflowconfig
131
+ obp_config_dir = path.expanduser(os.environ.get("OBP_CONFIG_DIR", config_dir))
132
+
133
+ ob_config_filename = f"ob_config_{profile}.json" if profile else "ob_config.json"
134
+ return os.path.expanduser(os.path.join(obp_config_dir, ob_config_filename))
135
+
136
+
137
+ def get_perimeter_config_url_if_set_in_ob_config(
138
+ config_dir: str, profile: str
139
+ ) -> Union[str, None]:
140
+ file_path = get_ob_config_file_path(config_dir, profile)
141
+
142
+ if os.path.exists(file_path):
143
+ with open(file_path, "r") as f:
144
+ ob_config = json.loads(f.read())
145
+
146
+ if CURRENT_PERIMETER_URL in ob_config:
147
+ return ob_config[CURRENT_PERIMETER_URL]
148
+ elif CURRENT_PERIMETER_URL_LEGACY_KEY in ob_config:
149
+ return ob_config[CURRENT_PERIMETER_URL_LEGACY_KEY]
150
+ else:
151
+ raise ValueError(
152
+ "{} does not contain the key {}".format(
153
+ file_path, CURRENT_PERIMETER_KEY
154
+ )
155
+ )
156
+ elif "OBP_CONFIG_DIR" in os.environ:
157
+ raise FileNotFoundError(
158
+ "Environment variable OBP_CONFIG_DIR is set to {} but this directory does not contain an ob_config.json file.".format(
159
+ os.environ["OBP_CONFIG_DIR"]
160
+ )
161
+ )
162
+ return None
@@ -59,14 +59,14 @@ class OuterboundsCommandResponse:
59
59
  if step.status == OuterboundsCommandStatus.FAIL:
60
60
  self.status = OuterboundsCommandStatus.FAIL
61
61
  self._code = 500
62
- self._message = "We found one or more errors with your installation."
62
+ self._message = "Encountered an error when trying to run command."
63
63
  elif (
64
64
  step.status == OuterboundsCommandStatus.WARN
65
65
  and self.status != OuterboundsCommandStatus.FAIL
66
66
  ):
67
67
  self.status = OuterboundsCommandStatus.WARN
68
68
  self._code = 200
69
- self._message = "We found one or more warnings with your installation."
69
+ self._message = "Encountered one or more warnings when running the command."
70
70
 
71
71
  def as_dict(self):
72
72
  self._data["steps"] = [step.as_dict() for step in self._steps]
@@ -0,0 +1,19 @@
1
+ import tempfile
2
+ import os
3
+
4
+ """
5
+ Writes the given data to file_loc. Ensures that the directory exists
6
+ and uses a temporary file to ensure that the file is written atomically.
7
+ """
8
+
9
+
10
+ def safe_write_to_disk(file_loc, data):
11
+ # Ensure the directory exists
12
+ os.makedirs(os.path.dirname(file_loc), exist_ok=True)
13
+
14
+ with tempfile.NamedTemporaryFile(
15
+ "w", dir=os.path.dirname(file_loc), delete=False
16
+ ) as f:
17
+ f.write(data)
18
+ tmp_file = f.name
19
+ os.rename(tmp_file, file_loc)