outerbounds 0.3.55rc8__py3-none-any.whl → 0.3.133__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. outerbounds/_vendor/PyYAML.LICENSE +20 -0
  2. outerbounds/_vendor/__init__.py +0 -0
  3. outerbounds/_vendor/_yaml/__init__.py +34 -0
  4. outerbounds/_vendor/click/__init__.py +73 -0
  5. outerbounds/_vendor/click/_compat.py +626 -0
  6. outerbounds/_vendor/click/_termui_impl.py +717 -0
  7. outerbounds/_vendor/click/_textwrap.py +49 -0
  8. outerbounds/_vendor/click/_winconsole.py +279 -0
  9. outerbounds/_vendor/click/core.py +2998 -0
  10. outerbounds/_vendor/click/decorators.py +497 -0
  11. outerbounds/_vendor/click/exceptions.py +287 -0
  12. outerbounds/_vendor/click/formatting.py +301 -0
  13. outerbounds/_vendor/click/globals.py +68 -0
  14. outerbounds/_vendor/click/parser.py +529 -0
  15. outerbounds/_vendor/click/py.typed +0 -0
  16. outerbounds/_vendor/click/shell_completion.py +580 -0
  17. outerbounds/_vendor/click/termui.py +787 -0
  18. outerbounds/_vendor/click/testing.py +479 -0
  19. outerbounds/_vendor/click/types.py +1073 -0
  20. outerbounds/_vendor/click/utils.py +580 -0
  21. outerbounds/_vendor/click.LICENSE +28 -0
  22. outerbounds/_vendor/vendor_any.txt +2 -0
  23. outerbounds/_vendor/yaml/__init__.py +471 -0
  24. outerbounds/_vendor/yaml/_yaml.cpython-311-darwin.so +0 -0
  25. outerbounds/_vendor/yaml/composer.py +146 -0
  26. outerbounds/_vendor/yaml/constructor.py +862 -0
  27. outerbounds/_vendor/yaml/cyaml.py +177 -0
  28. outerbounds/_vendor/yaml/dumper.py +138 -0
  29. outerbounds/_vendor/yaml/emitter.py +1239 -0
  30. outerbounds/_vendor/yaml/error.py +94 -0
  31. outerbounds/_vendor/yaml/events.py +104 -0
  32. outerbounds/_vendor/yaml/loader.py +62 -0
  33. outerbounds/_vendor/yaml/nodes.py +51 -0
  34. outerbounds/_vendor/yaml/parser.py +629 -0
  35. outerbounds/_vendor/yaml/reader.py +208 -0
  36. outerbounds/_vendor/yaml/representer.py +378 -0
  37. outerbounds/_vendor/yaml/resolver.py +245 -0
  38. outerbounds/_vendor/yaml/scanner.py +1555 -0
  39. outerbounds/_vendor/yaml/serializer.py +127 -0
  40. outerbounds/_vendor/yaml/tokens.py +129 -0
  41. outerbounds/command_groups/apps_cli.py +450 -0
  42. outerbounds/command_groups/cli.py +9 -5
  43. outerbounds/command_groups/local_setup_cli.py +247 -36
  44. outerbounds/command_groups/perimeters_cli.py +212 -32
  45. outerbounds/command_groups/tutorials_cli.py +111 -0
  46. outerbounds/command_groups/workstations_cli.py +2 -2
  47. outerbounds/utils/kubeconfig.py +2 -2
  48. outerbounds/utils/metaflowconfig.py +93 -16
  49. outerbounds/utils/schema.py +2 -2
  50. outerbounds/utils/utils.py +19 -0
  51. outerbounds/vendor.py +159 -0
  52. {outerbounds-0.3.55rc8.dist-info → outerbounds-0.3.133.dist-info}/METADATA +17 -6
  53. outerbounds-0.3.133.dist-info/RECORD +59 -0
  54. {outerbounds-0.3.55rc8.dist-info → outerbounds-0.3.133.dist-info}/WHEEL +1 -1
  55. outerbounds-0.3.55rc8.dist-info/RECORD +0 -15
  56. {outerbounds-0.3.55rc8.dist-info → outerbounds-0.3.133.dist-info}/entry_points.txt +0 -0
@@ -1,12 +1,16 @@
1
- import click
2
- from . import local_setup_cli
3
- from . import workstations_cli
4
- from . import perimeters_cli
1
+ from outerbounds._vendor import click
2
+ from . import local_setup_cli, workstations_cli, perimeters_cli, apps_cli, tutorials_cli
5
3
 
6
4
 
7
5
  @click.command(
8
6
  cls=click.CommandCollection,
9
- sources=[local_setup_cli.cli, workstations_cli.cli, perimeters_cli.cli],
7
+ sources=[
8
+ local_setup_cli.cli,
9
+ workstations_cli.cli,
10
+ perimeters_cli.cli,
11
+ apps_cli.cli,
12
+ tutorials_cli.cli,
13
+ ],
10
14
  )
11
15
  def cli(**kwargs):
12
16
  pass
@@ -11,9 +11,7 @@ from importlib.machinery import PathFinder
11
11
  from os import path
12
12
  from pathlib import Path
13
13
  from typing import Any, Callable, Dict, List
14
-
15
- import boto3
16
- import click
14
+ from outerbounds._vendor import click
17
15
  import requests
18
16
  from requests.exceptions import HTTPError
19
17
 
@@ -31,7 +29,8 @@ the Outerbounds platform.
31
29
  To remove that package, please try `python -m pip uninstall metaflow -y` or reach out to Outerbounds support.
32
30
  After uninstalling the Metaflow package, please reinstall the Outerbounds package using `python -m pip
33
31
  install outerbounds --force`.
34
- As always, please reach out to Outerbounds support for any questions."""
32
+ As always, please reach out to Outerbounds support for any questions.
33
+ """
35
34
 
36
35
  MISSING_EXTENSIONS_MESSAGE = (
37
36
  "The Outerbounds Platform extensions for Metaflow was not found."
@@ -56,11 +55,11 @@ class Narrator:
56
55
 
57
56
  def section_ok(self):
58
57
  if not self.verbose:
59
- click.secho("\U0001F600", err=True)
58
+ click.secho("\U00002705", err=True)
60
59
 
61
60
  def section_not_ok(self):
62
61
  if not self.verbose:
63
- click.secho("\U0001F641", err=True)
62
+ click.secho("\U0000274C", err=True)
64
63
 
65
64
  def announce_check(self, name):
66
65
  if self.verbose:
@@ -234,25 +233,33 @@ class ConfigEntrySpec:
234
233
  self.expected = expected
235
234
 
236
235
 
237
- def get_config_specs():
238
- return [
239
- ConfigEntrySpec(
240
- "METAFLOW_DATASTORE_SYSROOT_S3", "s3://[a-z0-9\-]+/metaflow[/]?"
241
- ),
242
- ConfigEntrySpec("METAFLOW_DATATOOLS_S3ROOT", "s3://[a-z0-9\-]+/data[/]?"),
236
+ def get_config_specs(default_datastore: str):
237
+ spec = [
243
238
  ConfigEntrySpec("METAFLOW_DEFAULT_AWS_CLIENT_PROVIDER", "obp", expected="obp"),
244
- ConfigEntrySpec("METAFLOW_DEFAULT_DATASTORE", "s3", expected="s3"),
245
239
  ConfigEntrySpec("METAFLOW_DEFAULT_METADATA", "service", expected="service"),
246
- ConfigEntrySpec(
247
- "METAFLOW_KUBERNETES_NAMESPACE", "jobs\-default", expected="jobs-default"
248
- ),
249
- ConfigEntrySpec("METAFLOW_KUBERNETES_SANDBOX_INIT_SCRIPT", "eval \$\(.*"),
250
- ConfigEntrySpec("METAFLOW_SERVICE_AUTH_KEY", "[a-zA-Z0-9!_\-\.]+"),
251
- ConfigEntrySpec("METAFLOW_SERVICE_URL", "https://metadata\..*"),
252
- ConfigEntrySpec("METAFLOW_UI_URL", "https://ui\..*"),
253
- ConfigEntrySpec("OBP_AUTH_SERVER", "auth\..*"),
240
+ ConfigEntrySpec("METAFLOW_KUBERNETES_NAMESPACE", r"jobs-.*"),
241
+ ConfigEntrySpec("METAFLOW_KUBERNETES_SANDBOX_INIT_SCRIPT", r"eval \$\(.*"),
242
+ ConfigEntrySpec("METAFLOW_SERVICE_AUTH_KEY", r"[a-zA-Z0-9!_\-\.]+"),
243
+ ConfigEntrySpec("METAFLOW_SERVICE_URL", r"https://metadata\..*"),
244
+ ConfigEntrySpec("METAFLOW_UI_URL", r"https://ui\..*"),
245
+ ConfigEntrySpec("OBP_AUTH_SERVER", r"auth\..*"),
254
246
  ]
255
247
 
248
+ if default_datastore == "s3":
249
+ spec.extend(
250
+ [
251
+ ConfigEntrySpec(
252
+ "METAFLOW_DATASTORE_SYSROOT_S3",
253
+ r"s3://[a-z0-9\-]+/metaflow(-[a-z0-9\-]+)?[/]?",
254
+ ),
255
+ ConfigEntrySpec(
256
+ "METAFLOW_DATATOOLS_S3ROOT",
257
+ r"s3://[a-z0-9\-]+/data(-[a-z0-9\-]+)?[/]?",
258
+ ),
259
+ ]
260
+ )
261
+ return spec
262
+
256
263
 
257
264
  def check_metaflow_config(narrator: Narrator) -> CommandStatus:
258
265
  narrator.announce_section("local Metaflow config")
@@ -263,8 +270,20 @@ def check_metaflow_config(narrator: Narrator) -> CommandStatus:
263
270
  mitigation="",
264
271
  )
265
272
 
266
- config = metaflowconfig.init_config()
267
- for spec in get_config_specs():
273
+ profile = os.environ.get("METAFLOW_PROFILE")
274
+ config_dir = os.path.expanduser(
275
+ os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
276
+ )
277
+
278
+ config = metaflowconfig.init_config(config_dir, profile)
279
+
280
+ if "OBP_METAFLOW_CONFIG_URL" in config:
281
+ # If the config is fetched from a remote source, not much to check
282
+ narrator.announce_check("config entry OBP_METAFLOW_CONFIG_URL")
283
+ narrator.ok()
284
+ return check_status
285
+
286
+ for spec in get_config_specs(config.get("METAFLOW_DEFAULT_DATASTORE", "")):
268
287
  narrator.announce_check("config entry " + spec.name)
269
288
  if spec.name not in config:
270
289
  reason = "Missing"
@@ -306,7 +325,12 @@ def check_metaflow_token(narrator: Narrator) -> CommandStatus:
306
325
  mitigation="",
307
326
  )
308
327
 
309
- config = metaflowconfig.init_config()
328
+ profile = os.environ.get("METAFLOW_PROFILE")
329
+ config_dir = os.path.expanduser(
330
+ os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
331
+ )
332
+
333
+ config = metaflowconfig.init_config(config_dir, profile)
310
334
  try:
311
335
  if "OBP_AUTH_SERVER" in config:
312
336
  k8s_response = requests.get(
@@ -365,7 +389,13 @@ def check_workstation_api_accessible(narrator: Narrator) -> CommandStatus:
365
389
  )
366
390
 
367
391
  try:
368
- config = metaflowconfig.init_config()
392
+ profile = os.environ.get("METAFLOW_PROFILE")
393
+ config_dir = os.path.expanduser(
394
+ os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
395
+ )
396
+
397
+ config = metaflowconfig.init_config(config_dir, profile)
398
+
369
399
  missing_keys = []
370
400
  if "METAFLOW_SERVICE_AUTH_KEY" not in config:
371
401
  missing_keys.append("METAFLOW_SERVICE_AUTH_KEY")
@@ -424,7 +454,13 @@ def check_kubeconfig_valid_for_workstations(narrator: Narrator) -> CommandStatus
424
454
  )
425
455
 
426
456
  try:
427
- config = metaflowconfig.init_config()
457
+ profile = os.environ.get("METAFLOW_PROFILE")
458
+ config_dir = os.path.expanduser(
459
+ os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
460
+ )
461
+
462
+ config = metaflowconfig.init_config(config_dir, profile)
463
+
428
464
  missing_keys = []
429
465
  if "METAFLOW_SERVICE_AUTH_KEY" not in config:
430
466
  missing_keys.append("METAFLOW_SERVICE_AUTH_KEY")
@@ -587,6 +623,7 @@ class ConfigurationWriter:
587
623
  self.decoded_config = None
588
624
  self.out_dir = out_dir
589
625
  self.profile = profile
626
+ self.selected_perimeter = None
590
627
 
591
628
  ob_config_dir = path.expanduser(os.getenv("OBP_CONFIG_DIR", out_dir))
592
629
  self.ob_config_path = path.join(
@@ -598,8 +635,11 @@ class ConfigurationWriter:
598
635
  self.decoded_config = deserialize(self.encoded_config)
599
636
 
600
637
  def process_decoded_config(self):
638
+ assert self.decoded_config is not None
601
639
  config_type = self.decoded_config.get("OB_CONFIG_TYPE", "inline")
602
640
  if config_type == "inline":
641
+ if "OBP_PERIMETER" in self.decoded_config:
642
+ self.selected_perimeter = self.decoded_config["OBP_PERIMETER"]
603
643
  if "OBP_METAFLOW_CONFIG_URL" in self.decoded_config:
604
644
  self.decoded_config = {
605
645
  "OBP_METAFLOW_CONFIG_URL": self.decoded_config[
@@ -618,6 +658,8 @@ class ConfigurationWriter:
618
658
  f"{str(e)} key is required for aws-ref config type"
619
659
  )
620
660
  try:
661
+ import boto3
662
+
621
663
  client = boto3.client("secretsmanager", region_name=region)
622
664
  response = client.get_secret_value(SecretId=secret_arn)
623
665
  self.decoded_config = json.loads(response["SecretBinary"])
@@ -635,6 +677,7 @@ class ConfigurationWriter:
635
677
  return path.join(self.out_dir, "config_{}.json".format(self.profile))
636
678
 
637
679
  def display(self):
680
+ assert self.decoded_config is not None
638
681
  # Create a copy so we can use the real config later, possibly
639
682
  display_config = dict()
640
683
  for k in self.decoded_config.keys():
@@ -649,6 +692,7 @@ class ConfigurationWriter:
649
692
  return self.confirm_overwrite_config(self.path())
650
693
 
651
694
  def write_config(self):
695
+ assert self.decoded_config is not None
652
696
  config_path = self.path()
653
697
  # TODO config contains auth token - restrict file/dir modes
654
698
  os.makedirs(os.path.dirname(config_path), exist_ok=True)
@@ -656,16 +700,13 @@ class ConfigurationWriter:
656
700
  with open(config_path, "w") as fd:
657
701
  json.dump(self.existing, fd, indent=4)
658
702
 
659
- # Every time a config is initialized, we should also reset the corresponding ob_config[_profile].json
660
- remote_config = metaflowconfig.init_config(self.out_dir, self.profile)
661
- if (
662
- "OBP_PERIMETER" in remote_config
663
- and "OBP_METAFLOW_CONFIG_URL" in remote_config
664
- ):
703
+ if self.selected_perimeter and "OBP_METAFLOW_CONFIG_URL" in self.decoded_config:
665
704
  with open(self.ob_config_path, "w") as fd:
666
705
  ob_config_dict = {
667
- "OB_CURRENT_PERIMETER": remote_config["OBP_PERIMETER"],
668
- PERIMETER_CONFIG_URL_KEY: remote_config["OBP_METAFLOW_CONFIG_URL"],
706
+ "OB_CURRENT_PERIMETER": self.selected_perimeter,
707
+ PERIMETER_CONFIG_URL_KEY: self.decoded_config[
708
+ "OBP_METAFLOW_CONFIG_URL"
709
+ ],
669
710
  }
670
711
  json.dump(ob_config_dict, fd, indent=4)
671
712
 
@@ -691,6 +732,64 @@ class ConfigurationWriter:
691
732
  return True
692
733
 
693
734
 
735
+ def get_gha_jwt(audience: str):
736
+ # These are specific environment variables that are set by GitHub Actions.
737
+ if (
738
+ "ACTIONS_ID_TOKEN_REQUEST_TOKEN" in os.environ
739
+ and "ACTIONS_ID_TOKEN_REQUEST_URL" in os.environ
740
+ ):
741
+ try:
742
+ response = requests.get(
743
+ url=os.environ["ACTIONS_ID_TOKEN_REQUEST_URL"],
744
+ headers={
745
+ "Authorization": f"Bearer {os.environ['ACTIONS_ID_TOKEN_REQUEST_TOKEN']}"
746
+ },
747
+ params={"audience": audience},
748
+ )
749
+ response.raise_for_status()
750
+ return response.json()["value"]
751
+ except Exception as e:
752
+ click.secho(
753
+ "Failed to fetch JWT token from GitHub Actions. Please make sure you are permission 'id-token: write' is set on the GHA jobs level.",
754
+ fg="red",
755
+ )
756
+ sys.exit(1)
757
+
758
+ click.secho(
759
+ "The --github-actions flag was set, but we didn't not find '$ACTIONS_ID_TOKEN_REQUEST_TOKEN' and '$ACTIONS_ID_TOKEN_REQUEST_URL' environment variables. Please make sure you are running this command in a GitHub Actions environment and with correct permissions as per https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-cloud-providers",
760
+ fg="red",
761
+ )
762
+ sys.exit(1)
763
+
764
+
765
+ def get_origin_token(
766
+ service_principal_name: str,
767
+ deployment: str,
768
+ perimeter: str,
769
+ token: str,
770
+ auth_server: str,
771
+ ):
772
+ try:
773
+ response = requests.get(
774
+ f"{auth_server}/generate/service-principal",
775
+ headers={"x-api-key": token},
776
+ data=json.dumps(
777
+ {
778
+ "servicePrincipalName": service_principal_name,
779
+ "deploymentName": deployment,
780
+ "perimeter": perimeter,
781
+ }
782
+ ),
783
+ )
784
+ response.raise_for_status()
785
+ return response.json()["token"]
786
+ except Exception as e:
787
+ click.secho(
788
+ f"Failed to get origin token from {auth_server}. Error: {str(e)}", fg="red"
789
+ )
790
+ sys.exit(1)
791
+
792
+
694
793
  @click.group(help="The Outerbounds Platform CLI", no_args_is_help=True)
695
794
  def cli(**kwargs):
696
795
  pass
@@ -732,7 +831,6 @@ def check(no_config, verbose, output, workstation=False):
732
831
  ]
733
832
  else:
734
833
  check_names = [
735
- "metaflow_config",
736
834
  "metaflow_token",
737
835
  "kubeconfig",
738
836
  "api_connectivity",
@@ -777,7 +875,7 @@ def check(no_config, verbose, output, workstation=False):
777
875
  )
778
876
  @click.argument("encoded_config", required=True)
779
877
  def configure(
780
- encoded_config=None, config_dir=None, profile=None, echo=None, force=False
878
+ encoded_config: str, config_dir=None, profile=None, echo=None, force=False
781
879
  ):
782
880
  writer = ConfigurationWriter(encoded_config, config_dir, profile)
783
881
  try:
@@ -799,3 +897,116 @@ def configure(
799
897
  except Exception as e:
800
898
  click.secho("Writing the configuration file '{}' failed.".format(writer.path()))
801
899
  click.secho("Error: {}".format(str(e)))
900
+
901
+
902
+ @cli.command(
903
+ help="Authenticate service principals using JWT minted by their IDPs and configure Metaflow"
904
+ )
905
+ @click.option(
906
+ "-n",
907
+ "--name",
908
+ default="",
909
+ help="The name of service principals to authenticate",
910
+ required=True,
911
+ )
912
+ @click.option(
913
+ "--deployment-domain",
914
+ default="",
915
+ help="The full domain of the target Outerbounds Platform deployment (eg. 'foo.obp.outerbounds.com')",
916
+ required=True,
917
+ )
918
+ @click.option(
919
+ "-p",
920
+ "--perimeter",
921
+ default="default",
922
+ help="The name of the perimeter to authenticate the service principal in",
923
+ )
924
+ @click.option(
925
+ "-t",
926
+ "--jwt-token",
927
+ default="",
928
+ help="The JWT token that will be used to authenticate against the OBP Auth Server.",
929
+ )
930
+ @click.option(
931
+ "--github-actions",
932
+ is_flag=True,
933
+ help="Set if the command is being run in a GitHub Actions environment. If both --jwt-token and --github-actions are specified the --github-actions flag will be ignored.",
934
+ )
935
+ @click.option(
936
+ "-d",
937
+ "--config-dir",
938
+ default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
939
+ help="Path to Metaflow configuration directory",
940
+ show_default=True,
941
+ )
942
+ @click.option(
943
+ "--profile",
944
+ default="",
945
+ help="Configure a named profile. Activate the profile by setting "
946
+ "`METAFLOW_PROFILE` environment variable.",
947
+ )
948
+ @click.option(
949
+ "-e",
950
+ "--echo",
951
+ is_flag=True,
952
+ help="Print decoded configuration to stdout",
953
+ )
954
+ @click.option(
955
+ "-f",
956
+ "--force",
957
+ is_flag=True,
958
+ help="Force overwrite of existing configuration",
959
+ )
960
+ def service_principal_configure(
961
+ name: str,
962
+ deployment_domain: str,
963
+ perimeter: str,
964
+ jwt_token="",
965
+ github_actions=False,
966
+ config_dir=None,
967
+ profile=None,
968
+ echo=None,
969
+ force=False,
970
+ ):
971
+ audience = f"https://{deployment_domain}"
972
+ if jwt_token == "" and github_actions:
973
+ jwt_token = get_gha_jwt(audience)
974
+
975
+ if jwt_token == "":
976
+ click.secho(
977
+ "No JWT token provided. Please provider either a valid jwt token or set --github-actions",
978
+ fg="red",
979
+ )
980
+ sys.exit(1)
981
+
982
+ auth_server = f"https://auth.{deployment_domain}"
983
+ deployment_name = deployment_domain.split(".")[0]
984
+ origin_token = get_origin_token(
985
+ name, deployment_name, perimeter, jwt_token, auth_server
986
+ )
987
+
988
+ api_server = f"https://api.{deployment_domain}"
989
+ metaflow_config = metaflowconfig.get_remote_metaflow_config_for_perimeter(
990
+ origin_token, perimeter, api_server
991
+ )
992
+
993
+ writer = ConfigurationWriter(serialize(metaflow_config), config_dir, profile)
994
+ try:
995
+ writer.decode()
996
+ except:
997
+ click.secho("Decoding the configuration text failed.", fg="red")
998
+ sys.exit(1)
999
+ try:
1000
+ writer.process_decoded_config()
1001
+ except DecodedConfigProcessingError as e:
1002
+ click.secho("Resolving the configuration remotely failed.", fg="red")
1003
+ click.secho(str(e), fg="magenta")
1004
+ sys.exit(1)
1005
+ try:
1006
+ if echo == True:
1007
+ writer.display()
1008
+ if force or writer.confirm_overwrite():
1009
+ writer.write_config()
1010
+ except Exception as e:
1011
+ click.secho("Writing the configuration file '{}' failed.".format(writer.path()))
1012
+ click.secho("Error: {}".format(str(e)))