numerai-cli 1.0.3__tar.gz → 1.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/PKG-INFO +2 -1
  2. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/README.md +1 -0
  3. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/__init__.py +1 -0
  4. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/constants.py +1 -7
  5. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/destroy_all.py +1 -0
  6. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/misc.py +47 -10
  7. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/node/__init__.py +19 -16
  8. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/node/config.py +18 -77
  9. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/node/destroy.py +2 -1
  10. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/node/test.py +2 -3
  11. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/setup.py +17 -13
  12. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/util/docker.py +37 -12
  13. numerai-cli-1.1.1/numerai/cli/util/terraform.py +78 -0
  14. {numerai-cli-1.0.3/numerai/examples/tournament-python3 → numerai-cli-1.1.1/numerai/examples/crypto-python3}/Dockerfile +3 -3
  15. numerai-cli-1.1.1/numerai/examples/crypto-python3/predict.py +105 -0
  16. numerai-cli-1.1.1/numerai/examples/crypto-python3/requirements.txt +5 -0
  17. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/examples/signals-python3/Dockerfile +4 -4
  18. numerai-cli-1.1.1/numerai/examples/signals-python3/predict.py +104 -0
  19. numerai-cli-1.1.1/numerai/examples/signals-python3/requirements.txt +5 -0
  20. numerai-cli-1.1.1/numerai/examples/tournament-python3/Dockerfile +35 -0
  21. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/examples/tournament-python3/predict.py +17 -7
  22. numerai-cli-1.1.1/numerai/examples/tournament-python3/requirements.txt +5 -0
  23. numerai-cli-1.1.1/numerai/examples/tournament-python3/train.py +10 -0
  24. numerai-cli-1.1.1/numerai/terraform/aws/aws/-locals.tf +4 -0
  25. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/aws/aws/cluster.tf +60 -16
  26. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/aws/aws/webhook.tf +1 -1
  27. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai_cli.egg-info/PKG-INFO +2 -1
  28. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai_cli.egg-info/SOURCES.txt +4 -1
  29. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/setup.py +1 -1
  30. numerai-cli-1.0.3/numerai/examples/signals-python3/data.py +0 -164
  31. numerai-cli-1.0.3/numerai/examples/signals-python3/predict.py +0 -160
  32. numerai-cli-1.0.3/numerai/examples/signals-python3/requirements.txt +0 -8
  33. numerai-cli-1.0.3/numerai/examples/signals-python3/train.py +0 -6
  34. numerai-cli-1.0.3/numerai/examples/tournament-python3/requirements.txt +0 -5
  35. numerai-cli-1.0.3/numerai/terraform/gcp/gcp/-locals.tf +0 -3
  36. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/MANIFEST.in +0 -0
  37. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/__init__.py +0 -0
  38. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/doctor.py +0 -0
  39. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/node/deploy.py +0 -0
  40. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/uninstall.py +0 -0
  41. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/upgrade.py +0 -0
  42. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/util/__init__.py +0 -0
  43. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/util/debug.py +0 -0
  44. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/util/files.py +0 -0
  45. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/util/keys.py +0 -0
  46. {numerai-cli-1.0.3/numerai/examples/tournament-python3 → numerai-cli-1.1.1/numerai/examples/signals-python3}/train.py +0 -0
  47. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/examples/tournament-rlang/Dockerfile +0 -0
  48. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/examples/tournament-rlang/install_packages.R +0 -0
  49. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/examples/tournament-rlang/main.R +0 -0
  50. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/aws/-inputs.tf +0 -0
  51. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/aws/-main.tf +0 -0
  52. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/aws/-outputs.tf +0 -0
  53. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/aws/aws/-inputs.tf +0 -0
  54. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/aws/aws/-outputs.tf +0 -0
  55. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/aws/aws/main.zip +0 -0
  56. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/aws/aws/network.tf +0 -0
  57. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/azure/cluster.tf +0 -0
  58. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/azure/input.tf +0 -0
  59. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/azure/locals.tf +0 -0
  60. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/azure/outputs.tf +0 -0
  61. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/azure/webhook.tf +0 -0
  62. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/azure_cron_trigger.zip +0 -0
  63. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/azure_trigger.zip +0 -0
  64. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/input.tf +0 -0
  65. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/main.tf +0 -0
  66. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/outputs.tf +0 -0
  67. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/registry.tf +0 -0
  68. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/-inputs.tf +0 -0
  69. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/-main.tf +0 -0
  70. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/-outputs.tf +0 -0
  71. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/cloud-function.zip +0 -0
  72. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/gcp/-inputs.tf +0 -0
  73. {numerai-cli-1.0.3/numerai/terraform/aws/aws → numerai-cli-1.1.1/numerai/terraform/gcp/gcp}/-locals.tf +0 -0
  74. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/gcp/-outputs.tf +0 -0
  75. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/gcp/cluster.tf +0 -0
  76. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/gcp/webhook.tf +0 -0
  77. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/gcp/workflow-source.yaml +0 -0
  78. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/registry.tf +0 -0
  79. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/services.tf +0 -0
  80. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai_cli.egg-info/dependency_links.txt +0 -0
  81. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai_cli.egg-info/entry_points.txt +0 -0
  82. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai_cli.egg-info/requires.txt +0 -0
  83. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai_cli.egg-info/top_level.txt +0 -0
  84. {numerai-cli-1.0.3 → numerai-cli-1.1.1}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: numerai-cli
3
- Version: 1.0.3
3
+ Version: 1.1.1
4
4
  Summary: A library for deploying Numer.ai Prediction Nodes.
5
5
  Home-page: https://github.com/numerai/numerai-cli
6
6
  Author: Numer.ai
@@ -11,6 +11,7 @@ Description: # numerai-cli
11
11
  [![PyPI](https://img.shields.io/pypi/v/numerai-cli.svg?color=brightgreen)](https://pypi.org/project/numerai-cli/)
12
12
 
13
13
  Welcome to the Numerai CLI for the [Numerai Tournament](https://docs.numer.ai/tournament/learn).
14
+ Numerai CLI (aka Compute Heavy) is a tool to help you set up cloud automation for your Numerai predictions.
14
15
  This README is designed to have EVERYTHING you need to setup and maintain a Numerai Compute Heavy node.
15
16
 
16
17
  This CLI runs on your local computer to configure a Numerai Prediction Node in the cloud.
@@ -3,6 +3,7 @@
3
3
  [![PyPI](https://img.shields.io/pypi/v/numerai-cli.svg?color=brightgreen)](https://pypi.org/project/numerai-cli/)
4
4
 
5
5
  Welcome to the Numerai CLI for the [Numerai Tournament](https://docs.numer.ai/tournament/learn).
6
+ Numerai CLI (aka Compute Heavy) is a tool to help you set up cloud automation for your Numerai predictions.
6
7
  This README is designed to have EVERYTHING you need to setup and maintain a Numerai Compute Heavy node.
7
8
 
8
9
  This CLI runs on your local computer to configure a Numerai Prediction Node in the cloud.
@@ -35,5 +35,6 @@ def main():
35
35
  numerai.add_command(upgrade.upgrade)
36
36
  numerai.add_command(misc.copy_example)
37
37
  numerai.add_command(misc.list_constants)
38
+ numerai.add_command(misc.add_volume_aws)
38
39
  numerai.add_command(destroy_all.destroy_all)
39
40
  numerai()
@@ -1,10 +1,9 @@
1
1
  """Constants for Numerai CLI"""
2
+
2
3
  import os
3
4
  import json
4
5
  from pathlib import Path
5
6
 
6
- TOURNAMENT_NUMERAI = 8
7
- TOURNAMENT_SIGNALS = 11
8
7
  PACKAGE_PATH = os.path.dirname(__file__)
9
8
  CONFIG_PATH = os.path.join(str(Path.home()), ".numerai")
10
9
  KEYS_PATH = os.path.join(CONFIG_PATH, ".keys")
@@ -65,11 +64,6 @@ DEFAULT_SETTINGS = {
65
64
  }
66
65
 
67
66
  CONSTANTS_STR = f"""Default values (not your configured node values):
68
-
69
- ---Tournament Numbers---
70
- TOURNAMENT_NUMERAI: {TOURNAMENT_NUMERAI}
71
- TOURNAMENT_SIGNALS: {TOURNAMENT_SIGNALS}
72
-
73
67
  ---Paths---
74
68
  PACKAGE_PATH: {PACKAGE_PATH}
75
69
  CONFIG_PATH: {CONFIG_PATH}
@@ -1,4 +1,5 @@
1
1
  """Destroy command for Numerai CLI"""
2
+
2
3
  import click
3
4
  from numerapi import base_api
4
5
 
@@ -1,9 +1,8 @@
1
- import json
2
-
3
- import click
4
-
5
1
  from numerai.cli.constants import *
6
2
  from numerai.cli.util import files
3
+ from numerai.cli.util.terraform import apply_terraform
4
+
5
+ import click
7
6
 
8
7
 
9
8
  @click.command()
@@ -12,13 +11,15 @@ from numerai.cli.util import files
12
11
  "-e",
13
12
  type=click.Choice(EXAMPLES),
14
13
  default=DEFAULT_EXAMPLE,
15
- help=f"Specify the example to copy, defaults to {DEFAULT_EXAMPLE}. " f"Options are {EXAMPLES}.",
14
+ help=f"Specify the example to copy, defaults to {DEFAULT_EXAMPLE}. "
15
+ f"Options are {EXAMPLES}.",
16
16
  )
17
17
  @click.option(
18
18
  "--dest",
19
19
  "-d",
20
20
  type=str,
21
- help=f"Destination folder to which example code is written. " f"Defaults to the name of the example.",
21
+ help=f"Destination folder to which example code is written. "
22
+ f"Defaults to the name of the example.",
22
23
  )
23
24
  @click.option("--verbose", "-v", is_flag=True)
24
25
  def copy_example(example, dest, verbose):
@@ -49,14 +50,50 @@ def list_constants():
49
50
  click.secho(
50
51
  f" {size} -> cpus: {preset[0] / 1024}, "
51
52
  f"mem: {preset[1] / 1024} GB {suffix}",
52
- fg="green"
53
- if size == DEFAULT_SIZE or size == DEFAULT_SIZE_GCP
54
- else "yellow",
53
+ fg=(
54
+ "green"
55
+ if size == DEFAULT_SIZE or size == DEFAULT_SIZE_GCP
56
+ else "yellow"
57
+ ),
55
58
  )
56
- click.secho("Due to GCP Cloud Run size constraints, 'mem' sizes are not allowed when using GCP.")
59
+ click.secho(
60
+ "Due to GCP Cloud Run size constraints, 'mem' sizes are not allowed when using GCP."
61
+ )
57
62
  click.secho(
58
63
  "For AWS, use one of these sizes, or specify your own CPU and Memory in cores and GB using --cpu and --memory options.\n"
59
64
  "See https://learn.microsoft.com/en-us/azure/container-apps/containers#configuration for Azure,\n"
60
65
  "or https://cloud.google.com/run/docs/configuring/services/memory-limits for GCP \n"
61
66
  "to learn more info about allowed size presets for those providers."
62
67
  )
68
+
69
+
70
+ @click.command()
71
+ @click.option(
72
+ "--size",
73
+ "-s",
74
+ type=int,
75
+ required=True,
76
+ help="Specify the volume size in GB you'd like your AWS nodes to share.",
77
+ )
78
+ @click.option("--verbose", "-v", is_flag=True)
79
+ def add_volume_aws(size, verbose):
80
+ """
81
+ Set the volume size for AWS nodes. This volume is shared by all nodes.
82
+ """
83
+ click.secho("Setting volume size for AWS nodes...", fg="yellow")
84
+ # get nodes config object
85
+ nodes_config = files.load_or_init_nodes()
86
+ print(nodes_config)
87
+ # set volume size for all nodes to same size
88
+ for node in nodes_config:
89
+ nodes_config[node]["volume"] = size
90
+ files.store_config(NODES_PATH, nodes_config)
91
+ files.copy_file(
92
+ NODES_PATH,
93
+ f"{CONFIG_PATH}/{PROVIDER_AWS}/",
94
+ force=True,
95
+ verbose=verbose,
96
+ )
97
+ click.secho(f"Applying terraform to add {size} GB volume...", fg="yellow")
98
+ apply_terraform(nodes_config, [PROVIDER_AWS], PROVIDER_AWS, verbose=verbose)
99
+ click.secho("Volume size updated successfully!", fg="green")
@@ -1,4 +1,5 @@
1
1
  """Init for node"""
2
+
2
3
  import json
3
4
  import logging
4
5
  import click
@@ -15,22 +16,22 @@ from numerai.cli.util.keys import get_numerai_keys
15
16
  # Setting azure's logging level "ERROR" to avoid spamming the terminal
16
17
 
17
18
 
18
- def get_models(signals):
19
- if signals:
20
- tournament = TOURNAMENT_SIGNALS
21
- name_prefix = "signals"
22
- else:
23
- tournament = TOURNAMENT_NUMERAI
24
- name_prefix = "numerai"
19
+ def tournaments_dict():
20
+ napi = base_api.Api()
21
+ tournaments = napi.raw_query('query { tournaments { name tournament } }')
22
+ return {t["tournament"]: t["name"] for t in tournaments["data"]["tournaments"]}
23
+
24
+
25
+ def get_models(tournament):
25
26
  napi = base_api.Api(*get_numerai_keys())
26
27
  models = napi.get_models(tournament)
27
-
28
+ name_prefix = tournaments_dict()[tournament]
28
29
  model_dict = {}
29
30
  for model_name, model_id in models.items():
30
31
  model_dict[model_name] = {
31
32
  "id": model_id,
32
33
  "name": f"{name_prefix}-{model_name}",
33
- "is_signals": signals,
34
+ "tournament": tournament,
34
35
  }
35
36
  return model_dict
36
37
 
@@ -43,16 +44,18 @@ def get_models(signals):
43
44
  type=str,
44
45
  prompt=True,
45
46
  help="The name of one of your models to configure the Prediction Node for."
46
- "It defaults to the first model returned from your account.",
47
+ " It defaults to the first model returned from your account.",
47
48
  )
48
49
  @click.option(
49
- "--signals",
50
- "-s",
51
- is_flag=True,
52
- help="Target a signals model with this name. Defaults to false.",
50
+ "--tournament",
51
+ "-t",
52
+ default=8,
53
+ help="Target a specific tournament number."
54
+ " Defaults to Numerai Tournament/Classic."
55
+ f" Available tournaments: {json.dumps(tournaments_dict(), indent=2)}",
53
56
  )
54
57
  @click.pass_context
55
- def node(ctx, verbose, model_name, signals):
58
+ def node(ctx, verbose, model_name, tournament):
56
59
  """
57
60
  Commands to manage and test Prediction Nodes.
58
61
  """
@@ -68,7 +71,7 @@ def node(ctx, verbose, model_name, signals):
68
71
  else:
69
72
  logger.setLevel(logging.ERROR)
70
73
 
71
- models = get_models(signals)
74
+ models = get_models(tournament)
72
75
 
73
76
  try:
74
77
  ctx.ensure_object(dict)
@@ -1,7 +1,7 @@
1
1
  """Config command for Numerai CLI"""
2
- import json
2
+
3
3
  import os
4
- import click
4
+
5
5
  from numerapi import base_api
6
6
  from numerai.cli.constants import (
7
7
  DEFAULT_PROVIDER,
@@ -18,7 +18,6 @@ from numerai.cli.constants import (
18
18
  CONFIG_PATH,
19
19
  PROVIDER_GCP,
20
20
  )
21
- from numerai.cli.util.docker import terraform, check_for_dockerfile
22
21
  from numerai.cli.util import docker
23
22
  from numerai.cli.util.files import (
24
23
  load_or_init_nodes,
@@ -26,7 +25,14 @@ from numerai.cli.util.files import (
26
25
  copy_example,
27
26
  copy_file,
28
27
  )
29
- from numerai.cli.util.keys import get_provider_keys, get_numerai_keys, load_or_init_keys
28
+ from numerai.cli.util.keys import get_provider_keys, get_numerai_keys
29
+ from numerai.cli.util.terraform import (
30
+ apply_terraform,
31
+ create_azure_registry,
32
+ create_gcp_registry,
33
+ )
34
+
35
+ import click
30
36
 
31
37
 
32
38
  @click.command()
@@ -226,7 +232,7 @@ def config(
226
232
  click.secho(f'Current node config: "{node_conf}"...')
227
233
 
228
234
  # double check there is a dockerfile in the path we are about to configure
229
- check_for_dockerfile(nodes_config[node]["path"])
235
+ docker.check_for_dockerfile(nodes_config[node]["path"])
230
236
  store_config(NODES_PATH, nodes_config)
231
237
 
232
238
  # Added after tf directory restructure: copy nodes.json to providers' tf directory
@@ -262,12 +268,12 @@ def config(
262
268
  provider_registry_conf = create_gcp_registry(provider, verbose=verbose)
263
269
  node_conf.update(provider_registry_conf)
264
270
  registry_parts = node_conf["registry_id"].split("/")
265
- node_conf[
266
- "artifact_registry_login_url"
267
- ] = f"https://{registry_parts[3]}-docker.pkg.dev/"
268
- node_conf[
269
- "docker_repo"
270
- ] = f"{registry_parts[3]}-docker.pkg.dev/{registry_parts[1]}/numerai-container-registry/{node}:latest"
271
+ node_conf["artifact_registry_login_url"] = (
272
+ f"https://{registry_parts[3]}-docker.pkg.dev/"
273
+ )
274
+ node_conf["docker_repo"] = (
275
+ f"{registry_parts[3]}-docker.pkg.dev/{registry_parts[1]}/numerai-container-registry/{node}:latest"
276
+ )
271
277
  docker.login(node_conf, verbose)
272
278
  try:
273
279
  docker.manifest_inspect(node_conf["docker_repo"], verbose)
@@ -279,37 +285,7 @@ def config(
279
285
 
280
286
  store_config(NODES_PATH, nodes_config)
281
287
 
282
- # Apply terraform for any affected provider
283
- for affected_provider in affected_providers:
284
- if affected_provider in PROVIDERS:
285
- click.secho(f"Updating resources in {affected_provider}")
286
- terraform(
287
- "apply -auto-approve",
288
- verbose,
289
- affected_provider,
290
- env_vars=load_or_init_keys(affected_provider),
291
- inputs={"node_config_file": "nodes.json"},
292
- )
293
- else:
294
- click.secho(f"provider {affected_provider} not supported", fg="red")
295
- exit(1)
296
- click.secho("cloud resources created successfully", fg="green")
297
-
298
- # terraform output for node config, same for aws and azure
299
- click.echo(f"saving node configuration to {NODES_PATH}...")
300
-
301
- res = terraform(f"output -json {provider}_nodes", verbose, provider).decode("utf-8")
302
- try:
303
- nodes = json.loads(res)
304
- except json.JSONDecodeError:
305
- click.secho("failed to save node configuration, please retry.", fg="red")
306
- return
307
- for node_name, data in nodes.items():
308
- nodes_config[node_name].update(data)
309
-
310
- store_config(NODES_PATH, nodes_config)
311
- if verbose:
312
- click.secho(f"new config:\n{json.dumps(load_or_init_nodes(), indent=2)}")
288
+ apply_terraform(nodes_config, affected_providers, provider, verbose)
313
289
 
314
290
  webhook_url = nodes_config[node]["webhook_url"]
315
291
  napi = base_api.Api(*get_numerai_keys())
@@ -325,38 +301,3 @@ def config(
325
301
  "Prediction Node configured successfully. " "Next: deploy and test your node",
326
302
  fg="green",
327
303
  )
328
-
329
-
330
- def create_azure_registry(provider, provider_keys, verbose):
331
- """Creates a registry for azure"""
332
- terraform("init -upgrade", verbose, provider)
333
- terraform(
334
- 'apply -target="azurerm_container_registry.registry[0]" -target="azurerm_resource_group.acr_rg[0]" -auto-approve ',
335
- verbose,
336
- "azure",
337
- env_vars=provider_keys,
338
- inputs={"node_config_file": "nodes.json"},
339
- )
340
- res = terraform("output -json acr_repo_details", True, provider).decode("utf-8")
341
- return json.loads(res)
342
-
343
-
344
- def create_gcp_registry(provider, verbose):
345
- """Creates a registry for GCP"""
346
- terraform("init -upgrade", verbose, provider)
347
- terraform(
348
- 'apply -target="google_project_service.cloud_resource_manager" -auto-approve ',
349
- verbose,
350
- "gcp",
351
- inputs={"node_config_file": "nodes.json"},
352
- )
353
- terraform(
354
- 'apply -target="google_artifact_registry_repository.registry[0]" -auto-approve ',
355
- verbose,
356
- "gcp",
357
- inputs={"node_config_file": "nodes.json"},
358
- )
359
- res = terraform("output -json artifact_registry_details", True, provider).decode(
360
- "utf-8"
361
- )
362
- return json.loads(res)
@@ -1,4 +1,5 @@
1
1
  """Destroy command for Numerai CLI"""
2
+
2
3
  import click
3
4
  from numerapi import base_api
4
5
 
@@ -41,7 +42,7 @@ def destroy(ctx, preserve_node_config, verbose):
41
42
  fg="red",
42
43
  )
43
44
  return
44
-
45
+
45
46
  if not preserve_node_config:
46
47
  click.secho("backing up nodes.json...")
47
48
  copy_file(NODES_PATH, f"{NODES_PATH}.backup", force=True, verbose=True)
@@ -61,7 +61,7 @@ def test(ctx, local, command, verbose):
61
61
  ctx.ensure_object(dict)
62
62
  model = ctx.obj["model"]
63
63
  node = model["name"]
64
- is_signals = model["is_signals"]
64
+ tournament = model["tournament"]
65
65
  node_config = load_or_init_nodes(node)
66
66
  provider = node_config["provider"]
67
67
 
@@ -143,7 +143,6 @@ def test(ctx, local, command, verbose):
143
143
  variables={"modelId": node_config["model_id"]},
144
144
  authorization=True,
145
145
  )
146
- tournament = TOURNAMENT_SIGNALS if is_signals else TOURNAMENT_NUMERAI
147
146
  curr_round = api.get_current_round(tournament)
148
147
  latest_subs = sorted(
149
148
  filter(
@@ -338,7 +337,7 @@ def get_recent_task_status_aws(cluster_arn, ecs_client, node, trigger_id):
338
337
  "red",
339
338
  )
340
339
  elif matched_task["lastStatus"] in stopped_codes:
341
- return matched_task, True, "Job execution succeeded!\r", "green"
340
+ return matched_task, True, "Job execution finished!\r", "green"
342
341
  elif matched_task["lastStatus"] in pending_codes:
343
342
  return matched_task, False, "Waiting for job to start...", "yellow"
344
343
  elif matched_task["lastStatus"] in running_codes:
@@ -1,4 +1,5 @@
1
1
  """Setup command for Numerai CLI"""
2
+
2
3
  import click
3
4
  import logging
4
5
 
@@ -16,8 +17,9 @@ from numerai.cli.util.keys import config_numerai_keys, config_provider_keys
16
17
  prompt=True,
17
18
  help=f"Initialize with this providers API keys.",
18
19
  )
20
+ @click.option("--skip-key-setup", "-s", is_flag=True)
19
21
  @click.option("--verbose", "-v", is_flag=True)
20
- def setup(provider, verbose):
22
+ def setup(provider, skip_key_setup, verbose):
21
23
  """
22
24
  Initializes cli and provider API keys.
23
25
  """
@@ -38,20 +40,22 @@ def setup(provider, verbose):
38
40
  return
39
41
 
40
42
  # setup numerai keys
41
- click.secho(
42
- "Initializing numerai keys " "(press enter to keep value in brackets)...",
43
- fg="yellow",
44
- )
45
- maybe_create(KEYS_PATH, protected=True)
46
- config_numerai_keys()
43
+ if not skip_key_setup:
44
+ click.secho(
45
+ "Initializing numerai keys " "(press enter to keep value in brackets)...",
46
+ fg="yellow",
47
+ )
48
+ maybe_create(KEYS_PATH, protected=True)
49
+ config_numerai_keys()
47
50
 
48
51
  # setup provider keys
49
- click.secho(
50
- f"\nInitializing {provider} keys "
51
- f"(press enter to keep value in brackets)...",
52
- fg="yellow",
53
- )
54
- config_provider_keys(provider)
52
+ if not skip_key_setup:
53
+ click.secho(
54
+ f"\nInitializing {provider} keys "
55
+ f"(press enter to keep value in brackets)...",
56
+ fg="yellow",
57
+ )
58
+ config_provider_keys(provider)
55
59
 
56
60
  # copy tf files
57
61
  click.secho("copying terraform files...")
@@ -166,9 +166,16 @@ def build(node_config, node, verbose):
166
166
  build_arg_str += f" --build-arg NODE={node}"
167
167
 
168
168
  cmd = (
169
- f'docker build --platform=linux/amd64 -t {node_config["docker_repo"]}' f"{build_arg_str} -f {path}/Dockerfile ."
169
+ f'docker build --platform=linux/amd64 --load -t {node_config["docker_repo"]}'
170
+ f"{build_arg_str} -f {path}/Dockerfile ."
170
171
  )
171
- execute(cmd, verbose)
172
+ stdout, stderr = execute(cmd, verbose)
173
+ if "unknown flag: --load" in (stdout.decode() + stderr.decode()):
174
+ click.secho(
175
+ "Docker version too old, please upgrade to at least 18.09",
176
+ fg="red",
177
+ )
178
+ exit(1)
172
179
 
173
180
 
174
181
  def run(node_config, verbose, command=""):
@@ -181,7 +188,9 @@ def login(node_config, verbose):
181
188
  username, password = login_aws()
182
189
  login_url = node_config['docker_repo']
183
190
  elif node_config["provider"] == PROVIDER_AZURE:
184
- username, password = login_azure(node_config["registry_rg_name"], node_config["registry_name"])
191
+ username, password = login_azure(
192
+ node_config["registry_rg_name"], node_config["registry_name"]
193
+ )
185
194
  login_url = node_config['docker_repo']
186
195
  elif node_config["provider"] == PROVIDER_GCP:
187
196
  username, password = login_gcp()
@@ -214,17 +223,23 @@ def login_aws():
214
223
  )
215
224
 
216
225
  token = ecr_client.get_authorization_token() # TODO: use registryIds
217
- username, password = base64.b64decode(token["authorizationData"][0]["authorizationToken"]).decode().split(":")
226
+ username, password = (
227
+ base64.b64decode(token["authorizationData"][0]["authorizationToken"])
228
+ .decode()
229
+ .split(":")
230
+ )
218
231
 
219
232
  return username, password
220
233
 
221
234
 
222
235
  def login_azure(resource_group_name, registry_name):
223
236
  azure_subs_id, azure_client, azure_tenant, azure_secret = get_azure_keys()
224
- credentials = ClientSecretCredential(client_id=azure_client, tenant_id=azure_tenant, client_secret=azure_secret)
225
- username_password = ContainerRegistryManagementClient(credentials, azure_subs_id).registries.list_credentials(
226
- resource_group_name, registry_name
237
+ credentials = ClientSecretCredential(
238
+ client_id=azure_client, tenant_id=azure_tenant, client_secret=azure_secret
227
239
  )
240
+ username_password = ContainerRegistryManagementClient(
241
+ credentials, azure_subs_id
242
+ ).registries.list_credentials(resource_group_name, registry_name)
228
243
  username = username_password.username
229
244
  password = username_password.passwords[0].value
230
245
  return username, password
@@ -288,24 +303,32 @@ def cleanup_aws(docker_repo):
288
303
 
289
304
  docker_repo_name = docker_repo.split("/")[-1]
290
305
 
291
- resp = ecr_client.list_images(repositoryName=docker_repo_name, filter={"tagStatus": "UNTAGGED"})
306
+ resp = ecr_client.list_images(
307
+ repositoryName=docker_repo_name, filter={"tagStatus": "UNTAGGED"}
308
+ )
292
309
 
293
310
  imageIds = resp["imageIds"]
294
311
  if len(imageIds) == 0:
295
312
  return []
296
313
 
297
- resp = ecr_client.batch_delete_image(repositoryName=docker_repo_name, imageIds=imageIds)
314
+ resp = ecr_client.batch_delete_image(
315
+ repositoryName=docker_repo_name, imageIds=imageIds
316
+ )
298
317
 
299
318
  return resp["imageIds"]
300
319
 
301
320
 
302
321
  def cleanup_azure(node_config):
303
322
  _, azure_client, azure_tenant, azure_secret = get_azure_keys()
304
- credentials = ClientSecretCredential(client_id=azure_client, tenant_id=azure_tenant, client_secret=azure_secret)
323
+ credentials = ClientSecretCredential(
324
+ client_id=azure_client, tenant_id=azure_tenant, client_secret=azure_secret
325
+ )
305
326
  acr_client = ContainerRegistryClient(node_config["acr_login_server"], credentials)
306
327
  docker_repo = node_config["docker_repo"]
307
328
  node_repo_name = [
308
- repo_name for repo_name in acr_client.list_repository_names() if repo_name == docker_repo.split("/")[-1]
329
+ repo_name
330
+ for repo_name in acr_client.list_repository_names()
331
+ if repo_name == docker_repo.split("/")[-1]
309
332
  ][0]
310
333
 
311
334
  # get all manifests, ordered by last update time
@@ -318,7 +341,9 @@ def cleanup_azure(node_config):
318
341
  # Remove all but the latest manifest
319
342
  removed_manifests = []
320
343
  for manifest in manifest_list[1:]:
321
- acr_client.update_manifest_properties(node_repo_name, manifest.digest, can_write=True, can_delete=True)
344
+ acr_client.update_manifest_properties(
345
+ node_repo_name, manifest.digest, can_write=True, can_delete=True
346
+ )
322
347
  removed_manifests.append(manifest.digest)
323
348
  acr_client.delete_manifest(node_repo_name, manifest.digest)
324
349
  return removed_manifests
@@ -0,0 +1,78 @@
1
+ import json
2
+
3
+ from numerai.cli.constants import PROVIDERS, NODES_PATH
4
+ from numerai.cli.util.docker import terraform
5
+ from numerai.cli.util import docker
6
+ from numerai.cli.util.files import load_or_init_nodes, store_config
7
+ from numerai.cli.util.keys import load_or_init_keys
8
+
9
+ import click
10
+
11
+
12
+ def apply_terraform(nodes_config, affected_providers, provider, verbose):
13
+ # Apply terraform for any affected provider
14
+ for affected_provider in affected_providers:
15
+ if affected_provider in PROVIDERS:
16
+ click.secho(f"Updating resources in {affected_provider}")
17
+ terraform(
18
+ "apply -auto-approve",
19
+ verbose,
20
+ affected_provider,
21
+ env_vars=load_or_init_keys(affected_provider),
22
+ inputs={"node_config_file": "nodes.json"},
23
+ )
24
+ else:
25
+ click.secho(f"provider {affected_provider} not supported", fg="red")
26
+ exit(1)
27
+ click.secho("cloud resources created successfully", fg="green")
28
+
29
+ # terraform output for node config, same for aws and azure
30
+ click.echo(f"saving node configuration to {NODES_PATH}...")
31
+
32
+ res = terraform(f"output -json {provider}_nodes", verbose, provider).decode("utf-8")
33
+ try:
34
+ nodes = json.loads(res)
35
+ except json.JSONDecodeError:
36
+ click.secho("failed to save node configuration, please retry.", fg="red")
37
+ return
38
+ for node_name, data in nodes.items():
39
+ nodes_config[node_name].update(data)
40
+
41
+ store_config(NODES_PATH, nodes_config)
42
+ if verbose:
43
+ click.secho(f"new config:\n{json.dumps(load_or_init_nodes(), indent=2)}")
44
+
45
+
46
+ def create_azure_registry(provider, provider_keys, verbose):
47
+ """Creates a registry for azure"""
48
+ terraform("init -upgrade", verbose, provider)
49
+ terraform(
50
+ 'apply -target="azurerm_container_registry.registry[0]" -target="azurerm_resource_group.acr_rg[0]" -auto-approve ',
51
+ verbose,
52
+ "azure",
53
+ env_vars=provider_keys,
54
+ inputs={"node_config_file": "nodes.json"},
55
+ )
56
+ res = terraform("output -json acr_repo_details", True, provider).decode("utf-8")
57
+ return json.loads(res)
58
+
59
+
60
+ def create_gcp_registry(provider, verbose):
61
+ """Creates a registry for GCP"""
62
+ terraform("init -upgrade", verbose, provider)
63
+ terraform(
64
+ 'apply -target="google_project_service.cloud_resource_manager" -auto-approve ',
65
+ verbose,
66
+ "gcp",
67
+ inputs={"node_config_file": "nodes.json"},
68
+ )
69
+ terraform(
70
+ 'apply -target="google_artifact_registry_repository.registry[0]" -auto-approve ',
71
+ verbose,
72
+ "gcp",
73
+ inputs={"node_config_file": "nodes.json"},
74
+ )
75
+ res = terraform("output -json artifact_registry_details", True, provider).decode(
76
+ "utf-8"
77
+ )
78
+ return json.loads(res)
@@ -4,9 +4,9 @@ FROM python:3.9
4
4
  # These are docker arguments that `numerai node deploy/test` will always pass into docker.
5
5
  # They are then set in your environment so that numerapi can access them when uploading submissions.
6
6
  # You can also access them from your script like so:
7
- # import os
8
- # public_id = os.environ["NUMERAI_PUBLIC_ID"]
9
- # secret_key = os.environ["NUMERAI_SECRET_KEY"]
7
+ # import os
8
+ # public_id = os.environ["NUMERAI_PUBLIC_ID"]
9
+ # secret_key = os.environ["NUMERAI_SECRET_KEY"]
10
10
  ARG NUMERAI_PUBLIC_ID
11
11
  ENV NUMERAI_PUBLIC_ID=$NUMERAI_PUBLIC_ID
12
12