numerai-cli 1.0.2__tar.gz → 1.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/PKG-INFO +2 -1
  2. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/README.md +1 -0
  3. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/cli/constants.py +1 -7
  4. numerai-cli-1.1.0/numerai/cli/destroy_all.py +85 -0
  5. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/cli/node/__init__.py +20 -16
  6. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/cli/node/config.py +55 -16
  7. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/cli/node/destroy.py +25 -26
  8. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/cli/node/test.py +2 -3
  9. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/cli/util/debug.py +6 -5
  10. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/cli/util/docker.py +37 -12
  11. {numerai-cli-1.0.2/numerai/examples/tournament-python3 → numerai-cli-1.1.0/numerai/examples/crypto-python3}/Dockerfile +3 -3
  12. numerai-cli-1.1.0/numerai/examples/crypto-python3/predict.py +105 -0
  13. numerai-cli-1.1.0/numerai/examples/crypto-python3/requirements.txt +5 -0
  14. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/examples/signals-python3/Dockerfile +4 -4
  15. numerai-cli-1.1.0/numerai/examples/signals-python3/predict.py +104 -0
  16. numerai-cli-1.1.0/numerai/examples/signals-python3/requirements.txt +5 -0
  17. numerai-cli-1.1.0/numerai/examples/tournament-python3/Dockerfile +35 -0
  18. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/examples/tournament-python3/predict.py +17 -7
  19. numerai-cli-1.1.0/numerai/examples/tournament-python3/requirements.txt +5 -0
  20. numerai-cli-1.1.0/numerai/examples/tournament-python3/train.py +10 -0
  21. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/aws/-inputs.tf +6 -0
  22. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/aws/-main.tf +1 -0
  23. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/aws/aws/-inputs.tf +6 -0
  24. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/aws/aws/cluster.tf +50 -5
  25. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai_cli.egg-info/PKG-INFO +2 -1
  26. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai_cli.egg-info/SOURCES.txt +3 -1
  27. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/setup.py +1 -1
  28. numerai-cli-1.0.2/numerai/cli/destroy_all.py +0 -38
  29. numerai-cli-1.0.2/numerai/examples/signals-python3/data.py +0 -164
  30. numerai-cli-1.0.2/numerai/examples/signals-python3/predict.py +0 -160
  31. numerai-cli-1.0.2/numerai/examples/signals-python3/requirements.txt +0 -8
  32. numerai-cli-1.0.2/numerai/examples/signals-python3/train.py +0 -6
  33. numerai-cli-1.0.2/numerai/examples/tournament-python3/requirements.txt +0 -5
  34. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/MANIFEST.in +0 -0
  35. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/__init__.py +0 -0
  36. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/cli/__init__.py +0 -0
  37. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/cli/doctor.py +0 -0
  38. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/cli/misc.py +0 -0
  39. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/cli/node/deploy.py +0 -0
  40. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/cli/setup.py +0 -0
  41. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/cli/uninstall.py +0 -0
  42. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/cli/upgrade.py +0 -0
  43. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/cli/util/__init__.py +0 -0
  44. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/cli/util/files.py +0 -0
  45. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/cli/util/keys.py +0 -0
  46. {numerai-cli-1.0.2/numerai/examples/tournament-python3 → numerai-cli-1.1.0/numerai/examples/signals-python3}/train.py +0 -0
  47. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/examples/tournament-rlang/Dockerfile +0 -0
  48. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/examples/tournament-rlang/install_packages.R +0 -0
  49. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/examples/tournament-rlang/main.R +0 -0
  50. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/aws/-outputs.tf +0 -0
  51. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/aws/aws/-locals.tf +0 -0
  52. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/aws/aws/-outputs.tf +0 -0
  53. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/aws/aws/main.zip +0 -0
  54. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/aws/aws/network.tf +0 -0
  55. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/aws/aws/webhook.tf +0 -0
  56. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/azure/azure/cluster.tf +0 -0
  57. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/azure/azure/input.tf +0 -0
  58. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/azure/azure/locals.tf +0 -0
  59. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/azure/azure/outputs.tf +0 -0
  60. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/azure/azure/webhook.tf +0 -0
  61. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/azure/azure_cron_trigger.zip +0 -0
  62. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/azure/azure_trigger.zip +0 -0
  63. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/azure/input.tf +0 -0
  64. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/azure/main.tf +0 -0
  65. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/azure/outputs.tf +0 -0
  66. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/azure/registry.tf +0 -0
  67. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/gcp/-inputs.tf +0 -0
  68. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/gcp/-main.tf +0 -0
  69. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/gcp/-outputs.tf +0 -0
  70. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/gcp/cloud-function.zip +0 -0
  71. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/gcp/gcp/-inputs.tf +0 -0
  72. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/gcp/gcp/-locals.tf +0 -0
  73. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/gcp/gcp/-outputs.tf +0 -0
  74. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/gcp/gcp/cluster.tf +0 -0
  75. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/gcp/gcp/webhook.tf +0 -0
  76. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/gcp/gcp/workflow-source.yaml +0 -0
  77. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/gcp/registry.tf +0 -0
  78. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai/terraform/gcp/services.tf +0 -0
  79. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai_cli.egg-info/dependency_links.txt +0 -0
  80. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai_cli.egg-info/entry_points.txt +0 -0
  81. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai_cli.egg-info/requires.txt +0 -0
  82. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/numerai_cli.egg-info/top_level.txt +0 -0
  83. {numerai-cli-1.0.2 → numerai-cli-1.1.0}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: numerai-cli
3
- Version: 1.0.2
3
+ Version: 1.1.0
4
4
  Summary: A library for deploying Numer.ai Prediction Nodes.
5
5
  Home-page: https://github.com/numerai/numerai-cli
6
6
  Author: Numer.ai
@@ -11,6 +11,7 @@ Description: # numerai-cli
11
11
  [![PyPI](https://img.shields.io/pypi/v/numerai-cli.svg?color=brightgreen)](https://pypi.org/project/numerai-cli/)
12
12
 
13
13
  Welcome to the Numerai CLI for the [Numerai Tournament](https://docs.numer.ai/tournament/learn).
14
+ Numerai CLI (aka Compute Heavy) is a tool to help you set up cloud automation for your Numerai predictions.
14
15
  This README is designed to have EVERYTHING you need to setup and maintain a Numerai Compute Heavy node.
15
16
 
16
17
  This CLI runs on your local computer to configure a Numerai Prediction Node in the cloud.
@@ -3,6 +3,7 @@
3
3
  [![PyPI](https://img.shields.io/pypi/v/numerai-cli.svg?color=brightgreen)](https://pypi.org/project/numerai-cli/)
4
4
 
5
5
  Welcome to the Numerai CLI for the [Numerai Tournament](https://docs.numer.ai/tournament/learn).
6
+ Numerai CLI (aka Compute Heavy) is a tool to help you set up cloud automation for your Numerai predictions.
6
7
  This README is designed to have EVERYTHING you need to setup and maintain a Numerai Compute Heavy node.
7
8
 
8
9
  This CLI runs on your local computer to configure a Numerai Prediction Node in the cloud.
@@ -1,10 +1,9 @@
1
1
  """Constants for Numerai CLI"""
2
+
2
3
  import os
3
4
  import json
4
5
  from pathlib import Path
5
6
 
6
- TOURNAMENT_NUMERAI = 8
7
- TOURNAMENT_SIGNALS = 11
8
7
  PACKAGE_PATH = os.path.dirname(__file__)
9
8
  CONFIG_PATH = os.path.join(str(Path.home()), ".numerai")
10
9
  KEYS_PATH = os.path.join(CONFIG_PATH, ".keys")
@@ -65,11 +64,6 @@ DEFAULT_SETTINGS = {
65
64
  }
66
65
 
67
66
  CONSTANTS_STR = f"""Default values (not your configured node values):
68
-
69
- ---Tournament Numbers---
70
- TOURNAMENT_NUMERAI: {TOURNAMENT_NUMERAI}
71
- TOURNAMENT_SIGNALS: {TOURNAMENT_SIGNALS}
72
-
73
67
  ---Paths---
74
68
  PACKAGE_PATH: {PACKAGE_PATH}
75
69
  CONFIG_PATH: {CONFIG_PATH}
@@ -0,0 +1,85 @@
1
+ """Destroy command for Numerai CLI"""
2
+ import click
3
+ from numerapi import base_api
4
+
5
+ from numerai.cli.constants import *
6
+ from numerai.cli.util.docker import terraform
7
+ from numerai.cli.util.files import load_or_init_nodes, store_config, copy_file
8
+ from numerai.cli.util.keys import get_provider_keys, get_numerai_keys
9
+
10
+
11
+ @click.command("destroy-all", help="Destroy all nodes")
12
+ @click.option("--verbose", "-v", is_flag=True)
13
+ @click.option("--preserve-node-config", "-p", is_flag=True)
14
+ @click.pass_context
15
+ def destroy_all(ctx, verbose, preserve_node_config):
16
+ """
17
+ Uses Terraform to destroy a Numerai Compute clusters for both Tournament and Signals.
18
+ This will delete everything, including:
19
+ - lambda url
20
+ - docker container and associated task
21
+ - all logs
22
+ This command is idempotent and safe to run multiple times.
23
+ """
24
+ if not os.path.exists(CONFIG_PATH):
25
+ click.secho(".numerai directory not setup, run `numerai setup`...", fg="red")
26
+ return
27
+
28
+ if not click.prompt(
29
+ "THIS WILL DELETE ALL YOUR NODES, ARE YOU SURE? (y/n)",
30
+ ):
31
+ exit(0)
32
+
33
+ nodes_config = load_or_init_nodes()
34
+
35
+ if len(nodes_config) == 0:
36
+ click.secho("No nodes to destroy", fg="green")
37
+ return
38
+
39
+ try:
40
+ provider_keys = {
41
+ nodes_config[node]["provider"]: get_provider_keys(node)
42
+ for node in nodes_config
43
+ }
44
+ except (KeyError, FileNotFoundError) as e:
45
+ click.secho(
46
+ f"make sure you run `numerai setup` and " f"`numerai node config` first...",
47
+ fg="red",
48
+ )
49
+ return
50
+
51
+ click.secho("backing up nodes.json and deleting current config...")
52
+ copy_file(NODES_PATH, f"{NODES_PATH}.backup", force=True, verbose=True)
53
+ store_config(NODES_PATH, {})
54
+
55
+ try:
56
+ click.secho(f"destroying nodes...")
57
+ for provider, provider_keys in provider_keys.items():
58
+ click.secho(f"deleting cloud resources for {provider}...")
59
+ terraform(
60
+ "destroy -auto-approve",
61
+ verbose,
62
+ provider,
63
+ env_vars=provider_keys,
64
+ inputs={"node_config_file": "nodes.json"},
65
+ )
66
+
67
+ except Exception as e:
68
+ click.secho(e.__str__(), fg="red")
69
+ click.secho("restoring nodes.json...", fg="green")
70
+ store_config(NODES_PATH, nodes_config)
71
+ return
72
+
73
+ napi = base_api.Api(*get_numerai_keys())
74
+ for node, node_config in nodes_config.items():
75
+ if "model_id" in node_config and "webhook_url" in node_config:
76
+ model_id = node_config["model_id"]
77
+ webhook_url = node_config["webhook_url"]
78
+ click.echo(f"deregistering webhook {webhook_url} for model {model_id}...")
79
+ napi.set_submission_webhook(model_id, None)
80
+
81
+ click.secho("Prediction Nodes destroyed successfully", fg="green")
82
+
83
+ if preserve_node_config:
84
+ click.secho("restoring nodes.json...", fg="green")
85
+ store_config(NODES_PATH, nodes_config)
@@ -1,4 +1,5 @@
1
1
  """Init for node"""
2
+
2
3
  import json
3
4
  import logging
4
5
  import click
@@ -15,22 +16,23 @@ from numerai.cli.util.keys import get_numerai_keys
15
16
  # Setting azure's logging level "ERROR" to avoid spamming the terminal
16
17
 
17
18
 
18
- def get_models(signals):
19
- if signals:
20
- tournament = TOURNAMENT_SIGNALS
21
- name_prefix = "signals"
22
- else:
23
- tournament = TOURNAMENT_NUMERAI
24
- name_prefix = "numerai"
19
+ def tournaments_dict():
20
+ napi = base_api.Api()
21
+ tournaments = napi.raw_query('query { tournaments { name tournament } }')
22
+ return {t["tournament"]: t["name"] for t in tournaments["data"]["tournaments"]}
23
+
24
+
25
+ def get_models(tournament):
25
26
  napi = base_api.Api(*get_numerai_keys())
26
27
  models = napi.get_models(tournament)
27
-
28
+ tournaments = napi.raw_query('query { tournaments { name tournament } }')
29
+ name_prefix = tournaments_dict()[tournament]
28
30
  model_dict = {}
29
31
  for model_name, model_id in models.items():
30
32
  model_dict[model_name] = {
31
33
  "id": model_id,
32
34
  "name": f"{name_prefix}-{model_name}",
33
- "is_signals": signals,
35
+ "tournament": tournament,
34
36
  }
35
37
  return model_dict
36
38
 
@@ -43,16 +45,18 @@ def get_models(signals):
43
45
  type=str,
44
46
  prompt=True,
45
47
  help="The name of one of your models to configure the Prediction Node for."
46
- "It defaults to the first model returned from your account.",
48
+ " It defaults to the first model returned from your account.",
47
49
  )
48
50
  @click.option(
49
- "--signals",
50
- "-s",
51
- is_flag=True,
52
- help="Target a signals model with this name. Defaults to false.",
51
+ "--tournament",
52
+ "-t",
53
+ default=8,
54
+ help="Target a specific tournament number."
55
+ " Defaults to Numerai Tournament/Classic."
56
+ f" Available tournaments: {json.dumps(tournaments_dict(), indent=2)}",
53
57
  )
54
58
  @click.pass_context
55
- def node(ctx, verbose, model_name, signals):
59
+ def node(ctx, verbose, model_name, tournament):
56
60
  """
57
61
  Commands to manage and test Prediction Nodes.
58
62
  """
@@ -68,7 +72,7 @@ def node(ctx, verbose, model_name, signals):
68
72
  else:
69
73
  logger.setLevel(logging.ERROR)
70
74
 
71
- models = get_models(signals)
75
+ models = get_models(tournament)
72
76
 
73
77
  try:
74
78
  ctx.ensure_object(dict)
@@ -1,4 +1,5 @@
1
1
  """Config command for Numerai CLI"""
2
+
2
3
  import json
3
4
  import os
4
5
  import click
@@ -35,7 +36,8 @@ from numerai.cli.util.keys import get_provider_keys, get_numerai_keys, load_or_i
35
36
  "--provider",
36
37
  "-P",
37
38
  type=str,
38
- help=f"Select a cloud provider. One of {PROVIDERS}. " f"Defaults to {DEFAULT_PROVIDER}.",
39
+ help=f"Select a cloud provider. One of {PROVIDERS}. "
40
+ f"Defaults to {DEFAULT_PROVIDER}.",
39
41
  )
40
42
  @click.option(
41
43
  "--size",
@@ -47,7 +49,8 @@ from numerai.cli.util.keys import get_provider_keys, get_numerai_keys, load_or_i
47
49
  @click.option(
48
50
  "--cpu",
49
51
  type=str,
50
- help=f"For AWS only, CPUs to allocate to your node" f"Defaults to 2 (run `numerai list-constants` to see options).",
52
+ help=f"For AWS only, CPUs to allocate to your node"
53
+ f"Defaults to 2 (run `numerai list-constants` to see options).",
51
54
  )
52
55
  @click.option(
53
56
  "--memory",
@@ -92,9 +95,26 @@ from numerai.cli.util.keys import get_provider_keys, get_numerai_keys, load_or_i
92
95
  help="Forces your webhook to register with Numerai. "
93
96
  "Use in conjunction with options that prevent webhook auto-registering.",
94
97
  )
98
+ @click.option(
99
+ "--volume",
100
+ "-v",
101
+ type=int,
102
+ help="Specify additional block storage in GB. Currently only supported in AWS.",
103
+ )
95
104
  @click.pass_context
96
105
  def config(
97
- ctx, verbose, provider, size, cpu, memory, path, example, cron, timeout_minutes, register_webhook
106
+ ctx,
107
+ verbose,
108
+ provider,
109
+ size,
110
+ cpu,
111
+ memory,
112
+ path,
113
+ example,
114
+ cron,
115
+ timeout_minutes,
116
+ register_webhook,
117
+ volume,
98
118
  ):
99
119
  """
100
120
  Uses Terraform to create a full Numerai Compute cluster in your desired provider.
@@ -133,7 +153,11 @@ def config(
133
153
  affected_providers = set(filter(None, affected_providers))
134
154
 
135
155
  nodes_config[node].update(
136
- {key: default for key, default in DEFAULT_SETTINGS.items() if key not in nodes_config[node]}
156
+ {
157
+ key: default
158
+ for key, default in DEFAULT_SETTINGS.items()
159
+ if key not in nodes_config[node]
160
+ }
137
161
  )
138
162
  # update node as needed
139
163
  node_conf = nodes_config[node]
@@ -147,21 +171,29 @@ def config(
147
171
  provider = node_conf["provider"]
148
172
 
149
173
  if timeout_minutes and provider == PROVIDER_AZURE:
150
- click.secho("Timeout settings are unavailable for Azure and this input will be ignored.", fg="yellow")
174
+ click.secho(
175
+ "Timeout settings are unavailable for Azure and this input will be ignored.",
176
+ fg="yellow",
177
+ )
151
178
  elif timeout_minutes:
152
179
  node_conf["timeout_minutes"] = timeout_minutes
153
180
 
154
181
  if provider == PROVIDER_GCP and size is not None and "mem-" in size:
155
182
  click.secho(
156
- "Invalid size: mem sizes are invalid for GCP due to sizing constraints with Google Cloud Run.", fg="red"
183
+ "Invalid size: mem sizes are invalid for GCP due to sizing constraints with Google Cloud Run.",
184
+ fg="red",
157
185
  )
158
186
  click.secho(
159
- "Visit https://cloud.google.com/run/docs/configuring/services/memory-limits to learn more.", fg="red"
187
+ "Visit https://cloud.google.com/run/docs/configuring/services/memory-limits to learn more.",
188
+ fg="red",
160
189
  )
161
190
  exit(1)
162
191
 
163
192
  if size and (cpu or memory):
164
- click.secho("Cannot provide size and CPU or Memory. Either use size or provide CPU and Memory.", fg="red")
193
+ click.secho(
194
+ "Cannot provide size and CPU or Memory. Either use size or provide CPU and Memory.",
195
+ fg="red",
196
+ )
165
197
  exit(1)
166
198
  if (cpu or memory) and node_conf["provider"] != PROVIDER_AWS:
167
199
  click.secho(
@@ -169,7 +201,9 @@ def config(
169
201
  fg="red",
170
202
  )
171
203
  exit(1)
172
- elif (cpu or memory) and (not (cpu or node_conf["cpu"]) or not (memory or node_conf["memory"])):
204
+ elif (cpu or memory) and (
205
+ not (cpu or node_conf["cpu"]) or not (memory or node_conf["memory"])
206
+ ):
173
207
  click.secho(
174
208
  "One of CPU and Memory is missing either from your options or from your node configuration."
175
209
  "Provide both CPU and Memory to configure node size, or use size."
@@ -218,7 +252,9 @@ def config(
218
252
 
219
253
  # Azure only: Need to create a master Azure Container Registry and push a dummy placeholder image, before deploying the rest of the resources
220
254
  if provider == "azure":
221
- provider_registry_conf = create_azure_registry(provider, provider_keys, verbose=verbose)
255
+ provider_registry_conf = create_azure_registry(
256
+ provider, provider_keys, verbose=verbose
257
+ )
222
258
  node_conf.update(provider_registry_conf)
223
259
  node_conf["docker_repo"] = f'{node_conf["acr_login_server"]}/{node}'
224
260
  docker.login(node_conf, verbose)
@@ -234,12 +270,12 @@ def config(
234
270
  provider_registry_conf = create_gcp_registry(provider, verbose=verbose)
235
271
  node_conf.update(provider_registry_conf)
236
272
  registry_parts = node_conf["registry_id"].split("/")
237
- node_conf[
238
- "artifact_registry_login_url"
239
- ] = f"https://{registry_parts[3]}-docker.pkg.dev/"
240
- node_conf[
241
- "docker_repo"
242
- ] = f"{registry_parts[3]}-docker.pkg.dev/{registry_parts[1]}/numerai-container-registry/{node}:latest"
273
+ node_conf["artifact_registry_login_url"] = (
274
+ f"https://{registry_parts[3]}-docker.pkg.dev/"
275
+ )
276
+ node_conf["docker_repo"] = (
277
+ f"{registry_parts[3]}-docker.pkg.dev/{registry_parts[1]}/numerai-container-registry/{node}:latest"
278
+ )
243
279
  docker.login(node_conf, verbose)
244
280
  try:
245
281
  docker.manifest_inspect(node_conf["docker_repo"], verbose)
@@ -248,6 +284,9 @@ def config(
248
284
  docker.tag("hello-world:linux", node_conf["docker_repo"], verbose)
249
285
  docker.push(node_conf["docker_repo"], verbose)
250
286
  nodes_config[node] = node_conf
287
+ elif provider == "aws":
288
+ if volume is not None:
289
+ node_conf["volume"] = volume
251
290
 
252
291
  store_config(NODES_PATH, nodes_config)
253
292
 
@@ -8,7 +8,27 @@ from numerai.cli.util.files import load_or_init_nodes, store_config, copy_file
8
8
  from numerai.cli.util.keys import get_provider_keys, get_numerai_keys
9
9
 
10
10
 
11
- def destroy_node(node, verbose, preserve_node_config=False):
11
+ @click.command()
12
+ @click.option("--preserve-node-config", "-p", is_flag=True)
13
+ @click.option("--verbose", "-v", is_flag=True)
14
+ @click.pass_context
15
+ def destroy(ctx, preserve_node_config, verbose):
16
+ """
17
+ Uses Terraform to destroy a Numerai Compute cluster.
18
+ This will delete everything, including:
19
+ - lambda url
20
+ - docker container and associated task
21
+ - all logs
22
+ This command is idempotent and safe to run multiple times.
23
+ """
24
+
25
+ ctx.ensure_object(dict)
26
+ model = ctx.obj["model"]
27
+ node = model["name"]
28
+ if not os.path.exists(CONFIG_PATH):
29
+ click.secho(".numerai directory not setup, run `numerai setup`...", fg="red")
30
+ return
31
+
12
32
  try:
13
33
  nodes_config = load_or_init_nodes()
14
34
  node_config = nodes_config[node]
@@ -21,6 +41,10 @@ def destroy_node(node, verbose, preserve_node_config=False):
21
41
  fg="red",
22
42
  )
23
43
  return
44
+
45
+ if not preserve_node_config:
46
+ click.secho("backing up nodes.json...")
47
+ copy_file(NODES_PATH, f"{NODES_PATH}.backup", force=True, verbose=True)
24
48
 
25
49
  try:
26
50
  click.secho(
@@ -30,7 +54,6 @@ def destroy_node(node, verbose, preserve_node_config=False):
30
54
  )
31
55
  del nodes_config[node]
32
56
  store_config(NODES_PATH, nodes_config)
33
- copy_file(NODES_PATH, f"{CONFIG_PATH}/{provider}/", force=True, verbose=True)
34
57
 
35
58
  click.secho("deleting cloud resources for node...")
36
59
  terraform(
@@ -60,27 +83,3 @@ def destroy_node(node, verbose, preserve_node_config=False):
60
83
  click.secho("re-adding node config to nodes.json...", fg="green")
61
84
  nodes_config[node] = node_config
62
85
  store_config(NODES_PATH, nodes_config)
63
-
64
-
65
- @click.command()
66
- @click.option("--preserve-node-config", "-p", is_flag=True)
67
- @click.option("--verbose", "-v", is_flag=True)
68
- @click.pass_context
69
- def destroy(ctx, preserve_node_config, verbose):
70
- """
71
- Uses Terraform to destroy a Numerai Compute cluster.
72
- This will delete everything, including:
73
- - lambda url
74
- - docker container and associated task
75
- - all logs
76
- This command is idempotent and safe to run multiple times.
77
- """
78
-
79
- ctx.ensure_object(dict)
80
- model = ctx.obj["model"]
81
- node = model["name"]
82
- if not os.path.exists(CONFIG_PATH):
83
- click.secho(".numerai directory not setup, run `numerai setup`...", fg="red")
84
- return
85
-
86
- destroy_node(node, verbose, preserve_node_config)
@@ -61,7 +61,7 @@ def test(ctx, local, command, verbose):
61
61
  ctx.ensure_object(dict)
62
62
  model = ctx.obj["model"]
63
63
  node = model["name"]
64
- is_signals = model["is_signals"]
64
+ tournament = model["tournament"]
65
65
  node_config = load_or_init_nodes(node)
66
66
  provider = node_config["provider"]
67
67
 
@@ -143,7 +143,6 @@ def test(ctx, local, command, verbose):
143
143
  variables={"modelId": node_config["model_id"]},
144
144
  authorization=True,
145
145
  )
146
- tournament = TOURNAMENT_SIGNALS if is_signals else TOURNAMENT_NUMERAI
147
146
  curr_round = api.get_current_round(tournament)
148
147
  latest_subs = sorted(
149
148
  filter(
@@ -338,7 +337,7 @@ def get_recent_task_status_aws(cluster_arn, ecs_client, node, trigger_id):
338
337
  "red",
339
338
  )
340
339
  elif matched_task["lastStatus"] in stopped_codes:
341
- return matched_task, True, "Job execution succeeded!\r", "green"
340
+ return matched_task, True, "Job execution finished!\r", "green"
342
341
  elif matched_task["lastStatus"] in pending_codes:
343
342
  return matched_task, False, "Waiting for job to start...", "yellow"
344
343
  elif matched_task["lastStatus"] in running_codes:
@@ -159,11 +159,12 @@ def root_cause(std_out, err_msg):
159
159
 
160
160
  if b"Cycle" in std_out:
161
161
  raise exception_with_msg(
162
- "You upgraded to 1.0+ and need to replace your AWS nodes."
163
- "\nRun the following commands:"
164
- "\n numerai node -m <model_name> destroy --preserve-node-config"
165
- "\n numerai node -m <model_name> config"
166
- "\n numerai node -m <model_name> deploy"
162
+ "You upgraded to 1.0+ and need to replace your AWS nodes before continuing!"
163
+ "\nTo do this now follow these instructions:"
164
+ '\n run "numerai destroy-all --preserve-node-config"'
165
+ '\n run "numerai node -m <model_name> config" for each node'
166
+ '\n run "numerai node -m <model_name> deploy" for each node'
167
+ "\nIf you do not want to do this, downgrade to 0.4.1 to continue."
167
168
  )
168
169
 
169
170
  raise exception_with_msg(
@@ -166,9 +166,16 @@ def build(node_config, node, verbose):
166
166
  build_arg_str += f" --build-arg NODE={node}"
167
167
 
168
168
  cmd = (
169
- f'docker build --platform=linux/amd64 -t {node_config["docker_repo"]}' f"{build_arg_str} -f {path}/Dockerfile ."
169
+ f'docker build --platform=linux/amd64 --load -t {node_config["docker_repo"]}'
170
+ f"{build_arg_str} -f {path}/Dockerfile ."
170
171
  )
171
- execute(cmd, verbose)
172
+ stdout, stderr = execute(cmd, verbose)
173
+ if "unknown flag: --load" in (stdout.decode() + stderr.decode()):
174
+ click.secho(
175
+ "Docker version too old, please upgrade to at least 18.09",
176
+ fg="red",
177
+ )
178
+ exit(1)
172
179
 
173
180
 
174
181
  def run(node_config, verbose, command=""):
@@ -181,7 +188,9 @@ def login(node_config, verbose):
181
188
  username, password = login_aws()
182
189
  login_url = node_config['docker_repo']
183
190
  elif node_config["provider"] == PROVIDER_AZURE:
184
- username, password = login_azure(node_config["registry_rg_name"], node_config["registry_name"])
191
+ username, password = login_azure(
192
+ node_config["registry_rg_name"], node_config["registry_name"]
193
+ )
185
194
  login_url = node_config['docker_repo']
186
195
  elif node_config["provider"] == PROVIDER_GCP:
187
196
  username, password = login_gcp()
@@ -214,17 +223,23 @@ def login_aws():
214
223
  )
215
224
 
216
225
  token = ecr_client.get_authorization_token() # TODO: use registryIds
217
- username, password = base64.b64decode(token["authorizationData"][0]["authorizationToken"]).decode().split(":")
226
+ username, password = (
227
+ base64.b64decode(token["authorizationData"][0]["authorizationToken"])
228
+ .decode()
229
+ .split(":")
230
+ )
218
231
 
219
232
  return username, password
220
233
 
221
234
 
222
235
  def login_azure(resource_group_name, registry_name):
223
236
  azure_subs_id, azure_client, azure_tenant, azure_secret = get_azure_keys()
224
- credentials = ClientSecretCredential(client_id=azure_client, tenant_id=azure_tenant, client_secret=azure_secret)
225
- username_password = ContainerRegistryManagementClient(credentials, azure_subs_id).registries.list_credentials(
226
- resource_group_name, registry_name
237
+ credentials = ClientSecretCredential(
238
+ client_id=azure_client, tenant_id=azure_tenant, client_secret=azure_secret
227
239
  )
240
+ username_password = ContainerRegistryManagementClient(
241
+ credentials, azure_subs_id
242
+ ).registries.list_credentials(resource_group_name, registry_name)
228
243
  username = username_password.username
229
244
  password = username_password.passwords[0].value
230
245
  return username, password
@@ -288,24 +303,32 @@ def cleanup_aws(docker_repo):
288
303
 
289
304
  docker_repo_name = docker_repo.split("/")[-1]
290
305
 
291
- resp = ecr_client.list_images(repositoryName=docker_repo_name, filter={"tagStatus": "UNTAGGED"})
306
+ resp = ecr_client.list_images(
307
+ repositoryName=docker_repo_name, filter={"tagStatus": "UNTAGGED"}
308
+ )
292
309
 
293
310
  imageIds = resp["imageIds"]
294
311
  if len(imageIds) == 0:
295
312
  return []
296
313
 
297
- resp = ecr_client.batch_delete_image(repositoryName=docker_repo_name, imageIds=imageIds)
314
+ resp = ecr_client.batch_delete_image(
315
+ repositoryName=docker_repo_name, imageIds=imageIds
316
+ )
298
317
 
299
318
  return resp["imageIds"]
300
319
 
301
320
 
302
321
  def cleanup_azure(node_config):
303
322
  _, azure_client, azure_tenant, azure_secret = get_azure_keys()
304
- credentials = ClientSecretCredential(client_id=azure_client, tenant_id=azure_tenant, client_secret=azure_secret)
323
+ credentials = ClientSecretCredential(
324
+ client_id=azure_client, tenant_id=azure_tenant, client_secret=azure_secret
325
+ )
305
326
  acr_client = ContainerRegistryClient(node_config["acr_login_server"], credentials)
306
327
  docker_repo = node_config["docker_repo"]
307
328
  node_repo_name = [
308
- repo_name for repo_name in acr_client.list_repository_names() if repo_name == docker_repo.split("/")[-1]
329
+ repo_name
330
+ for repo_name in acr_client.list_repository_names()
331
+ if repo_name == docker_repo.split("/")[-1]
309
332
  ][0]
310
333
 
311
334
  # get all manifests, ordered by last update time
@@ -318,7 +341,9 @@ def cleanup_azure(node_config):
318
341
  # Remove all but the latest manifest
319
342
  removed_manifests = []
320
343
  for manifest in manifest_list[1:]:
321
- acr_client.update_manifest_properties(node_repo_name, manifest.digest, can_write=True, can_delete=True)
344
+ acr_client.update_manifest_properties(
345
+ node_repo_name, manifest.digest, can_write=True, can_delete=True
346
+ )
322
347
  removed_manifests.append(manifest.digest)
323
348
  acr_client.delete_manifest(node_repo_name, manifest.digest)
324
349
  return removed_manifests
@@ -4,9 +4,9 @@ FROM python:3.9
4
4
  # These are docker arguments that `numerai node deploy/test` will always pass into docker.
5
5
  # They are then set in your environment so that numerapi can access them when uploading submissions.
6
6
  # You can also access them from your script like so:
7
- # import os
8
- # public_id = os.environ["NUMERAI_PUBLIC_ID"]
9
- # secret_key = os.environ["NUMERAI_SECRET_KEY"]
7
+ # import os
8
+ # public_id = os.environ["NUMERAI_PUBLIC_ID"]
9
+ # secret_key = os.environ["NUMERAI_SECRET_KEY"]
10
10
  ARG NUMERAI_PUBLIC_ID
11
11
  ENV NUMERAI_PUBLIC_ID=$NUMERAI_PUBLIC_ID
12
12