numerai-cli 1.0.3__tar.gz → 1.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/PKG-INFO +2 -1
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/README.md +1 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/__init__.py +1 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/constants.py +1 -7
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/destroy_all.py +1 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/misc.py +47 -10
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/node/__init__.py +19 -16
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/node/config.py +18 -77
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/node/destroy.py +2 -1
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/node/test.py +2 -3
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/setup.py +17 -13
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/util/docker.py +37 -12
- numerai-cli-1.1.1/numerai/cli/util/terraform.py +78 -0
- {numerai-cli-1.0.3/numerai/examples/tournament-python3 → numerai-cli-1.1.1/numerai/examples/crypto-python3}/Dockerfile +3 -3
- numerai-cli-1.1.1/numerai/examples/crypto-python3/predict.py +105 -0
- numerai-cli-1.1.1/numerai/examples/crypto-python3/requirements.txt +5 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/examples/signals-python3/Dockerfile +4 -4
- numerai-cli-1.1.1/numerai/examples/signals-python3/predict.py +104 -0
- numerai-cli-1.1.1/numerai/examples/signals-python3/requirements.txt +5 -0
- numerai-cli-1.1.1/numerai/examples/tournament-python3/Dockerfile +35 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/examples/tournament-python3/predict.py +17 -7
- numerai-cli-1.1.1/numerai/examples/tournament-python3/requirements.txt +5 -0
- numerai-cli-1.1.1/numerai/examples/tournament-python3/train.py +10 -0
- numerai-cli-1.1.1/numerai/terraform/aws/aws/-locals.tf +4 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/aws/aws/cluster.tf +60 -16
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/aws/aws/webhook.tf +1 -1
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai_cli.egg-info/PKG-INFO +2 -1
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai_cli.egg-info/SOURCES.txt +4 -1
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/setup.py +1 -1
- numerai-cli-1.0.3/numerai/examples/signals-python3/data.py +0 -164
- numerai-cli-1.0.3/numerai/examples/signals-python3/predict.py +0 -160
- numerai-cli-1.0.3/numerai/examples/signals-python3/requirements.txt +0 -8
- numerai-cli-1.0.3/numerai/examples/signals-python3/train.py +0 -6
- numerai-cli-1.0.3/numerai/examples/tournament-python3/requirements.txt +0 -5
- numerai-cli-1.0.3/numerai/terraform/gcp/gcp/-locals.tf +0 -3
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/MANIFEST.in +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/__init__.py +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/doctor.py +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/node/deploy.py +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/uninstall.py +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/upgrade.py +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/util/__init__.py +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/util/debug.py +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/util/files.py +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/cli/util/keys.py +0 -0
- {numerai-cli-1.0.3/numerai/examples/tournament-python3 → numerai-cli-1.1.1/numerai/examples/signals-python3}/train.py +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/examples/tournament-rlang/Dockerfile +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/examples/tournament-rlang/install_packages.R +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/examples/tournament-rlang/main.R +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/aws/-inputs.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/aws/-main.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/aws/-outputs.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/aws/aws/-inputs.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/aws/aws/-outputs.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/aws/aws/main.zip +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/aws/aws/network.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/azure/cluster.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/azure/input.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/azure/locals.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/azure/outputs.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/azure/webhook.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/azure_cron_trigger.zip +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/azure_trigger.zip +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/input.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/main.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/outputs.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/azure/registry.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/-inputs.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/-main.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/-outputs.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/cloud-function.zip +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/gcp/-inputs.tf +0 -0
- {numerai-cli-1.0.3/numerai/terraform/aws/aws → numerai-cli-1.1.1/numerai/terraform/gcp/gcp}/-locals.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/gcp/-outputs.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/gcp/cluster.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/gcp/webhook.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/gcp/workflow-source.yaml +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/registry.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai/terraform/gcp/services.tf +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai_cli.egg-info/dependency_links.txt +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai_cli.egg-info/entry_points.txt +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai_cli.egg-info/requires.txt +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/numerai_cli.egg-info/top_level.txt +0 -0
- {numerai-cli-1.0.3 → numerai-cli-1.1.1}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: numerai-cli
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.1.1
|
|
4
4
|
Summary: A library for deploying Numer.ai Prediction Nodes.
|
|
5
5
|
Home-page: https://github.com/numerai/numerai-cli
|
|
6
6
|
Author: Numer.ai
|
|
@@ -11,6 +11,7 @@ Description: # numerai-cli
|
|
|
11
11
|
[](https://pypi.org/project/numerai-cli/)
|
|
12
12
|
|
|
13
13
|
Welcome to the Numerai CLI for the [Numerai Tournament](https://docs.numer.ai/tournament/learn).
|
|
14
|
+
Numerai CLI (aka Compute Heavy) is a tool to help you set up cloud automation for your Numerai predictions.
|
|
14
15
|
This README is designed to have EVERYTHING you need to setup and maintain a Numerai Compute Heavy node.
|
|
15
16
|
|
|
16
17
|
This CLI runs on your local computer to configure a Numerai Prediction Node in the cloud.
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
[](https://pypi.org/project/numerai-cli/)
|
|
4
4
|
|
|
5
5
|
Welcome to the Numerai CLI for the [Numerai Tournament](https://docs.numer.ai/tournament/learn).
|
|
6
|
+
Numerai CLI (aka Compute Heavy) is a tool to help you set up cloud automation for your Numerai predictions.
|
|
6
7
|
This README is designed to have EVERYTHING you need to setup and maintain a Numerai Compute Heavy node.
|
|
7
8
|
|
|
8
9
|
This CLI runs on your local computer to configure a Numerai Prediction Node in the cloud.
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
"""Constants for Numerai CLI"""
|
|
2
|
+
|
|
2
3
|
import os
|
|
3
4
|
import json
|
|
4
5
|
from pathlib import Path
|
|
5
6
|
|
|
6
|
-
TOURNAMENT_NUMERAI = 8
|
|
7
|
-
TOURNAMENT_SIGNALS = 11
|
|
8
7
|
PACKAGE_PATH = os.path.dirname(__file__)
|
|
9
8
|
CONFIG_PATH = os.path.join(str(Path.home()), ".numerai")
|
|
10
9
|
KEYS_PATH = os.path.join(CONFIG_PATH, ".keys")
|
|
@@ -65,11 +64,6 @@ DEFAULT_SETTINGS = {
|
|
|
65
64
|
}
|
|
66
65
|
|
|
67
66
|
CONSTANTS_STR = f"""Default values (not your configured node values):
|
|
68
|
-
|
|
69
|
-
---Tournament Numbers---
|
|
70
|
-
TOURNAMENT_NUMERAI: {TOURNAMENT_NUMERAI}
|
|
71
|
-
TOURNAMENT_SIGNALS: {TOURNAMENT_SIGNALS}
|
|
72
|
-
|
|
73
67
|
---Paths---
|
|
74
68
|
PACKAGE_PATH: {PACKAGE_PATH}
|
|
75
69
|
CONFIG_PATH: {CONFIG_PATH}
|
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
import json
|
|
2
|
-
|
|
3
|
-
import click
|
|
4
|
-
|
|
5
1
|
from numerai.cli.constants import *
|
|
6
2
|
from numerai.cli.util import files
|
|
3
|
+
from numerai.cli.util.terraform import apply_terraform
|
|
4
|
+
|
|
5
|
+
import click
|
|
7
6
|
|
|
8
7
|
|
|
9
8
|
@click.command()
|
|
@@ -12,13 +11,15 @@ from numerai.cli.util import files
|
|
|
12
11
|
"-e",
|
|
13
12
|
type=click.Choice(EXAMPLES),
|
|
14
13
|
default=DEFAULT_EXAMPLE,
|
|
15
|
-
help=f"Specify the example to copy, defaults to {DEFAULT_EXAMPLE}. "
|
|
14
|
+
help=f"Specify the example to copy, defaults to {DEFAULT_EXAMPLE}. "
|
|
15
|
+
f"Options are {EXAMPLES}.",
|
|
16
16
|
)
|
|
17
17
|
@click.option(
|
|
18
18
|
"--dest",
|
|
19
19
|
"-d",
|
|
20
20
|
type=str,
|
|
21
|
-
help=f"Destination folder to which example code is written. "
|
|
21
|
+
help=f"Destination folder to which example code is written. "
|
|
22
|
+
f"Defaults to the name of the example.",
|
|
22
23
|
)
|
|
23
24
|
@click.option("--verbose", "-v", is_flag=True)
|
|
24
25
|
def copy_example(example, dest, verbose):
|
|
@@ -49,14 +50,50 @@ def list_constants():
|
|
|
49
50
|
click.secho(
|
|
50
51
|
f" {size} -> cpus: {preset[0] / 1024}, "
|
|
51
52
|
f"mem: {preset[1] / 1024} GB {suffix}",
|
|
52
|
-
fg=
|
|
53
|
-
|
|
54
|
-
|
|
53
|
+
fg=(
|
|
54
|
+
"green"
|
|
55
|
+
if size == DEFAULT_SIZE or size == DEFAULT_SIZE_GCP
|
|
56
|
+
else "yellow"
|
|
57
|
+
),
|
|
55
58
|
)
|
|
56
|
-
click.secho(
|
|
59
|
+
click.secho(
|
|
60
|
+
"Due to GCP Cloud Run size constraints, 'mem' sizes are not allowed when using GCP."
|
|
61
|
+
)
|
|
57
62
|
click.secho(
|
|
58
63
|
"For AWS, use one of these sizes, or specify your own CPU and Memory in cores and GB using --cpu and --memory options.\n"
|
|
59
64
|
"See https://learn.microsoft.com/en-us/azure/container-apps/containers#configuration for Azure,\n"
|
|
60
65
|
"or https://cloud.google.com/run/docs/configuring/services/memory-limits for GCP \n"
|
|
61
66
|
"to learn more info about allowed size presets for those providers."
|
|
62
67
|
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@click.command()
|
|
71
|
+
@click.option(
|
|
72
|
+
"--size",
|
|
73
|
+
"-s",
|
|
74
|
+
type=int,
|
|
75
|
+
required=True,
|
|
76
|
+
help="Specify the volume size in GB you'd like your AWS nodes to share.",
|
|
77
|
+
)
|
|
78
|
+
@click.option("--verbose", "-v", is_flag=True)
|
|
79
|
+
def add_volume_aws(size, verbose):
|
|
80
|
+
"""
|
|
81
|
+
Set the volume size for AWS nodes. This volume is shared by all nodes.
|
|
82
|
+
"""
|
|
83
|
+
click.secho("Setting volume size for AWS nodes...", fg="yellow")
|
|
84
|
+
# get nodes config object
|
|
85
|
+
nodes_config = files.load_or_init_nodes()
|
|
86
|
+
print(nodes_config)
|
|
87
|
+
# set volume size for all nodes to same size
|
|
88
|
+
for node in nodes_config:
|
|
89
|
+
nodes_config[node]["volume"] = size
|
|
90
|
+
files.store_config(NODES_PATH, nodes_config)
|
|
91
|
+
files.copy_file(
|
|
92
|
+
NODES_PATH,
|
|
93
|
+
f"{CONFIG_PATH}/{PROVIDER_AWS}/",
|
|
94
|
+
force=True,
|
|
95
|
+
verbose=verbose,
|
|
96
|
+
)
|
|
97
|
+
click.secho(f"Applying terraform to add {size} GB volume...", fg="yellow")
|
|
98
|
+
apply_terraform(nodes_config, [PROVIDER_AWS], PROVIDER_AWS, verbose=verbose)
|
|
99
|
+
click.secho("Volume size updated successfully!", fg="green")
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""Init for node"""
|
|
2
|
+
|
|
2
3
|
import json
|
|
3
4
|
import logging
|
|
4
5
|
import click
|
|
@@ -15,22 +16,22 @@ from numerai.cli.util.keys import get_numerai_keys
|
|
|
15
16
|
# Setting azure's logging level "ERROR" to avoid spamming the terminal
|
|
16
17
|
|
|
17
18
|
|
|
18
|
-
def
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
19
|
+
def tournaments_dict():
|
|
20
|
+
napi = base_api.Api()
|
|
21
|
+
tournaments = napi.raw_query('query { tournaments { name tournament } }')
|
|
22
|
+
return {t["tournament"]: t["name"] for t in tournaments["data"]["tournaments"]}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def get_models(tournament):
|
|
25
26
|
napi = base_api.Api(*get_numerai_keys())
|
|
26
27
|
models = napi.get_models(tournament)
|
|
27
|
-
|
|
28
|
+
name_prefix = tournaments_dict()[tournament]
|
|
28
29
|
model_dict = {}
|
|
29
30
|
for model_name, model_id in models.items():
|
|
30
31
|
model_dict[model_name] = {
|
|
31
32
|
"id": model_id,
|
|
32
33
|
"name": f"{name_prefix}-{model_name}",
|
|
33
|
-
"
|
|
34
|
+
"tournament": tournament,
|
|
34
35
|
}
|
|
35
36
|
return model_dict
|
|
36
37
|
|
|
@@ -43,16 +44,18 @@ def get_models(signals):
|
|
|
43
44
|
type=str,
|
|
44
45
|
prompt=True,
|
|
45
46
|
help="The name of one of your models to configure the Prediction Node for."
|
|
46
|
-
"It defaults to the first model returned from your account.",
|
|
47
|
+
" It defaults to the first model returned from your account.",
|
|
47
48
|
)
|
|
48
49
|
@click.option(
|
|
49
|
-
"--
|
|
50
|
-
"-
|
|
51
|
-
|
|
52
|
-
help="Target a
|
|
50
|
+
"--tournament",
|
|
51
|
+
"-t",
|
|
52
|
+
default=8,
|
|
53
|
+
help="Target a specific tournament number."
|
|
54
|
+
" Defaults to Numerai Tournament/Classic."
|
|
55
|
+
f" Available tournaments: {json.dumps(tournaments_dict(), indent=2)}",
|
|
53
56
|
)
|
|
54
57
|
@click.pass_context
|
|
55
|
-
def node(ctx, verbose, model_name,
|
|
58
|
+
def node(ctx, verbose, model_name, tournament):
|
|
56
59
|
"""
|
|
57
60
|
Commands to manage and test Prediction Nodes.
|
|
58
61
|
"""
|
|
@@ -68,7 +71,7 @@ def node(ctx, verbose, model_name, signals):
|
|
|
68
71
|
else:
|
|
69
72
|
logger.setLevel(logging.ERROR)
|
|
70
73
|
|
|
71
|
-
models = get_models(
|
|
74
|
+
models = get_models(tournament)
|
|
72
75
|
|
|
73
76
|
try:
|
|
74
77
|
ctx.ensure_object(dict)
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""Config command for Numerai CLI"""
|
|
2
|
-
|
|
2
|
+
|
|
3
3
|
import os
|
|
4
|
-
|
|
4
|
+
|
|
5
5
|
from numerapi import base_api
|
|
6
6
|
from numerai.cli.constants import (
|
|
7
7
|
DEFAULT_PROVIDER,
|
|
@@ -18,7 +18,6 @@ from numerai.cli.constants import (
|
|
|
18
18
|
CONFIG_PATH,
|
|
19
19
|
PROVIDER_GCP,
|
|
20
20
|
)
|
|
21
|
-
from numerai.cli.util.docker import terraform, check_for_dockerfile
|
|
22
21
|
from numerai.cli.util import docker
|
|
23
22
|
from numerai.cli.util.files import (
|
|
24
23
|
load_or_init_nodes,
|
|
@@ -26,7 +25,14 @@ from numerai.cli.util.files import (
|
|
|
26
25
|
copy_example,
|
|
27
26
|
copy_file,
|
|
28
27
|
)
|
|
29
|
-
from numerai.cli.util.keys import get_provider_keys, get_numerai_keys
|
|
28
|
+
from numerai.cli.util.keys import get_provider_keys, get_numerai_keys
|
|
29
|
+
from numerai.cli.util.terraform import (
|
|
30
|
+
apply_terraform,
|
|
31
|
+
create_azure_registry,
|
|
32
|
+
create_gcp_registry,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
import click
|
|
30
36
|
|
|
31
37
|
|
|
32
38
|
@click.command()
|
|
@@ -226,7 +232,7 @@ def config(
|
|
|
226
232
|
click.secho(f'Current node config: "{node_conf}"...')
|
|
227
233
|
|
|
228
234
|
# double check there is a dockerfile in the path we are about to configure
|
|
229
|
-
check_for_dockerfile(nodes_config[node]["path"])
|
|
235
|
+
docker.check_for_dockerfile(nodes_config[node]["path"])
|
|
230
236
|
store_config(NODES_PATH, nodes_config)
|
|
231
237
|
|
|
232
238
|
# Added after tf directory restructure: copy nodes.json to providers' tf directory
|
|
@@ -262,12 +268,12 @@ def config(
|
|
|
262
268
|
provider_registry_conf = create_gcp_registry(provider, verbose=verbose)
|
|
263
269
|
node_conf.update(provider_registry_conf)
|
|
264
270
|
registry_parts = node_conf["registry_id"].split("/")
|
|
265
|
-
node_conf[
|
|
266
|
-
"
|
|
267
|
-
|
|
268
|
-
node_conf[
|
|
269
|
-
"
|
|
270
|
-
|
|
271
|
+
node_conf["artifact_registry_login_url"] = (
|
|
272
|
+
f"https://{registry_parts[3]}-docker.pkg.dev/"
|
|
273
|
+
)
|
|
274
|
+
node_conf["docker_repo"] = (
|
|
275
|
+
f"{registry_parts[3]}-docker.pkg.dev/{registry_parts[1]}/numerai-container-registry/{node}:latest"
|
|
276
|
+
)
|
|
271
277
|
docker.login(node_conf, verbose)
|
|
272
278
|
try:
|
|
273
279
|
docker.manifest_inspect(node_conf["docker_repo"], verbose)
|
|
@@ -279,37 +285,7 @@ def config(
|
|
|
279
285
|
|
|
280
286
|
store_config(NODES_PATH, nodes_config)
|
|
281
287
|
|
|
282
|
-
|
|
283
|
-
for affected_provider in affected_providers:
|
|
284
|
-
if affected_provider in PROVIDERS:
|
|
285
|
-
click.secho(f"Updating resources in {affected_provider}")
|
|
286
|
-
terraform(
|
|
287
|
-
"apply -auto-approve",
|
|
288
|
-
verbose,
|
|
289
|
-
affected_provider,
|
|
290
|
-
env_vars=load_or_init_keys(affected_provider),
|
|
291
|
-
inputs={"node_config_file": "nodes.json"},
|
|
292
|
-
)
|
|
293
|
-
else:
|
|
294
|
-
click.secho(f"provider {affected_provider} not supported", fg="red")
|
|
295
|
-
exit(1)
|
|
296
|
-
click.secho("cloud resources created successfully", fg="green")
|
|
297
|
-
|
|
298
|
-
# terraform output for node config, same for aws and azure
|
|
299
|
-
click.echo(f"saving node configuration to {NODES_PATH}...")
|
|
300
|
-
|
|
301
|
-
res = terraform(f"output -json {provider}_nodes", verbose, provider).decode("utf-8")
|
|
302
|
-
try:
|
|
303
|
-
nodes = json.loads(res)
|
|
304
|
-
except json.JSONDecodeError:
|
|
305
|
-
click.secho("failed to save node configuration, please retry.", fg="red")
|
|
306
|
-
return
|
|
307
|
-
for node_name, data in nodes.items():
|
|
308
|
-
nodes_config[node_name].update(data)
|
|
309
|
-
|
|
310
|
-
store_config(NODES_PATH, nodes_config)
|
|
311
|
-
if verbose:
|
|
312
|
-
click.secho(f"new config:\n{json.dumps(load_or_init_nodes(), indent=2)}")
|
|
288
|
+
apply_terraform(nodes_config, affected_providers, provider, verbose)
|
|
313
289
|
|
|
314
290
|
webhook_url = nodes_config[node]["webhook_url"]
|
|
315
291
|
napi = base_api.Api(*get_numerai_keys())
|
|
@@ -325,38 +301,3 @@ def config(
|
|
|
325
301
|
"Prediction Node configured successfully. " "Next: deploy and test your node",
|
|
326
302
|
fg="green",
|
|
327
303
|
)
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
def create_azure_registry(provider, provider_keys, verbose):
|
|
331
|
-
"""Creates a registry for azure"""
|
|
332
|
-
terraform("init -upgrade", verbose, provider)
|
|
333
|
-
terraform(
|
|
334
|
-
'apply -target="azurerm_container_registry.registry[0]" -target="azurerm_resource_group.acr_rg[0]" -auto-approve ',
|
|
335
|
-
verbose,
|
|
336
|
-
"azure",
|
|
337
|
-
env_vars=provider_keys,
|
|
338
|
-
inputs={"node_config_file": "nodes.json"},
|
|
339
|
-
)
|
|
340
|
-
res = terraform("output -json acr_repo_details", True, provider).decode("utf-8")
|
|
341
|
-
return json.loads(res)
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
def create_gcp_registry(provider, verbose):
|
|
345
|
-
"""Creates a registry for GCP"""
|
|
346
|
-
terraform("init -upgrade", verbose, provider)
|
|
347
|
-
terraform(
|
|
348
|
-
'apply -target="google_project_service.cloud_resource_manager" -auto-approve ',
|
|
349
|
-
verbose,
|
|
350
|
-
"gcp",
|
|
351
|
-
inputs={"node_config_file": "nodes.json"},
|
|
352
|
-
)
|
|
353
|
-
terraform(
|
|
354
|
-
'apply -target="google_artifact_registry_repository.registry[0]" -auto-approve ',
|
|
355
|
-
verbose,
|
|
356
|
-
"gcp",
|
|
357
|
-
inputs={"node_config_file": "nodes.json"},
|
|
358
|
-
)
|
|
359
|
-
res = terraform("output -json artifact_registry_details", True, provider).decode(
|
|
360
|
-
"utf-8"
|
|
361
|
-
)
|
|
362
|
-
return json.loads(res)
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""Destroy command for Numerai CLI"""
|
|
2
|
+
|
|
2
3
|
import click
|
|
3
4
|
from numerapi import base_api
|
|
4
5
|
|
|
@@ -41,7 +42,7 @@ def destroy(ctx, preserve_node_config, verbose):
|
|
|
41
42
|
fg="red",
|
|
42
43
|
)
|
|
43
44
|
return
|
|
44
|
-
|
|
45
|
+
|
|
45
46
|
if not preserve_node_config:
|
|
46
47
|
click.secho("backing up nodes.json...")
|
|
47
48
|
copy_file(NODES_PATH, f"{NODES_PATH}.backup", force=True, verbose=True)
|
|
@@ -61,7 +61,7 @@ def test(ctx, local, command, verbose):
|
|
|
61
61
|
ctx.ensure_object(dict)
|
|
62
62
|
model = ctx.obj["model"]
|
|
63
63
|
node = model["name"]
|
|
64
|
-
|
|
64
|
+
tournament = model["tournament"]
|
|
65
65
|
node_config = load_or_init_nodes(node)
|
|
66
66
|
provider = node_config["provider"]
|
|
67
67
|
|
|
@@ -143,7 +143,6 @@ def test(ctx, local, command, verbose):
|
|
|
143
143
|
variables={"modelId": node_config["model_id"]},
|
|
144
144
|
authorization=True,
|
|
145
145
|
)
|
|
146
|
-
tournament = TOURNAMENT_SIGNALS if is_signals else TOURNAMENT_NUMERAI
|
|
147
146
|
curr_round = api.get_current_round(tournament)
|
|
148
147
|
latest_subs = sorted(
|
|
149
148
|
filter(
|
|
@@ -338,7 +337,7 @@ def get_recent_task_status_aws(cluster_arn, ecs_client, node, trigger_id):
|
|
|
338
337
|
"red",
|
|
339
338
|
)
|
|
340
339
|
elif matched_task["lastStatus"] in stopped_codes:
|
|
341
|
-
return matched_task, True, "Job execution
|
|
340
|
+
return matched_task, True, "Job execution finished!\r", "green"
|
|
342
341
|
elif matched_task["lastStatus"] in pending_codes:
|
|
343
342
|
return matched_task, False, "Waiting for job to start...", "yellow"
|
|
344
343
|
elif matched_task["lastStatus"] in running_codes:
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""Setup command for Numerai CLI"""
|
|
2
|
+
|
|
2
3
|
import click
|
|
3
4
|
import logging
|
|
4
5
|
|
|
@@ -16,8 +17,9 @@ from numerai.cli.util.keys import config_numerai_keys, config_provider_keys
|
|
|
16
17
|
prompt=True,
|
|
17
18
|
help=f"Initialize with this providers API keys.",
|
|
18
19
|
)
|
|
20
|
+
@click.option("--skip-key-setup", "-s", is_flag=True)
|
|
19
21
|
@click.option("--verbose", "-v", is_flag=True)
|
|
20
|
-
def setup(provider, verbose):
|
|
22
|
+
def setup(provider, skip_key_setup, verbose):
|
|
21
23
|
"""
|
|
22
24
|
Initializes cli and provider API keys.
|
|
23
25
|
"""
|
|
@@ -38,20 +40,22 @@ def setup(provider, verbose):
|
|
|
38
40
|
return
|
|
39
41
|
|
|
40
42
|
# setup numerai keys
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
43
|
+
if not skip_key_setup:
|
|
44
|
+
click.secho(
|
|
45
|
+
"Initializing numerai keys " "(press enter to keep value in brackets)...",
|
|
46
|
+
fg="yellow",
|
|
47
|
+
)
|
|
48
|
+
maybe_create(KEYS_PATH, protected=True)
|
|
49
|
+
config_numerai_keys()
|
|
47
50
|
|
|
48
51
|
# setup provider keys
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
52
|
+
if not skip_key_setup:
|
|
53
|
+
click.secho(
|
|
54
|
+
f"\nInitializing {provider} keys "
|
|
55
|
+
f"(press enter to keep value in brackets)...",
|
|
56
|
+
fg="yellow",
|
|
57
|
+
)
|
|
58
|
+
config_provider_keys(provider)
|
|
55
59
|
|
|
56
60
|
# copy tf files
|
|
57
61
|
click.secho("copying terraform files...")
|
|
@@ -166,9 +166,16 @@ def build(node_config, node, verbose):
|
|
|
166
166
|
build_arg_str += f" --build-arg NODE={node}"
|
|
167
167
|
|
|
168
168
|
cmd = (
|
|
169
|
-
f'docker build --platform=linux/amd64 -t {node_config["docker_repo"]}'
|
|
169
|
+
f'docker build --platform=linux/amd64 --load -t {node_config["docker_repo"]}'
|
|
170
|
+
f"{build_arg_str} -f {path}/Dockerfile ."
|
|
170
171
|
)
|
|
171
|
-
execute(cmd, verbose)
|
|
172
|
+
stdout, stderr = execute(cmd, verbose)
|
|
173
|
+
if "unknown flag: --load" in (stdout.decode() + stderr.decode()):
|
|
174
|
+
click.secho(
|
|
175
|
+
"Docker version too old, please upgrade to at least 18.09",
|
|
176
|
+
fg="red",
|
|
177
|
+
)
|
|
178
|
+
exit(1)
|
|
172
179
|
|
|
173
180
|
|
|
174
181
|
def run(node_config, verbose, command=""):
|
|
@@ -181,7 +188,9 @@ def login(node_config, verbose):
|
|
|
181
188
|
username, password = login_aws()
|
|
182
189
|
login_url = node_config['docker_repo']
|
|
183
190
|
elif node_config["provider"] == PROVIDER_AZURE:
|
|
184
|
-
username, password = login_azure(
|
|
191
|
+
username, password = login_azure(
|
|
192
|
+
node_config["registry_rg_name"], node_config["registry_name"]
|
|
193
|
+
)
|
|
185
194
|
login_url = node_config['docker_repo']
|
|
186
195
|
elif node_config["provider"] == PROVIDER_GCP:
|
|
187
196
|
username, password = login_gcp()
|
|
@@ -214,17 +223,23 @@ def login_aws():
|
|
|
214
223
|
)
|
|
215
224
|
|
|
216
225
|
token = ecr_client.get_authorization_token() # TODO: use registryIds
|
|
217
|
-
username, password =
|
|
226
|
+
username, password = (
|
|
227
|
+
base64.b64decode(token["authorizationData"][0]["authorizationToken"])
|
|
228
|
+
.decode()
|
|
229
|
+
.split(":")
|
|
230
|
+
)
|
|
218
231
|
|
|
219
232
|
return username, password
|
|
220
233
|
|
|
221
234
|
|
|
222
235
|
def login_azure(resource_group_name, registry_name):
|
|
223
236
|
azure_subs_id, azure_client, azure_tenant, azure_secret = get_azure_keys()
|
|
224
|
-
credentials = ClientSecretCredential(
|
|
225
|
-
|
|
226
|
-
resource_group_name, registry_name
|
|
237
|
+
credentials = ClientSecretCredential(
|
|
238
|
+
client_id=azure_client, tenant_id=azure_tenant, client_secret=azure_secret
|
|
227
239
|
)
|
|
240
|
+
username_password = ContainerRegistryManagementClient(
|
|
241
|
+
credentials, azure_subs_id
|
|
242
|
+
).registries.list_credentials(resource_group_name, registry_name)
|
|
228
243
|
username = username_password.username
|
|
229
244
|
password = username_password.passwords[0].value
|
|
230
245
|
return username, password
|
|
@@ -288,24 +303,32 @@ def cleanup_aws(docker_repo):
|
|
|
288
303
|
|
|
289
304
|
docker_repo_name = docker_repo.split("/")[-1]
|
|
290
305
|
|
|
291
|
-
resp = ecr_client.list_images(
|
|
306
|
+
resp = ecr_client.list_images(
|
|
307
|
+
repositoryName=docker_repo_name, filter={"tagStatus": "UNTAGGED"}
|
|
308
|
+
)
|
|
292
309
|
|
|
293
310
|
imageIds = resp["imageIds"]
|
|
294
311
|
if len(imageIds) == 0:
|
|
295
312
|
return []
|
|
296
313
|
|
|
297
|
-
resp = ecr_client.batch_delete_image(
|
|
314
|
+
resp = ecr_client.batch_delete_image(
|
|
315
|
+
repositoryName=docker_repo_name, imageIds=imageIds
|
|
316
|
+
)
|
|
298
317
|
|
|
299
318
|
return resp["imageIds"]
|
|
300
319
|
|
|
301
320
|
|
|
302
321
|
def cleanup_azure(node_config):
|
|
303
322
|
_, azure_client, azure_tenant, azure_secret = get_azure_keys()
|
|
304
|
-
credentials = ClientSecretCredential(
|
|
323
|
+
credentials = ClientSecretCredential(
|
|
324
|
+
client_id=azure_client, tenant_id=azure_tenant, client_secret=azure_secret
|
|
325
|
+
)
|
|
305
326
|
acr_client = ContainerRegistryClient(node_config["acr_login_server"], credentials)
|
|
306
327
|
docker_repo = node_config["docker_repo"]
|
|
307
328
|
node_repo_name = [
|
|
308
|
-
repo_name
|
|
329
|
+
repo_name
|
|
330
|
+
for repo_name in acr_client.list_repository_names()
|
|
331
|
+
if repo_name == docker_repo.split("/")[-1]
|
|
309
332
|
][0]
|
|
310
333
|
|
|
311
334
|
# get all manifests, ordered by last update time
|
|
@@ -318,7 +341,9 @@ def cleanup_azure(node_config):
|
|
|
318
341
|
# Remove all but the latest manifest
|
|
319
342
|
removed_manifests = []
|
|
320
343
|
for manifest in manifest_list[1:]:
|
|
321
|
-
acr_client.update_manifest_properties(
|
|
344
|
+
acr_client.update_manifest_properties(
|
|
345
|
+
node_repo_name, manifest.digest, can_write=True, can_delete=True
|
|
346
|
+
)
|
|
322
347
|
removed_manifests.append(manifest.digest)
|
|
323
348
|
acr_client.delete_manifest(node_repo_name, manifest.digest)
|
|
324
349
|
return removed_manifests
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
from numerai.cli.constants import PROVIDERS, NODES_PATH
|
|
4
|
+
from numerai.cli.util.docker import terraform
|
|
5
|
+
from numerai.cli.util import docker
|
|
6
|
+
from numerai.cli.util.files import load_or_init_nodes, store_config
|
|
7
|
+
from numerai.cli.util.keys import load_or_init_keys
|
|
8
|
+
|
|
9
|
+
import click
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def apply_terraform(nodes_config, affected_providers, provider, verbose):
|
|
13
|
+
# Apply terraform for any affected provider
|
|
14
|
+
for affected_provider in affected_providers:
|
|
15
|
+
if affected_provider in PROVIDERS:
|
|
16
|
+
click.secho(f"Updating resources in {affected_provider}")
|
|
17
|
+
terraform(
|
|
18
|
+
"apply -auto-approve",
|
|
19
|
+
verbose,
|
|
20
|
+
affected_provider,
|
|
21
|
+
env_vars=load_or_init_keys(affected_provider),
|
|
22
|
+
inputs={"node_config_file": "nodes.json"},
|
|
23
|
+
)
|
|
24
|
+
else:
|
|
25
|
+
click.secho(f"provider {affected_provider} not supported", fg="red")
|
|
26
|
+
exit(1)
|
|
27
|
+
click.secho("cloud resources created successfully", fg="green")
|
|
28
|
+
|
|
29
|
+
# terraform output for node config, same for aws and azure
|
|
30
|
+
click.echo(f"saving node configuration to {NODES_PATH}...")
|
|
31
|
+
|
|
32
|
+
res = terraform(f"output -json {provider}_nodes", verbose, provider).decode("utf-8")
|
|
33
|
+
try:
|
|
34
|
+
nodes = json.loads(res)
|
|
35
|
+
except json.JSONDecodeError:
|
|
36
|
+
click.secho("failed to save node configuration, please retry.", fg="red")
|
|
37
|
+
return
|
|
38
|
+
for node_name, data in nodes.items():
|
|
39
|
+
nodes_config[node_name].update(data)
|
|
40
|
+
|
|
41
|
+
store_config(NODES_PATH, nodes_config)
|
|
42
|
+
if verbose:
|
|
43
|
+
click.secho(f"new config:\n{json.dumps(load_or_init_nodes(), indent=2)}")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def create_azure_registry(provider, provider_keys, verbose):
|
|
47
|
+
"""Creates a registry for azure"""
|
|
48
|
+
terraform("init -upgrade", verbose, provider)
|
|
49
|
+
terraform(
|
|
50
|
+
'apply -target="azurerm_container_registry.registry[0]" -target="azurerm_resource_group.acr_rg[0]" -auto-approve ',
|
|
51
|
+
verbose,
|
|
52
|
+
"azure",
|
|
53
|
+
env_vars=provider_keys,
|
|
54
|
+
inputs={"node_config_file": "nodes.json"},
|
|
55
|
+
)
|
|
56
|
+
res = terraform("output -json acr_repo_details", True, provider).decode("utf-8")
|
|
57
|
+
return json.loads(res)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def create_gcp_registry(provider, verbose):
|
|
61
|
+
"""Creates a registry for GCP"""
|
|
62
|
+
terraform("init -upgrade", verbose, provider)
|
|
63
|
+
terraform(
|
|
64
|
+
'apply -target="google_project_service.cloud_resource_manager" -auto-approve ',
|
|
65
|
+
verbose,
|
|
66
|
+
"gcp",
|
|
67
|
+
inputs={"node_config_file": "nodes.json"},
|
|
68
|
+
)
|
|
69
|
+
terraform(
|
|
70
|
+
'apply -target="google_artifact_registry_repository.registry[0]" -auto-approve ',
|
|
71
|
+
verbose,
|
|
72
|
+
"gcp",
|
|
73
|
+
inputs={"node_config_file": "nodes.json"},
|
|
74
|
+
)
|
|
75
|
+
res = terraform("output -json artifact_registry_details", True, provider).decode(
|
|
76
|
+
"utf-8"
|
|
77
|
+
)
|
|
78
|
+
return json.loads(res)
|
|
@@ -4,9 +4,9 @@ FROM python:3.9
|
|
|
4
4
|
# These are docker arguments that `numerai node deploy/test` will always pass into docker.
|
|
5
5
|
# They are then set in your environment so that numerapi can access them when uploading submissions.
|
|
6
6
|
# You can also access them from your script like so:
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
# import os
|
|
8
|
+
# public_id = os.environ["NUMERAI_PUBLIC_ID"]
|
|
9
|
+
# secret_key = os.environ["NUMERAI_SECRET_KEY"]
|
|
10
10
|
ARG NUMERAI_PUBLIC_ID
|
|
11
11
|
ENV NUMERAI_PUBLIC_ID=$NUMERAI_PUBLIC_ID
|
|
12
12
|
|