numerai-cli 1.1.0__tar.gz → 1.1.2__tar.gz

{numerai-cli-1.1.0 → numerai-cli-1.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: numerai-cli
-Version: 1.1.0
+Version: 1.1.2
 Summary: A library for deploying Numer.ai Prediction Nodes.
 Home-page: https://github.com/numerai/numerai-cli
 Author: Numer.ai

{numerai-cli-1.1.0 → numerai-cli-1.1.2}/numerai/cli/__init__.py

@@ -35,5 +35,6 @@ def main():
     numerai.add_command(upgrade.upgrade)
     numerai.add_command(misc.copy_example)
     numerai.add_command(misc.list_constants)
+    numerai.add_command(misc.add_volume_aws)
     numerai.add_command(destroy_all.destroy_all)
     numerai()

{numerai-cli-1.1.0 → numerai-cli-1.1.2}/numerai/cli/destroy_all.py

@@ -1,4 +1,5 @@
 """Destroy command for Numerai CLI"""
+
 import click
 from numerapi import base_api
 

{numerai-cli-1.1.0 → numerai-cli-1.1.2}/numerai/cli/misc.py

@@ -1,9 +1,8 @@
-import json
-
-import click
-
 from numerai.cli.constants import *
 from numerai.cli.util import files
+from numerai.cli.util.terraform import apply_terraform
+
+import click
 
 
 @click.command()
@@ -12,13 +11,15 @@ from numerai.cli.util import files
     "-e",
     type=click.Choice(EXAMPLES),
     default=DEFAULT_EXAMPLE,
-    help=f"Specify the example to copy, defaults to {DEFAULT_EXAMPLE}. " f"Options are {EXAMPLES}.",
+    help=f"Specify the example to copy, defaults to {DEFAULT_EXAMPLE}. "
+    f"Options are {EXAMPLES}.",
 )
 @click.option(
     "--dest",
     "-d",
     type=str,
-    help=f"Destination folder to which example code is written. " f"Defaults to the name of the example.",
+    help=f"Destination folder to which example code is written. "
+    f"Defaults to the name of the example.",
 )
 @click.option("--verbose", "-v", is_flag=True)
 def copy_example(example, dest, verbose):
@@ -49,14 +50,50 @@ def list_constants():
         click.secho(
             f"  {size} -> cpus: {preset[0] / 1024}, "
             f"mem: {preset[1] / 1024} GB {suffix}",
-            fg="green"
-            if size == DEFAULT_SIZE or size == DEFAULT_SIZE_GCP
-            else "yellow",
+            fg=(
+                "green"
+                if size == DEFAULT_SIZE or size == DEFAULT_SIZE_GCP
+                else "yellow"
+            ),
         )
-    click.secho("Due to GCP Cloud Run size constraints, 'mem' sizes are not allowed when using GCP.")
+    click.secho(
+        "Due to GCP Cloud Run size constraints, 'mem' sizes are not allowed when using GCP."
+    )
     click.secho(
         "For AWS, use one of these sizes, or specify your own CPU and Memory in cores and GB using --cpu and --memory options.\n"
         "See https://learn.microsoft.com/en-us/azure/container-apps/containers#configuration for Azure,\n"
         "or https://cloud.google.com/run/docs/configuring/services/memory-limits for GCP \n"
         "to learn more info about allowed size presets for those providers."
     )
+
+
+@click.command()
+@click.option(
+    "--size",
+    "-s",
+    type=int,
+    required=True,
+    help="Specify the volume size in GB you'd like your AWS nodes to share.",
+)
+@click.option("--verbose", "-v", is_flag=True)
+def add_volume_aws(size, verbose):
+    """
+    Set the volume size for AWS nodes. This volume is shared by all nodes.
+    """
+    click.secho("Setting volume size for AWS nodes...", fg="yellow")
+    # get nodes config object
+    nodes_config = files.load_or_init_nodes()
+    print(nodes_config)
+    # set volume size for all nodes to same size
+    for node in nodes_config:
+        nodes_config[node]["volume"] = size
+    files.store_config(NODES_PATH, nodes_config)
+    files.copy_file(
+        NODES_PATH,
+        f"{CONFIG_PATH}/{PROVIDER_AWS}/",
+        force=True,
+        verbose=verbose,
+    )
+    click.secho(f"Applying terraform to add {size} GB volume...", fg="yellow")
+    apply_terraform(nodes_config, [PROVIDER_AWS], PROVIDER_AWS, verbose=verbose)
+    click.secho("Volume size updated successfully!", fg="green")

{numerai-cli-1.1.0 → numerai-cli-1.1.2}/numerai/cli/node/__init__.py

@@ -25,7 +25,6 @@ def tournaments_dict():
 def get_models(tournament):
     napi = base_api.Api(*get_numerai_keys())
     models = napi.get_models(tournament)
-    tournaments = napi.raw_query('query { tournaments { name tournament } }')
     name_prefix = tournaments_dict()[tournament]
     model_dict = {}
     for model_name, model_id in models.items():

{numerai-cli-1.1.0 → numerai-cli-1.1.2}/numerai/cli/node/config.py

@@ -1,8 +1,7 @@
 """Config command for Numerai CLI"""
 
-import json
 import os
-import click
+
 from numerapi import base_api
 from numerai.cli.constants import (
     DEFAULT_PROVIDER,
@@ -19,7 +18,6 @@ from numerai.cli.constants import (
     CONFIG_PATH,
     PROVIDER_GCP,
 )
-from numerai.cli.util.docker import terraform, check_for_dockerfile
 from numerai.cli.util import docker
 from numerai.cli.util.files import (
     load_or_init_nodes,
@@ -27,7 +25,14 @@ from numerai.cli.util.files import (
     copy_example,
     copy_file,
 )
-from numerai.cli.util.keys import get_provider_keys, get_numerai_keys, load_or_init_keys
+from numerai.cli.util.keys import get_provider_keys, get_numerai_keys
+from numerai.cli.util.terraform import (
+    apply_terraform,
+    create_azure_registry,
+    create_gcp_registry,
+)
+
+import click
 
 
 @click.command()
@@ -95,12 +100,6 @@ from numerai.cli.util.keys import get_provider_keys, get_numerai_keys, load_or_i
     help="Forces your webhook to register with Numerai. "
     "Use in conjunction with options that prevent webhook auto-registering.",
 )
-@click.option(
-    "--volume",
-    "-v",
-    type=int,
-    help="Specify additional block storage in GB. Currently only supported in AWS.",
-)
 @click.pass_context
 def config(
     ctx,
@@ -114,7 +113,6 @@ def config(
     cron,
     timeout_minutes,
     register_webhook,
-    volume,
 ):
     """
     Uses Terraform to create a full Numerai Compute cluster in your desired provider.
@@ -234,7 +232,7 @@ def config(
     click.secho(f'Current node config: "{node_conf}"...')
 
     # double check there is a dockerfile in the path we are about to configure
-    check_for_dockerfile(nodes_config[node]["path"])
+    docker.check_for_dockerfile(nodes_config[node]["path"])
     store_config(NODES_PATH, nodes_config)
 
     # Added after tf directory restructure: copy nodes.json to providers' tf directory
@@ -284,43 +282,10 @@ def config(
             docker.tag("hello-world:linux", node_conf["docker_repo"], verbose)
             docker.push(node_conf["docker_repo"], verbose)
         nodes_config[node] = node_conf
-    elif provider == "aws":
-        if volume is not None:
-            node_conf["volume"] = volume
 
     store_config(NODES_PATH, nodes_config)
 
-    # Apply terraform for any affected provider
-    for affected_provider in affected_providers:
-        if affected_provider in PROVIDERS:
-            click.secho(f"Updating resources in {affected_provider}")
-            terraform(
-                "apply -auto-approve",
-                verbose,
-                affected_provider,
-                env_vars=load_or_init_keys(affected_provider),
-                inputs={"node_config_file": "nodes.json"},
-            )
-        else:
-            click.secho(f"provider {affected_provider} not supported", fg="red")
-            exit(1)
-    click.secho("cloud resources created successfully", fg="green")
-
-    # terraform output for node config, same for aws and azure
-    click.echo(f"saving node configuration to {NODES_PATH}...")
-
-    res = terraform(f"output -json {provider}_nodes", verbose, provider).decode("utf-8")
-    try:
-        nodes = json.loads(res)
-    except json.JSONDecodeError:
-        click.secho("failed to save node configuration, please retry.", fg="red")
-        return
-    for node_name, data in nodes.items():
-        nodes_config[node_name].update(data)
-
-    store_config(NODES_PATH, nodes_config)
-    if verbose:
-        click.secho(f"new config:\n{json.dumps(load_or_init_nodes(), indent=2)}")
+    apply_terraform(nodes_config, affected_providers, provider, verbose)
 
     webhook_url = nodes_config[node]["webhook_url"]
     napi = base_api.Api(*get_numerai_keys())
@@ -336,38 +301,3 @@ def config(
         "Prediction Node configured successfully. " "Next: deploy and test your node",
         fg="green",
     )
-
-
-def create_azure_registry(provider, provider_keys, verbose):
-    """Creates a registry for azure"""
-    terraform("init -upgrade", verbose, provider)
-    terraform(
-        'apply -target="azurerm_container_registry.registry[0]" -target="azurerm_resource_group.acr_rg[0]" -auto-approve ',
-        verbose,
-        "azure",
-        env_vars=provider_keys,
-        inputs={"node_config_file": "nodes.json"},
-    )
-    res = terraform("output -json acr_repo_details", True, provider).decode("utf-8")
-    return json.loads(res)
-
-
-def create_gcp_registry(provider, verbose):
-    """Creates a registry for GCP"""
-    terraform("init -upgrade", verbose, provider)
-    terraform(
-        'apply -target="google_project_service.cloud_resource_manager" -auto-approve ',
-        verbose,
-        "gcp",
-        inputs={"node_config_file": "nodes.json"},
-    )
-    terraform(
-        'apply -target="google_artifact_registry_repository.registry[0]" -auto-approve ',
-        verbose,
-        "gcp",
-        inputs={"node_config_file": "nodes.json"},
-    )
-    res = terraform("output -json artifact_registry_details", True, provider).decode(
-        "utf-8"
-    )
-    return json.loads(res)

{numerai-cli-1.1.0 → numerai-cli-1.1.2}/numerai/cli/node/destroy.py

@@ -1,4 +1,5 @@
 """Destroy command for Numerai CLI"""
+
 import click
 from numerapi import base_api
 
@@ -41,7 +42,7 @@ def destroy(ctx, preserve_node_config, verbose):
             fg="red",
         )
         return
-    
+
     if not preserve_node_config:
         click.secho("backing up nodes.json...")
         copy_file(NODES_PATH, f"{NODES_PATH}.backup", force=True, verbose=True)

{numerai-cli-1.1.0 → numerai-cli-1.1.2}/numerai/cli/setup.py

@@ -1,4 +1,5 @@
 """Setup command for Numerai CLI"""
+
 import click
 import logging
 
@@ -16,8 +17,9 @@ from numerai.cli.util.keys import config_numerai_keys, config_provider_keys
     prompt=True,
     help=f"Initialize with this providers API keys.",
 )
+@click.option("--skip-key-setup", "-s", is_flag=True)
 @click.option("--verbose", "-v", is_flag=True)
-def setup(provider, verbose):
+def setup(provider, skip_key_setup, verbose):
     """
     Initializes cli and provider API keys.
     """
@@ -38,20 +40,22 @@ def setup(provider, verbose):
         return
 
     # setup numerai keys
-    click.secho(
-        "Initializing numerai keys " "(press enter to keep value in brackets)...",
-        fg="yellow",
-    )
-    maybe_create(KEYS_PATH, protected=True)
-    config_numerai_keys()
+    if not skip_key_setup:
+        click.secho(
+            "Initializing numerai keys " "(press enter to keep value in brackets)...",
+            fg="yellow",
+        )
+        maybe_create(KEYS_PATH, protected=True)
+        config_numerai_keys()
 
     # setup provider keys
-    click.secho(
-        f"\nInitializing {provider} keys "
-        f"(press enter to keep value in brackets)...",
-        fg="yellow",
-    )
-    config_provider_keys(provider)
+    if not skip_key_setup:
+        click.secho(
+            f"\nInitializing {provider} keys "
+            f"(press enter to keep value in brackets)...",
+            fg="yellow",
+        )
+        config_provider_keys(provider)
 
     # copy tf files
     click.secho("copying terraform files...")

numerai-cli-1.1.2/numerai/cli/util/terraform.py

@@ -0,0 +1,78 @@
+import json
+
+from numerai.cli.constants import PROVIDERS, NODES_PATH
+from numerai.cli.util.docker import terraform
+from numerai.cli.util import docker
+from numerai.cli.util.files import load_or_init_nodes, store_config
+from numerai.cli.util.keys import load_or_init_keys
+
+import click
+
+
+def apply_terraform(nodes_config, affected_providers, provider, verbose):
+    # Apply terraform for any affected provider
+    for affected_provider in affected_providers:
+        if affected_provider in PROVIDERS:
+            click.secho(f"Updating resources in {affected_provider}")
+            terraform(
+                "apply -auto-approve",
+                verbose,
+                affected_provider,
+                env_vars=load_or_init_keys(affected_provider),
+                inputs={"node_config_file": "nodes.json"},
+            )
+        else:
+            click.secho(f"provider {affected_provider} not supported", fg="red")
+            exit(1)
+    click.secho("cloud resources created successfully", fg="green")
+
+    # terraform output for node config, same for aws and azure
+    click.echo(f"saving node configuration to {NODES_PATH}...")
+
+    res = terraform(f"output -json {provider}_nodes", verbose, provider).decode("utf-8")
+    try:
+        nodes = json.loads(res)
+    except json.JSONDecodeError:
+        click.secho("failed to save node configuration, please retry.", fg="red")
+        return
+    for node_name, data in nodes.items():
+        nodes_config[node_name].update(data)
+
+    store_config(NODES_PATH, nodes_config)
+    if verbose:
+        click.secho(f"new config:\n{json.dumps(load_or_init_nodes(), indent=2)}")
+
+
+def create_azure_registry(provider, provider_keys, verbose):
+    """Creates a registry for azure"""
+    terraform("init -upgrade", verbose, provider)
+    terraform(
+        'apply -target="azurerm_container_registry.registry[0]" -target="azurerm_resource_group.acr_rg[0]" -auto-approve ',
+        verbose,
+        "azure",
+        env_vars=provider_keys,
+        inputs={"node_config_file": "nodes.json"},
+    )
+    res = terraform("output -json acr_repo_details", True, provider).decode("utf-8")
+    return json.loads(res)
+
+
+def create_gcp_registry(provider, verbose):
+    """Creates a registry for GCP"""
+    terraform("init -upgrade", verbose, provider)
+    terraform(
+        'apply -target="google_project_service.cloud_resource_manager" -auto-approve ',
+        verbose,
+        "gcp",
+        inputs={"node_config_file": "nodes.json"},
+    )
+    terraform(
+        'apply -target="google_artifact_registry_repository.registry[0]" -auto-approve ',
+        verbose,
+        "gcp",
+        inputs={"node_config_file": "nodes.json"},
+    )
+    res = terraform("output -json artifact_registry_details", True, provider).decode(
+        "utf-8"
+    )
+    return json.loads(res)

{numerai-cli-1.1.0 → numerai-cli-1.1.2}/numerai/terraform/aws/-inputs.tf

@@ -27,9 +27,3 @@ variable "gateway_stage_path" {
   type        = string
   default     = "v1"
 }
-
-variable "volume_size" {
-  description = "Size of the EC2 volumes in GB"
-  type        = number
-  default     = 0
-}

{numerai-cli-1.1.0 → numerai-cli-1.1.2}/numerai/terraform/aws/-main.tf

@@ -23,5 +23,4 @@ module "aws" {
   nodes               = local.aws_nodes
   node_container_port = var.node_container_port
   gateway_stage_path  = var.gateway_stage_path
-  volume_size         = var.volume_size
 }

{numerai-cli-1.1.0 → numerai-cli-1.1.2}/numerai/terraform/aws/aws/-inputs.tf

@@ -26,9 +26,3 @@ variable "gateway_stage_path" {
   type        = string
   default     = "v1"
 }
-
-variable "volume_size" {
-  description = "Size of the EC2 volumes in GB"
-  type        = number
-  nullable    = true
-}

numerai-cli-1.1.2/numerai/terraform/aws/aws/-locals.tf

@@ -0,0 +1,4 @@
+locals {
+  node_prefix = "numerai-submission"
+  max_node_volume_size = max([for node, config in var.nodes : config.volume]...)
+}

{numerai-cli-1.1.0 → numerai-cli-1.1.2}/numerai/terraform/aws/aws/cluster.tf

@@ -94,14 +94,15 @@ data "aws_ami" "ecs_al2" {
 
 resource "aws_launch_template" "node" {
   image_id = data.aws_ami.ecs_al2.id
+  update_default_version = true
   dynamic "block_device_mappings" {
-    for_each = var.volume_size > 0 ? {size: var.volume_size} : {}
+    for_each = local.max_node_volume_size > 0 ? {size: local.max_node_volume_size} : {}
     content {
       device_name = "/dev/xvda"
 
       ebs {
         encrypted = true
-        volume_size = each.size
+        volume_size = local.max_node_volume_size
         volume_type = "gp3"
       }
     }
@@ -116,7 +117,7 @@ resource "aws_batch_compute_environment" "node" {
 
     launch_template {
       launch_template_id = aws_launch_template.node.id
-      version            = "$Latest"
+      version            = aws_launch_template.node.latest_version
     }
 
     max_vcpus = 64
@@ -142,22 +143,8 @@ resource "aws_batch_compute_environment" "node" {
 }
 
 
-#############
-# Job Setup #
-#############
-
-resource "aws_cloudwatch_log_group" "ec2" {
-  for_each = { for name, config in var.nodes : name => config }
-
-  name              = "/ec2/service/${each.key}"
-  retention_in_days = "14"
-}
-
-
 resource "aws_batch_job_queue" "node" {
-  for_each = { for name, config in var.nodes : name => config }
-
-  name = each.key
+  name = "${local.node_prefix}-queue"
 
   state    = "ENABLED"
   priority = 1
@@ -168,6 +155,18 @@ resource "aws_batch_job_queue" "node" {
   }
 }
 
+
+#############
+# Job Setup #
+#############
+
+resource "aws_cloudwatch_log_group" "ec2" {
+  for_each = { for name, config in var.nodes : name => config }
+
+  name              = "/ec2/service/${each.key}"
+  retention_in_days = "14"
+}
+
 resource "aws_batch_job_definition" "node" {
   for_each = { for name, config in var.nodes : name => config }
 

{numerai-cli-1.1.0 → numerai-cli-1.1.2}/numerai/terraform/aws/aws/webhook.tf

@@ -15,7 +15,7 @@ resource "aws_lambda_function" "submission" {
   environment {
     variables = {
       JOB_DEFINITION = aws_batch_job_definition.node[each.key].name
-      JOB_QUEUE      = aws_batch_job_queue.node[each.key].name
+      JOB_QUEUE      = aws_batch_job_queue.node.name
     }
   }
 }

{numerai-cli-1.1.0 → numerai-cli-1.1.2}/numerai_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: numerai-cli
-Version: 1.1.0
+Version: 1.1.2
 Summary: A library for deploying Numer.ai Prediction Nodes.
 Home-page: https://github.com/numerai/numerai-cli
 Author: Numer.ai

{numerai-cli-1.1.0 → numerai-cli-1.1.2}/numerai_cli.egg-info/SOURCES.txt

@@ -20,6 +20,7 @@ numerai/cli/util/debug.py
 numerai/cli/util/docker.py
 numerai/cli/util/files.py
 numerai/cli/util/keys.py
+numerai/cli/util/terraform.py
 numerai/examples/crypto-python3/Dockerfile
 numerai/examples/crypto-python3/predict.py
 numerai/examples/crypto-python3/requirements.txt

{numerai-cli-1.1.0 → numerai-cli-1.1.2}/setup.py

@@ -9,7 +9,7 @@ with open(path.join(this_directory, "README.md"), encoding="utf-8") as f:
 
 setup(
     name="numerai-cli",
-    version="1.1.0",
+    version="1.1.2",
     description="A library for deploying Numer.ai Prediction Nodes.",
     url="https://github.com/numerai/numerai-cli",
     author="Numer.ai",

numerai-cli-1.1.0/numerai/terraform/gcp/gcp/-locals.tf

@@ -1,3 +0,0 @@
-locals {
-  node_prefix = "numerai-submission"
-}