wandb 0.17.0rc2__py3-none-any.whl → 0.17.2__py3-none-any.whl

wandb/cli/cli.py

@@ -30,10 +30,12 @@ import wandb
 import wandb.env
 
 # from wandb.old.core import wandb_dir
+import wandb.errors
 import wandb.sdk.verify.verify as wandb_verify
 from wandb import Config, Error, env, util, wandb_agent, wandb_sdk
 from wandb.apis import InternalApi, PublicApi
 from wandb.apis.public import RunQueue
+from wandb.errors import WandbCoreNotAvailableError
 from wandb.integration.magic import magic_install
 from wandb.sdk.artifacts.artifact_file_cache import get_artifact_file_cache
 from wandb.sdk.launch import utils as launch_utils
@@ -44,6 +46,7 @@ from wandb.sdk.launch.sweeps.scheduler import Scheduler
 from wandb.sdk.lib import filesystem
 from wandb.sdk.lib.wburls import wburls
 from wandb.sync import SyncManager, get_run_from_path, get_runs
+from wandb.util import get_core_path
 
 # Send cli logs to wandb/debug-cli.<username>.log by default and fallback to a temp dir.
 _wandb_dir = wandb.old.core.wandb_dir(env.get_dir())
@@ -194,9 +197,9 @@ def projects(entity, display=True):
     api = _get_cling_api()
     projects = api.list_projects(entity=entity)
     if len(projects) == 0:
-        message = "No projects found for %s" % entity
+        message = "No projects found for {}".format(entity)
     else:
-        message = 'Latest projects for "%s"' % entity
+        message = 'Latest projects for "{}"'.format(entity)
     if display:
         click.echo(click.style(message, bold=True))
         for project in projects:
@@ -420,7 +423,7 @@ def init(ctx, project, entity, reset, mode):
         """
         ).format(
             code1=click.style("import wandb", bold=True),
-            code2=click.style('wandb.init(project="%s")' % project, bold=True),
+            code2=click.style('wandb.init(project="{}")'.format(project), bold=True),
             run=click.style("python <train.py>", bold=True),
         )
     )
@@ -430,14 +433,15 @@ def init(ctx, project, entity, reset, mode):
 def beta():
     """Beta versions of wandb CLI commands. Requires wandb-core."""
     # this is the future that requires wandb-core!
-    from wandb.util import get_core_path
+    wandb._sentry.configure_scope(process_context="wandb_beta")
+    wandb.require("core")
 
-    if not get_core_path():
+    try:
+        get_core_path()
+    except WandbCoreNotAvailableError as e:
+        wandb._sentry.exception(f"using `wandb beta`. failed with {e}")
         click.secho(
-            (
-                "wandb beta commands require wandb-core, please install with"
-                " `pip install wandb-core`"
-            ),
+            (e),
             fg="red",
             err=True,
         )
@@ -876,6 +880,14 @@ def sync(
     default=False,
     help="Resume a sweep to continue running new runs.",
 )
+@click.option(
+    "--prior_run",
+    "-R",
+    "prior_runs",
+    multiple=True,
+    default=None,
+    help="ID of an existing run to add to this sweep",
+)
 @click.argument("config_yaml_or_sweep_id")
 @click.pass_context
 @display_error
@@ -893,6 +905,7 @@ def sweep(
     cancel,
     pause,
     resume,
+    prior_runs,
     config_yaml_or_sweep_id,
 ):
     state_args = "stop", "cancel", "pause", "resume"
@@ -923,7 +936,7 @@ def sweep(
             "resume": "Resuming",
         }
         wandb.termlog(f"{ings[state]} sweep {entity}/{project}/{sweep_id}")
-        getattr(api, "%s_sweep" % state)(sweep_id, entity=entity, project=project)
+        getattr(api, "{}_sweep".format(state))(sweep_id, entity=entity, project=project)
         wandb.termlog("Done.")
         return
     else:
@@ -1034,6 +1047,7 @@ def sweep(
         project=project,
         entity=entity,
         obj_id=sweep_obj_id,
+        prior_runs=prior_runs,
     )
     sweep_utils.handle_sweep_config_violations(warnings)
 
@@ -1100,6 +1114,14 @@ def sweep(
     default=None,
     help="Resume a launch sweep by passing an 8-char sweep id. Queue required",
 )
+@click.option(
+    "--prior_run",
+    "-R",
+    "prior_runs",
+    multiple=True,
+    default=None,
+    help="ID of an existing run to add to this sweep",
+)
 @click.argument("config", required=False, type=click.Path(exists=True))
 @click.pass_context
 @display_error
@@ -1110,6 +1132,7 @@ def launch_sweep(
     queue,
     config,
     resume_id,
+    prior_runs,
 ):
     api = _get_cling_api()
     env = os.environ
@@ -1294,6 +1317,7 @@ def launch_sweep(
         obj_id=sweep_obj_id,  # if resuming
         launch_scheduler=launch_scheduler_with_queue,
         state="PENDING",
+        prior_runs=prior_runs,
     )
     sweep_utils.handle_sweep_config_violations(warnings)
     # Log nicely formatted sweep information
@@ -1307,7 +1331,14 @@ def launch_sweep(
 
 
 @cli.command(help=f"Launch or queue a W&B Job. See {wburls.get('cli_launch')}")
-@click.option("--uri", "-u", metavar="(str)", default=None, hidden=True)
+@click.option(
+    "--uri",
+    "-u",
+    metavar="(str)",
+    default=None,
+    help="Local path or git repo uri to launch. If provided this command will "
+    "create a job from the specified uri.",
+)
 @click.option(
     "--job",
     "-j",
@@ -1332,6 +1363,20 @@ def launch_sweep(
     hidden=True,
     help="Version of the project to run, as a Git commit reference for Git projects.",
 )
+@click.option(
+    "--build-context",
+    metavar="(str)",
+    help="Path to the build context within the source code. Defaults to the "
+    "root of the source code. Compatible only with -u.",
+)
+@click.option(
+    "--job-name",
+    "-J",
+    metavar="(str)",
+    default=None,
+    hidden=True,
+    help="Name for the job created if the -u,--uri flag is passed in.",
+)
 @click.option(
     "--name",
     envvar="WANDB_NAME",
@@ -1456,6 +1501,7 @@ def launch(
     job,
     entry_point,
     git_version,
+    build_context,
     name,
     resource,
     entity,
@@ -1471,6 +1517,7 @@ def launch(
     project_queue,
     dockerfile,
     priority,
+    job_name,
 ):
     """Start a W&B run from the given URI.
 
@@ -1486,6 +1533,8 @@ def launch(
         f"=== Launch called with kwargs {locals()} CLI Version: {wandb.__version__}==="
     )
     from wandb.sdk.launch._launch import _launch
+    from wandb.sdk.launch.create_job import _create_job
+    from wandb.sdk.launch.utils import _is_git_uri
 
     api = _get_cling_api()
     wandb._sentry.configure_scope(process_context="launch_cli")
@@ -1532,6 +1581,36 @@ def launch(
 
     run_id = config.get("run_id")
 
+    # If URI was provided, we need to create a job from it.
+    if uri:
+        if entry_point is None:
+            raise LaunchError(
+                "Cannot provide a uri without an entry point. Please provide an "
+                "entry point with --entry-point or -E."
+            )
+        if job is not None:
+            raise LaunchError("Cannot provide both a uri and a job name.")
+        job_type = (
+            "git" if _is_git_uri(uri) else "code"
+        )  # TODO: Add support for local URIs with git.
+        if entity is None:
+            entity = launch_utils.get_default_entity(api, config)
+        artifact, _, _ = _create_job(
+            api,
+            job_type,
+            uri,
+            entrypoint=" ".join(entry_point),
+            git_hash=git_version,
+            name=job_name,
+            project=project,
+            build_context=build_context,
+            dockerfile=dockerfile,
+            entity=entity,
+        )
+        if artifact is None:
+            raise LaunchError(f"Failed to create job from uri: {uri}")
+        job = f"{entity}/{project}/{artifact.name}"
+
     if dockerfile:
         if "overrides" in config:
             config["overrides"]["dockerfile"] = dockerfile
@@ -1565,7 +1644,6 @@ def launch(
             run = asyncio.run(
                 _launch(
                     api,
-                    uri,
                     job,
                     project=project,
                     entity=entity,
@@ -1602,7 +1680,6 @@ def launch(
         try:
             _launch_add(
                 api,
-                uri,
                 job,
                 config,
                 template_variables,
@@ -1641,13 +1718,6 @@ def launch(
     metavar="<queue(s)>",
     help="The name of a queue for the agent to watch. Multiple -q flags supported.",
 )
-@click.option(
-    "--project",
-    "-p",
-    default=None,
-    help="""Name of the project which the agent will watch.
-    If passed in, will override the project value passed in using a config file.""",
-)
 @click.option(
     "--entity",
     "-e",
@@ -1684,7 +1754,6 @@ def launch(
 @display_error
 def launch_agent(
     ctx,
-    project=None,
     entity=None,
     queues=None,
     max_jobs=None,
@@ -1709,7 +1778,7 @@ def launch_agent(
     api = _get_cling_api()
     wandb._sentry.configure_scope(process_context="launch_agent")
     agent_config, api = _launch.resolve_agent_config(
-        entity, project, max_jobs, queues, config, verbose
+        entity, max_jobs, queues, config, verbose
     )
 
     if len(agent_config.get("queues")) == 0:
@@ -1907,14 +1976,16 @@ def describe(job):
     "--entry-point",
     "-E",
     "entrypoint",
-    help="Entrypoint to the script, including an executable and an entrypoint file. Required for code or repo jobs",
+    help="Entrypoint to the script, including an executable and an entrypoint "
+    "file. Required for code or repo jobs. If --build-context is provided, "
+    "paths in the entrypoint command will be relative to the build context.",
 )
 @click.option(
     "--git-hash",
     "-g",
     "git_hash",
     type=str,
-    help="Hash to a specific git commit.",
+    help="Commit reference to use as the source for git jobs",
 )
 @click.option(
     "--runtime",
@@ -1922,6 +1993,20 @@ def describe(job):
     type=str,
     help="Python runtime to execute the job",
 )
+@click.option(
+    "--build-context",
+    "-b",
+    type=str,
+    help="Path to the build context from the root of the job source code. If "
+    "provided, this is used as the base path for the Dockerfile and entrypoint.",
+)
+@click.option(
+    "--dockerfile",
+    "-D",
+    type=str,
+    help="Path to the Dockerfile for the job. If --build-context is provided, "
+    "the Dockerfile path will be relative to the build context.",
+)
 @click.argument(
     "job_type",
     type=click.Choice(("git", "code", "image")),
@@ -1938,6 +2023,8 @@ def create(
     entrypoint,
     git_hash,
     runtime,
+    build_context,
+    dockerfile,
 ):
     """Create a job from a source, without a wandb run.
 
@@ -1980,6 +2067,8 @@ def create(
         entrypoint=entrypoint,
         git_hash=git_hash,
         runtime=runtime,
+        build_context=build_context,
+        dockerfile=dockerfile,
     )
     if not artifact:
         wandb.termerror("Job creation failed")
@@ -2038,13 +2127,13 @@ def docker_run(ctx, docker_run_args):
     if image:
         resolved_image = wandb.docker.image_id(image)
     if resolved_image:
-        args = ["-e", "WANDB_DOCKER=%s" % resolved_image] + args
+        args = ["-e", "WANDB_DOCKER={}".format(resolved_image)] + args
     else:
         wandb.termlog(
             "Couldn't detect image argument, running command without the WANDB_DOCKER env variable"
         )
     if api.api_key:
-        args = ["-e", "WANDB_API_KEY=%s" % api.api_key] + args
+        args = ["-e", "WANDB_API_KEY={}".format(api.api_key)] + args
     else:
         wandb.termlog(
             "Not logged in, run `wandb login` from the host machine to enable result logging"
@@ -2135,14 +2224,17 @@ def docker(
     resolved_image = wandb.docker.image_id(image)
     if resolved_image is None:
         raise ClickException(
-            "Couldn't find image locally or in a registry, try running `docker pull %s`"
-            % image
+            "Couldn't find image locally or in a registry, try running `docker pull {}`".format(
+                image
+            )
         )
     if digest:
         sys.stdout.write(resolved_image)
         exit(0)
 
-    existing = wandb.docker.shell(["ps", "-f", "ancestor=%s" % resolved_image, "-q"])
+    existing = wandb.docker.shell(
+        ["ps", "-f", "ancestor={}".format(resolved_image), "-q"]
+    )
     if existing:
         if click.confirm(
             "Found running container with the same image, do you want to attach?"
@@ -2156,7 +2248,7 @@ def docker(
         "-e",
         "LANG=C.UTF-8",
         "-e",
-        "WANDB_DOCKER=%s" % resolved_image,
+        "WANDB_DOCKER={}".format(resolved_image),
         "--ipc=host",
         "-v",
         wandb.docker.entrypoint + ":/wandb-entrypoint.sh",
@@ -2169,7 +2261,7 @@ def docker(
         #  TODO: We should default to the working directory if defined
         command.extend(["-v", cwd + ":" + dir, "-w", dir])
     if api.api_key:
-        command.extend(["-e", "WANDB_API_KEY=%s" % api.api_key])
+        command.extend(["-e", "WANDB_API_KEY={}".format(api.api_key)])
     else:
         wandb.termlog(
             "Couldn't find WANDB_API_KEY, run `wandb login` to enable streaming metrics"
@@ -2177,16 +2269,15 @@ def docker(
     if jupyter:
         command.extend(["-e", "WANDB_ENSURE_JUPYTER=1", "-p", port + ":8888"])
         no_tty = True
-        cmd = (
-            "jupyter lab --no-browser --ip=0.0.0.0 --allow-root --NotebookApp.token= --notebook-dir %s"
-            % dir
+        cmd = "jupyter lab --no-browser --ip=0.0.0.0 --allow-root --NotebookApp.token= --notebook-dir {}".format(
+            dir
         )
     command.extend(args)
     if no_tty:
         command.extend([image, shell, "-c", cmd])
     else:
         if cmd:
-            command.extend(["-e", "WANDB_COMMAND=%s" % cmd])
+            command.extend(["-e", "WANDB_COMMAND={}".format(cmd)])
         command.extend(["-it", image, shell])
         wandb.termlog("Launching docker container \U0001f6a2")
     subprocess.call(command)
@@ -2272,7 +2363,7 @@ def start(ctx, port, env, daemon, upgrade, edge):
             exit(1)
     image = "docker.pkg.github.com/wandb/core/local" if edge else "wandb/local"
     username = getpass.getuser()
-    env_vars = ["-e", "LOCAL_USERNAME=%s" % username]
+    env_vars = ["-e", "LOCAL_USERNAME={}".format(username)]
     for e in env:
         env_vars.append("-e")
         env_vars.append(e)
@@ -2306,7 +2397,9 @@ def start(ctx, port, env, daemon, upgrade, edge):
             )
             exit(1)
         else:
-            wandb.termlog("W&B server started at http://localhost:%s \U0001f680" % port)
+            wandb.termlog(
+                "W&B server started at http://localhost:{} \U0001f680".format(port)
+            )
             wandb.termlog("You can stop the server by running `wandb server stop`")
             if not api.api_key:
                 # Let the server start before potentially launching a browser
@@ -2443,7 +2536,7 @@ def get(path, root, type):
         )
         artifact = public_api.artifact(full_path, type=type)
         path = artifact.download(root=root)
-        wandb.termlog("Artifact downloaded to %s" % path)
+        wandb.termlog("Artifact downloaded to {}".format(path))
     except ValueError:
         raise ClickException("Unable to download artifact")
 
@@ -2521,17 +2614,17 @@ def pull(run, project, entity):
 
     for name in urls:
         if api.file_current(name, urls[name]["md5"]):
-            click.echo("File %s is up to date" % name)
+            click.echo("File {} is up to date".format(name))
         else:
             length, response = api.download_file(urls[name]["url"])
             # TODO: I had to add this because some versions in CI broke click.progressbar
-            sys.stdout.write("File %s\r" % name)
+            sys.stdout.write("File {}\r".format(name))
             dirname = os.path.dirname(name)
             if dirname != "":
                 filesystem.mkdir_exists_ok(dirname)
             with click.progressbar(
                 length=length,
-                label="File %s" % name,
+                label="File {}".format(name),
                 fill_char=click.style("&", fg="green"),
             ) as bar:
                 with open(name, "wb") as f:
@@ -2577,11 +2670,8 @@ def restore(ctx, run, no_git, branch, project, entity):
     )
     repo = metadata.get("git", {}).get("repo")
     image = metadata.get("docker")
-    restore_message = (
-        """`wandb restore` needs to be run from the same git repository as the original run.
-Run `git clone %s` and restore from there or pass the --no-git flag."""
-        % repo
-    )
+    restore_message = """`wandb restore` needs to be run from the same git repository as the original run.
+Run `git clone {}` and restore from there or pass the --no-git flag.""".format(repo)
     if no_git:
         commit = None
     elif not api.git.enabled:
@@ -2626,18 +2716,21 @@ Run `git clone %s` and restore from there or pass the --no-git flag."""
             else:
                 patch_path = None
 
-        branch_name = "wandb/%s" % run
+        branch_name = "wandb/{}".format(run)
         if branch and branch_name not in api.git.repo.branches:
             api.git.repo.git.checkout(commit, b=branch_name)
-            wandb.termlog("Created branch %s" % click.style(branch_name, bold=True))
+            wandb.termlog(
+                "Created branch {}".format(click.style(branch_name, bold=True))
+            )
         elif branch:
             wandb.termlog(
-                "Using existing branch, run `git branch -D %s` from master for a clean checkout"
-                % branch_name
+                "Using existing branch, run `git branch -D {}` from master for a clean checkout".format(
+                    branch_name
+                )
             )
             api.git.repo.git.checkout(branch_name)
         else:
-            wandb.termlog("Checking out %s in detached mode" % commit)
+            wandb.termlog("Checking out {} in detached mode".format(commit))
             api.git.repo.git.checkout(commit)
 
         if patch_path:
@@ -2675,7 +2768,7 @@ Run `git clone %s` and restore from there or pass the --no-git flag."""
     with open(config_path, "w") as f:
         f.write(s)
 
-    wandb.termlog("Restored config variables to %s" % config_path)
+    wandb.termlog("Restored config variables to {}".format(config_path))
     if image:
         if not metadata["program"].startswith("<") and metadata.get("args") is not None:
             # TODO: we may not want to default to python here.
@@ -2710,7 +2803,9 @@ def magic(ctx, program, args):
         with open(program, "rb") as fp:
             code = compile(fp.read(), program, "exec")
     except OSError:
-        click.echo(click.style("Could not launch program: %s" % program, fg="red"))
+        click.echo(
+            click.style("Could not launch program: {}".format(program), fg="red")
+        )
         sys.exit(1)
     globs = {
         "__file__": program,
@@ -2718,14 +2813,11 @@ def magic(ctx, program, args):
         "__package__": None,
         "wandb_magic_install": magic_install,
     }
-    prep = (
-        """
+    prep = """
 import __main__
-__main__.__file__ = "%s"
+__main__.__file__ = "{}"
 wandb_magic_install()
-"""
-        % program
-    )
+""".format(program)
     magic_run(prep, globs, None)
     magic_run(code, globs, None)
 

wandb/docker/__init__.py

@@ -202,7 +202,7 @@ def default_image(gpu: bool = False) -> str:
     tag = "all"
     if not gpu:
         tag += "-cpu"
-    return "wandb/deepo:%s" % tag
+    return "wandb/deepo:{}".format(tag)
 
 
 def parse_repository_tag(repo_name: str) -> Tuple[str, Optional[str]]:

wandb/errors/term.py

@@ -27,7 +27,10 @@ def termsetup(settings, logger) -> None:
 
 
 def termlog(
-    string: str = "", newline: bool = True, repeat: bool = True, prefix: bool = True
+    string: str = "",
+    newline: bool = True,
+    repeat: bool = True,
+    prefix: bool = True,
 ) -> None:
     """Log to standard error with formatting.
 
@@ -68,7 +71,12 @@ def termerror(string: str, **kwargs: Any) -> None:
 
 
 def _log(
-    string="", newline=True, repeat=True, prefix=True, silent=False, level=logging.INFO
+    string="",
+    newline=True,
+    repeat=True,
+    prefix=True,
+    silent=False,
+    level=logging.INFO,
 ):
     global _logger
     silent = silent or _silent

wandb/filesync/step_checksum.py

@@ -17,7 +17,7 @@ if TYPE_CHECKING:
 
     from wandb.filesync import stats
     from wandb.sdk.artifacts.artifact_manifest import ArtifactManifest
-    from wandb.sdk.artifacts.artifact_saver import SaveFn, SaveFnAsync
+    from wandb.sdk.artifacts.artifact_saver import SaveFn
     from wandb.sdk.internal import internal_api
 
 
@@ -31,7 +31,6 @@ class RequestStoreManifestFiles(NamedTuple):
     manifest: "ArtifactManifest"
     artifact_id: str
     save_fn: "SaveFn"
-    save_fn_async: "SaveFnAsync"
 
 
 class RequestCommitArtifact(NamedTuple):
@@ -96,7 +95,6 @@ class StepChecksum:
                         req.copy,
                         None,
                         None,
-                        None,
                     )
                 )
             elif isinstance(req, RequestStoreManifestFiles):
@@ -115,7 +113,6 @@ class StepChecksum:
                                 entry.digest,
                                 False,
                                 functools.partial(req.save_fn, entry),
-                                functools.partial(req.save_fn_async, entry),
                                 entry.digest,
                             )
                         )

wandb/filesync/step_prepare.py

@@ -1,7 +1,5 @@
 """Batching file prepare requests to our API."""
 
-import asyncio
-import functools
 import queue
 import threading
 import time
@@ -29,10 +27,7 @@ if TYPE_CHECKING:
 # Request for a file to be prepared.
 class RequestPrepare(NamedTuple):
     file_spec: "CreateArtifactFileSpecInput"
-    response_channel: Union[
-        "queue.Queue[ResponsePrepare]",
-        Tuple["asyncio.AbstractEventLoop", "asyncio.Future[ResponsePrepare]"],
-    ]
+    response_channel: "queue.Queue[ResponsePrepare]"
 
 
 class RequestFinish(NamedTuple):
@@ -110,7 +105,7 @@ def prepare_response(response: "CreateArtifactFilesResponseFile") -> ResponsePre
 class StepPrepare:
     """A thread that batches requests to our file prepare API.
 
-    Any number of threads may call prepare_async() in parallel. The PrepareBatcher thread
+    Any number of threads may call prepare() in parallel. The PrepareBatcher thread
     will batch requests up and send them all to the backend at once.
     """
 
@@ -145,11 +140,7 @@ class StepPrepare:
                     name = prepare_request.file_spec["name"]
                     response_file = batch_response[name]
                     response = prepare_response(response_file)
-                    if isinstance(prepare_request.response_channel, queue.Queue):
-                        prepare_request.response_channel.put(response)
-                    else:
-                        loop, future = prepare_request.response_channel
-                        loop.call_soon_threadsafe(future.set_result, response)
+                    prepare_request.response_channel.put(response)
             if finish:
                 break
 
@@ -167,18 +158,7 @@ class StepPrepare:
         """
         return self._api.create_artifact_files([req.file_spec for req in batch])
 
-    def prepare_async(
-        self, file_spec: "CreateArtifactFileSpecInput"
-    ) -> "asyncio.Future[ResponsePrepare]":
-        """Request the backend to prepare a file for upload."""
-        response: asyncio.Future[ResponsePrepare] = asyncio.Future()
-        self._request_queue.put(
-            RequestPrepare(file_spec, (asyncio.get_event_loop(), response))
-        )
-        return response
-
-    @functools.wraps(prepare_async)
-    def prepare_sync(
+    def prepare(
         self, file_spec: "CreateArtifactFileSpecInput"
     ) -> "queue.Queue[ResponsePrepare]":
         response_queue: queue.Queue[ResponsePrepare] = queue.Queue()