wandb 0.17.0rc1__py3-none-win_amd64.whl → 0.17.1__py3-none-win_amd64.whl

wandb/sdk/launch/utils.py

@@ -1,4 +1,3 @@
-# heavily inspired by https://github.com/mlflow/mlflow/blob/master/mlflow/projects/utils.py
 import asyncio
 import json
 import logging
@@ -16,7 +15,6 @@ import wandb
 import wandb.docker as docker
 from wandb import util
 from wandb.apis.internal import Api
-from wandb.errors import CommError
 from wandb.sdk.launch.errors import LaunchError
 from wandb.sdk.launch.git_reference import GitReference
 from wandb.sdk.launch.wandb_reference import WandbReference
@@ -32,12 +30,13 @@ FAILED_PACKAGES_REGEX = re.compile(
 )
 
 if TYPE_CHECKING:  # pragma: no cover
-    from wandb.sdk.artifacts.artifact import Artifact
     from wandb.sdk.launch.agent.job_status_tracker import JobAndRunStatusTracker
 
 
 # TODO: this should be restricted to just Git repos and not S3 and stuff like that
-_GIT_URI_REGEX = re.compile(r"^[^/|^~|^\.].*(git|bitbucket)")
+_GIT_URI_REGEX = re.compile(
+    r"^[^/|^~|^\.].*(git|bitbucket|dev\.azure\.com|\.visualstudio\.com)"
+)
 _VALID_IP_REGEX = r"^https?://[0-9]+(?:\.[0-9]+){3}(:[0-9]+)?"
 _VALID_PIP_PACKAGE_REGEX = r"^[a-zA-Z0-9_.-]+$"
 _VALID_WANDB_REGEX = r"^https?://(api.)?wandb"
@@ -75,6 +74,7 @@ AZURE_BLOB_REGEX = re.compile(
     r"^https://([^\.]+)\.blob\.core\.windows\.net/([^/]+)/?(.*)$"
 )
 
+ARN_PARTITION_RE = re.compile(r"^arn:([^:]+):[^:]*:[^:]*:[^:]*:[^:]*$")
 
 PROJECT_SYNCHRONOUS = "SYNCHRONOUS"
 
@@ -316,16 +316,13 @@ def construct_launch_spec(
 
 
 def validate_launch_spec_source(launch_spec: Dict[str, Any]) -> None:
-    uri = launch_spec.get("uri")
     job = launch_spec.get("job")
     docker_image = launch_spec.get("docker", {}).get("docker_image")
-
-    if not bool(uri) and not bool(job) and not bool(docker_image):
-        raise LaunchError("Must specify a uri, job or docker image")
-    elif bool(uri) and bool(docker_image):
-        raise LaunchError("Found both uri and docker-image, only one can be set")
-    elif sum(map(bool, [uri, job, docker_image])) > 1:
-        raise LaunchError("Must specify exactly one of uri, job or image")
+    if bool(job) == bool(docker_image):
+        raise LaunchError(
+            "Exactly one of job or docker_image must be specified in the launch "
+            "spec."
+        )
 
 
 def parse_wandb_uri(uri: str) -> Tuple[str, str, str]:
@@ -336,77 +333,6 @@ def parse_wandb_uri(uri: str) -> Tuple[str, str, str]:
     return (ref.entity, ref.project, ref.run_id)
 
 
-def is_bare_wandb_uri(uri: str) -> bool:
-    """Check that a wandb uri is valid.
-
-    URI must be in the format
-    `/<entity>/<project>/runs/<run_name>[other stuff]`
-    or
-    `/<entity>/<project>/artifacts/job/<job_name>[other stuff]`.
-    """
-    _logger.info(f"Checking if uri {uri} is bare...")
-    return uri.startswith("/") and WandbReference.is_uri_job_or_run(uri)
-
-
-def fetch_wandb_project_run_info(
-    entity: str, project: str, run_name: str, api: Api
-) -> Any:
-    _logger.info("Fetching run info...")
-    try:
-        result = api.get_run_info(entity, project, run_name)
-    except CommError:
-        result = None
-    if result is None:
-        raise LaunchError(
-            f"Run info is invalid or doesn't exist for {api.settings('base_url')}/{entity}/{project}/runs/{run_name}"
-        )
-    if result.get("codePath") is None:
-        # TODO: we don't currently expose codePath in the runInfo endpoint, this downloads
-        # it from wandb-metadata.json if we can.
-        metadata = api.download_url(
-            project, "wandb-metadata.json", run=run_name, entity=entity
-        )
-        if metadata is not None:
-            _, response = api.download_file(metadata["url"])
-            data = response.json()
-            result["codePath"] = data.get("codePath")
-            result["cudaVersion"] = data.get("cuda", None)
-
-    return result
-
-
-def download_entry_point(
-    entity: str, project: str, run_name: str, api: Api, entry_point: str, dir: str
-) -> bool:
-    metadata = api.download_url(
-        project, f"code/{entry_point}", run=run_name, entity=entity
-    )
-    if metadata is not None:
-        _, response = api.download_file(metadata["url"])
-        with util.fsync_open(os.path.join(dir, entry_point), "wb") as file:
-            for data in response.iter_content(chunk_size=1024):
-                file.write(data)
-        return True
-    return False
-
-
-def download_wandb_python_deps(
-    entity: str, project: str, run_name: str, api: Api, dir: str
-) -> Optional[str]:
-    reqs = api.download_url(project, "requirements.txt", run=run_name, entity=entity)
-    if reqs is not None:
-        _logger.info("Downloading python dependencies")
-        _, response = api.download_file(reqs["url"])
-
-        with util.fsync_open(
-            os.path.join(dir, "requirements.frozen.txt"), "wb"
-        ) as file:
-            for data in response.iter_content(chunk_size=1024):
-                file.write(data)
-        return "requirements.frozen.txt"
-    return None
-
-
 def get_local_python_deps(
     dir: str, filename: str = "requirements.local.txt"
 ) -> Optional[str]:
@@ -498,19 +424,6 @@ def validate_wandb_python_deps(
     _logger.warning("Unable to validate local python dependencies")
 
 
-def fetch_project_diff(
-    entity: str, project: str, run_name: str, api: Api
-) -> Optional[str]:
-    """Fetches project diff from wandb servers."""
-    _logger.info("Searching for diff.patch")
-    patch = None
-    try:
-        (_, _, patch, _) = api.run_config(project, run_name, entity)
-    except CommError:
-        pass
-    return patch
-
-
 def apply_patch(patch_string: str, dst_dir: str) -> None:
     """Applies a patch file to a directory."""
     _logger.info("Applying diff.patch")
@@ -531,17 +444,6 @@ def apply_patch(patch_string: str, dst_dir: str) -> None:
         raise wandb.Error("Failed to apply diff.patch associated with run.")
 
 
-def _make_refspec_from_version(version: Optional[str]) -> List[str]:
-    """Create a refspec that checks for the existence of origin/main and the version."""
-    if version:
-        return [f"+{version}"]
-
-    return [
-        "+refs/heads/main*:refs/remotes/origin/main*",
-        "+refs/heads/master*:refs/remotes/origin/master*",
-    ]
-
-
 def _fetch_git_repo(dst_dir: str, uri: str, version: Optional[str]) -> Optional[str]:
     """Clones the git repo at ``uri`` into ``dst_dir``.
 
@@ -561,13 +463,6 @@ def _fetch_git_repo(dst_dir: str, uri: str, version: Optional[str]) -> Optional[
     return version
 
 
-def merge_parameters(
-    higher_priority_params: Dict[str, Any], lower_priority_params: Dict[str, Any]
-) -> Dict[str, Any]:
-    """Merge the contents of two dicts, keeping values from higher_priority_params if there are conflicts."""
-    return {**lower_priority_params, **higher_priority_params}
-
-
 def convert_jupyter_notebook_to_script(fname: str, project_dir: str) -> str:
     nbconvert = wandb.util.get_module(
         "nbconvert", "nbformat and nbconvert are required to use launch with notebooks"
@@ -597,25 +492,6 @@ def convert_jupyter_notebook_to_script(fname: str, project_dir: str) -> str:
     return new_name
 
 
-def check_and_download_code_artifacts(
-    entity: str, project: str, run_name: str, internal_api: Api, project_dir: str
-) -> Optional["Artifact"]:
-    _logger.info("Checking for code artifacts")
-    public_api = wandb.PublicApi(
-        overrides={"base_url": internal_api.settings("base_url")}
-    )
-
-    run = public_api.run(f"{entity}/{project}/{run_name}")
-    run_artifacts = run.logged_artifacts()
-
-    for artifact in run_artifacts:
-        if hasattr(artifact, "type") and artifact.type == "code":
-            artifact.download(project_dir)
-            return artifact  # type: ignore
-
-    return None
-
-
 def to_camel_case(maybe_snake_str: str) -> str:
     if "_" not in maybe_snake_str:
         return maybe_snake_str
@@ -623,11 +499,6 @@ def to_camel_case(maybe_snake_str: str) -> str:
     return "".join(x.title() if x else "_" for x in components)
 
 
-def run_shell(args: List[str]) -> Tuple[str, str]:
-    out = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    return out.stdout.decode("utf-8").strip(), out.stderr.decode("utf-8").strip()
-
-
 def validate_build_and_registry_configs(
     build_config: Dict[str, Any], registry_config: Dict[str, Any]
 ) -> None:
@@ -864,3 +735,10 @@ def get_entrypoint_file(entrypoint: List[str]) -> Optional[str]:
     if len(entrypoint) < 2:
         return None
     return entrypoint[1]
+
+
+def get_current_python_version() -> Tuple[str, str]:
+    full_version = sys.version.split()[0].split(".")
+    major = full_version[0]
+    version = ".".join(full_version[:2]) if len(full_version) >= 2 else major + ".0"
+    return version, major

wandb/sdk/lib/_settings_toposort_generated.py

@@ -13,7 +13,6 @@ else:
 _Setting = Literal[
     "_args",
     "_aws_lambda",
-    "_async_upload_concurrency_limit",
     "_cli_only_mode",
     "_code_path_local",
     "_colab",
@@ -25,7 +24,6 @@ _Setting = Literal[
     "_disable_update_check",
     "_disable_viewer",
     "_disable_machine_info",
-    "_except_exit",
     "_executable",
     "_extra_http_headers",
     "_file_stream_retry_max",
@@ -92,6 +90,7 @@ _Setting = Literal[
     "colab_url",
     "config_paths",
     "console",
+    "console_multipart",
     "deployment",
     "disable_code",
     "disable_git",
@@ -104,6 +103,7 @@ _Setting = Literal[
     "files_dir",
     "force",
     "fork_from",
+    "resume_from",
     "git_commit",
     "git_remote",
     "git_remote_url",
@@ -126,7 +126,6 @@ _Setting = Literal[
     "login_timeout",
     "mode",
     "notebook_name",
-    "problem",
     "program",
     "program_abspath",
     "program_relpath",
@@ -179,7 +178,6 @@ _Setting = Literal[
 ]
 
 SETTINGS_TOPOLOGICALLY_SORTED: Final[Tuple[_Setting, ...]] = (
-    "_async_upload_concurrency_limit",
     "_service_wait",
     "_stats_sample_rate_seconds",
     "_stats_samples_to_average",
@@ -189,7 +187,6 @@ SETTINGS_TOPOLOGICALLY_SORTED: Final[Tuple[_Setting, ...]] = (
     "console",
     "job_source",
     "mode",
-    "problem",
     "project",
     "run_id",
     "start_method",

wandb/sdk/lib/apikey.py

@@ -202,7 +202,7 @@ def write_netrc(host: str, entity: str, key: str) -> Optional[bool]:
                     elif skip:
                         skip -= 1
                     else:
-                        f.write("%s\n" % line)
+                        f.write("{}\n".format(line))
             f.write(
                 textwrap.dedent(
                     """\
@@ -236,7 +236,9 @@ def write_key(
     _, suffix = key.split("-", 1) if "-" in key else ("", key)
 
     if len(suffix) != 40:
-        raise ValueError("API key must be 40 characters long, yours was %s" % len(key))
+        raise ValueError(
+            "API key must be 40 characters long, yours was {}".format(len(key))
+        )
 
     if anonymous:
         api.set_setting("anonymous", "true", globally=True, persist=True)

wandb/sdk/lib/config_util.py

@@ -66,13 +66,13 @@ def dict_from_config_file(
 ) -> Optional[Dict[str, Any]]:
     if not os.path.exists(filename):
         if must_exist:
-            raise ConfigError("config file %s doesn't exist" % filename)
-        logger.debug("no default config file found in %s" % filename)
+            raise ConfigError("config file {} doesn't exist".format(filename))
+        logger.debug("no default config file found in {}".format(filename))
         return None
     try:
         conf_file = open(filename)
     except OSError:
-        raise ConfigError("Couldn't read config file: %s" % filename)
+        raise ConfigError("Couldn't read config file: {}".format(filename))
     try:
         loaded = load_yaml(conf_file)
     except yaml.parser.ParserError:

wandb/sdk/lib/import_hooks.py

@@ -143,7 +143,7 @@ class _ImportHookChainedLoader:
         # None, so handle None as well. The module may not support attribute
         # assignment, in which case we simply skip it. Note that we also deal
         # with __loader__ not existing at all. This is to future proof things
-        # due to proposal to remove the attribue as described in the GitHub
+        # due to proposal to remove the attribute as described in the GitHub
         # issue at https://github.com/python/cpython/issues/77458. Also prior
         # to Python 3.3, the __loader__ attribute was only set if a custom
         # module loader was used. It isn't clear whether the attribute still

wandb/sdk/lib/proto_util.py

@@ -12,7 +12,28 @@ if TYPE_CHECKING:  # pragma: no cover
 
 
 def dict_from_proto_list(obj_list: "RepeatedCompositeFieldContainer") -> Dict[str, Any]:
-    return {item.key: json.loads(item.value_json) for item in obj_list}
+    result: Dict[str, Any] = {}
+
+    for item in obj_list:
+        # Start from the root of the result dict
+        current_level = result
+
+        if len(item.nested_key) > 0:
+            keys = list(item.nested_key)
+        else:
+            keys = [item.key]
+
+        for key in keys[:-1]:
+            if key not in current_level:
+                current_level[key] = {}
+            # Move the reference deeper into the nested dictionary
+            current_level = current_level[key]
+
+        # Set the value at the final key location, parsing JSON from the value_json field
+        final_key = keys[-1]
+        current_level[final_key] = json.loads(item.value_json)
+
+    return result
 
 
 def _result_from_record(record: "pb.Record") -> "pb.Result":

wandb/sdk/lib/redirect.py

@@ -224,7 +224,7 @@ class TerminalEmulator:
     def carriage_return(self):
         self.cursor.x = 0
 
-    def cursor_postion(self, line, column):
+    def cursor_position(self, line, column):
         self.cursor.x = min(column, 1) - 1
         self.cursor.y = min(line, 1) - 1
 
@@ -393,25 +393,30 @@ class TerminalEmulator:
                         p = (int(p[0]), 1)
                     else:
                         p = (1, 1)
-                    self.cursor_postion(*p)
+                    self.cursor_position(*p)
         except Exception:
             pass
 
     def _get_line(self, n):
         line = self.buffer[n]
         line_len = self._get_line_len(n)
-        # We have to loop through each character in the line and check if foreground, background and
-        # other attributes (italics, bold, underline, etc) of the ith character are different from those of the
-        # (i-1)th character. If different, the appropriate ascii character for switching the color/attribute
-        # should be appended to the output string before appending the actual character. This loop and subsequent
-        # checks can be expensive, especially because 99% of terminal output use default colors and formatting. Even
-        # in outputs that do contain colors and styles, its unlikely that they will change on a per character basis.
-
-        # So instead we create a character list without any ascii codes (`out`), and a list of all the foregrounds
-        # in the line (`fgs`) on which we call np.diff() and np.where() to find the indices where the foreground change,
-        # and insert the ascii characters in the output list (`out`) on those indices. All of this is the done ony if
-        # there are more than 1 foreground color in the line in the first place (`if len(set(fgs)) > 1 else None`).
-        # Same logic is repeated for background colors and other attributes.
+        # We have to loop through each character in the line and check if foreground,
+        # background and other attributes (italics, bold, underline, etc) of the ith
+        # character are different from those of the (i-1)th character. If different, the
+        # appropriate ascii character for switching the color/attribute should be
+        # appended to the output string before appending the actual character. This loop
+        # and subsequent checks can be expensive, especially because 99% of terminal
+        # output use default colors and formatting. Even in outputs that do contain
+        # colors and styles, its unlikely that they will change on a per character
+        # basis.
+
+        # So instead we create a character list without any ascii codes (`out`), and a
+        # list of all the foregrounds in the line (`fgs`) on which we call np.diff() and
+        # np.where() to find the indices where the foreground change, and insert the
+        # ascii characters in the output list (`out`) on those indices. All of this is
+        # the done only if there are more than 1 foreground color in the line in the
+        # first place (`if len(set(fgs)) > 1 else None`). Same logic is repeated for
+        # background colors and other attributes.
 
         out = [line[i].data for i in range(line_len)]
 
@@ -499,7 +504,7 @@ class RedirectBase:
 
     @property
     def src_stream(self):
-        return getattr(sys, "__%s__" % self.src)
+        return getattr(sys, "__{}__".format(self.src))
 
     @property
     def src_fd(self):

wandb/sdk/lib/tracelog.py

@@ -6,7 +6,7 @@ Functions:
     log_message_send    - message sent to socket
     log_message_recv    - message received from socket
     log_message_process - message processed by thread
-    log_message_link    - message linked to another mesage
+    log_message_link    - message linked to another message
     log_message_assert  - message encountered problem
 
 """

wandb/sdk/service/service.py

@@ -110,7 +110,8 @@ class _Service:
                     f"The wandb service process exited with {proc.returncode}. "
                     "Ensure that `sys.executable` is a valid python interpreter. "
                     "You can override it with the `_executable` setting "
-                    "or with the `WANDB__EXECUTABLE` environment variable.",
+                    "or with the `WANDB__EXECUTABLE` environment variable."
+                    f"\n{context}",
                     context=context,
                 )
             if not os.path.isfile(fname):

wandb/sdk/service/streams.py

@@ -319,8 +319,8 @@ class StreamMux:
         # These could be done in parallel in the future
         for _sid, stream in started_streams.items():
             # dispatch all our final requests
-            poll_exit_handle = stream.interface.deliver_poll_exit()
             server_info_handle = stream.interface.deliver_request_server_info()
+            poll_exit_handle = stream.interface.deliver_poll_exit()
             final_summary_handle = stream.interface.deliver_get_summary()
             sampled_history_handle = stream.interface.deliver_request_sampled_history()
             internal_messages_handle = stream.interface.deliver_internal_messages()
@@ -330,14 +330,14 @@ class StreamMux:
             internal_messages_response = result.response.internal_messages_response
 
             # wait for them, it's ok to do this serially but this can be improved
-            result = poll_exit_handle.wait(timeout=-1)
-            assert result
-            poll_exit_response = result.response.poll_exit_response
-
             result = server_info_handle.wait(timeout=-1)
             assert result
             server_info_response = result.response.server_info_response
 
+            result = poll_exit_handle.wait(timeout=-1)
+            assert result
+            poll_exit_response = result.response.poll_exit_response
+
             result = sampled_history_handle.wait(timeout=-1)
             assert result
             sampled_history = result.response.sampled_history_response

wandb/sdk/wandb_init.py

@@ -15,7 +15,6 @@ import os
 import platform
 import sys
 import tempfile
-import traceback
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Union
 
 import wandb
@@ -555,7 +554,7 @@ class _WandbInit:
         percent_done = handle.percent_done
         self.printer.progress_update(line, percent_done=percent_done)
 
-    def init(self) -> Union[Run, RunDisabled, None]:  # noqa: C901
+    def init(self) -> Union[Run, RunDisabled]:  # noqa: C901
         if logger is None:
             raise RuntimeError("Logger not initialized")
         logger.info("calling init triggers")
@@ -654,9 +653,6 @@ class _WandbInit:
             if self.settings.launch:
                 tel.feature.launch = True
 
-            if self.settings._async_upload_concurrency_limit:
-                tel.feature.async_uploads = True
-
             for module_name in telemetry.list_telemetry_imports(only_imported=True):
                 setattr(tel.imports_init, module_name, True)
 
@@ -843,13 +839,6 @@ class _WandbInit:
         return run
 
 
-def getcaller() -> None:
-    if not logger:
-        return None
-    src, line, func, stack = logger.findCaller(stack_info=True)
-    print("Problem at:", src, line, func)
-
-
 def _attach(
     attach_id: Optional[str] = None,
     run_id: Optional[str] = None,
@@ -957,8 +946,9 @@ def init(
     save_code: Optional[bool] = None,
     id: Optional[str] = None,
     fork_from: Optional[str] = None,
+    resume_from: Optional[str] = None,
     settings: Union[Settings, Dict[str, Any], None] = None,
-) -> Union[Run, RunDisabled, None]:
+) -> Union[Run, RunDisabled]:
     r"""Start a new run to track and log to W&B.
 
     In an ML training pipeline, you could add `wandb.init()`
@@ -1006,7 +996,7 @@ def init(
             there, so make sure to create your account or team in the UI before
             starting to log runs.
             If you don't specify an entity, the run will be sent to your default
-            entity, which is usually your username. Change your default entity
+            entity. Change your default entity
             in [your settings](https://wandb.ai/settings) under "default location
             to create new projects".
         config: (dict, argparse, absl.flags, str, optional)
@@ -1164,57 +1154,33 @@ def init(
     wandb._assert_is_user_process()
 
     kwargs = dict(locals())
-    error_seen = None
-    except_exit = None
-    run: Optional[Union[Run, RunDisabled]] = None
 
-    # convert fork_from into a version that can be passed to settings
-    if fork_from is not None and resume is not None:
-        raise ValueError("Cannot specify both `fork_from` and `resume`")
+    num_resume_options_set = (
+        (fork_from is not None)  # wrap
+        + (resume is not None)
+        + (resume_from is not None)
+    )
+    if num_resume_options_set > 1:
+        raise ValueError(
+            "You cannot specify more than one of `fork_from`, `resume`, or `resume_from`"
+        )
 
     try:
         wi = _WandbInit()
         wi.setup(kwargs)
-        assert wi.settings
-        except_exit = wi.settings._except_exit
-        try:
-            run = wi.init()
-            except_exit = wi.settings._except_exit
-        except (KeyboardInterrupt, Exception) as e:
-            if not isinstance(e, KeyboardInterrupt):
-                wandb._sentry.exception(e)
-            if not (
-                wandb.wandb_agent._is_running() and isinstance(e, KeyboardInterrupt)
-            ):
-                getcaller()
-            assert logger
-            if wi.settings.problem == "fatal":
-                raise
-            if wi.settings.problem == "warn":
-                pass
-            # TODO(jhr): figure out how to make this RunDummy
-            run = None
-    except Error as e:
-        if logger is not None:
-            logger.exception(str(e))
-        raise e
+        return wi.init()
+
     except KeyboardInterrupt as e:
-        assert logger
-        logger.warning("interrupted", exc_info=e)
-        raise e
+        if logger is not None:
+            logger.warning("interrupted", exc_info=e)
+
+        raise
+
     except Exception as e:
-        error_seen = e
-        traceback.print_exc()
-        assert logger
-        logger.error("error", exc_info=e)
+        if logger is not None:
+            logger.exception("error in wandb.init()", exc_info=e)
+
         # Need to build delay into this sentry capture because our exit hooks
         # mess with sentry's ability to send out errors before the program ends.
-        wandb._sentry.exception(e)
-        # reraise(*sys.exc_info())
-    finally:
-        if error_seen:
-            if except_exit:
-                wandb.termerror("Abnormal program exit")
-                os._exit(1)
-            raise Error("An unexpected error occurred") from error_seen
-    return run
+        wandb._sentry.reraise(e)
+        raise AssertionError()  # unreachable