wandb 0.15.9__py3-none-any.whl → 0.15.11__py3-none-any.whl

wandb/__init__.py

@@ -11,7 +11,8 @@ For scripts and interactive notebooks, see https://github.com/wandb/examples.
 
 For reference documentation, see https://docs.wandb.com/ref/python.
 """
-__version__ = "0.15.9"
+__version__ = "0.15.11"
+_minimum_nexus_version = "0.16.0b1"
 
 # Used with pypi checks and other messages related to pip
 _wandb_module = "wandb"
@@ -84,6 +85,8 @@ from wandb import plots  # deprecating this
 from wandb.integration.sagemaker import sagemaker_auth
 from wandb.sdk.internal import profiler
 
+# Artifact import types
+from wandb.sdk.artifacts.artifact_ttl import ArtifactTTL
 
 # Used to make sure we don't use some code in the incorrect process context
 _IS_INTERNAL_PROCESS = False
@@ -223,4 +226,5 @@ __all__ = (
     "Object3D",
     "Molecule",
     "Histogram",
+    "ArtifactTTL",
 )

wandb/apis/public.py

@@ -55,7 +55,7 @@ from wandb.sdk.launch.utils import (
     apply_patch,
     convert_jupyter_notebook_to_script,
 )
-from wandb.sdk.lib import ipython, retry, runid
+from wandb.sdk.lib import ipython, json_util, retry, runid
 from wandb.sdk.lib.gql_request import GraphQLSession
 from wandb.sdk.lib.paths import LogicalPath
 
@@ -63,6 +63,8 @@ if TYPE_CHECKING:
     import wandb.apis.reports
     import wandb.apis.reports.util
 
+from wandb.sdk.artifacts.artifact_state import ArtifactState
+
 logger = logging.getLogger(__name__)
 
 # Only retry requests for 20 seconds in the public api
@@ -253,7 +255,7 @@ class Api:
             You can also set defaults for `entity`, `project`, and `run`.
     """
 
-    _HTTP_TIMEOUT = env.get_http_timeout(9)
+    _HTTP_TIMEOUT = env.get_http_timeout(19)
     VIEWER_QUERY = gql(
         """
         query Viewer{
@@ -393,6 +395,9 @@ class Api:
         auth = None
         if not _thread_local_api_settings.cookies:
             auth = ("api", self.api_key)
+        proxies = self.settings.get("_proxies") or json.loads(
+            os.environ.get("WANDB__PROXIES", "{}")
+        )
         self._base_client = Client(
             transport=GraphQLSession(
                 headers={
@@ -407,6 +412,7 @@ class Api:
                 auth=auth,
                 url="%s/graphql" % self.settings["base_url"],
                 cookies=_thread_local_api_settings.cookies,
+                proxies=proxies,
             )
         )
         self._client = RetryingClient(self._base_client)
@@ -507,7 +513,7 @@ class Api:
         # 2. create default resource config, receive config id
         config_json = json.dumps({"resource_args": {type: config}})
         create_config_result = api.create_default_resource_config(
-            entity, LAUNCH_DEFAULT_PROJECT, type, config_json
+            entity, type, config_json
         )
         if not create_config_result["success"]:
             raise wandb.Error("failed to create default resource config")
@@ -1835,6 +1841,7 @@ class Run(Attrs):
         read_only (boolean): Whether the run is editable
         history_keys (str): Keys of the history metrics that have been logged
             with `wandb.log({key: value})`
+        metadata (str): Metadata about the run from wandb-metadata.json
     """
 
     def __init__(
@@ -1867,6 +1874,7 @@ class Run(Attrs):
         except OSError:
             pass
         self._summary = None
+        self._metadata: Optional[Dict[str, Any]] = None
         self._state = _attrs.get("state", "not found")
 
         self.load(force=not _attrs)
@@ -1988,11 +1996,18 @@ class Run(Attrs):
                     withRuns=False,
                 )
 
-        self._attrs["summaryMetrics"] = (
-            json.loads(self._attrs["summaryMetrics"])
-            if self._attrs.get("summaryMetrics")
-            else {}
-        )
+        try:
+            self._attrs["summaryMetrics"] = (
+                json.loads(self._attrs["summaryMetrics"])
+                if self._attrs.get("summaryMetrics")
+                else {}
+            )
+        except json.decoder.JSONDecodeError:
+            # ignore invalid utf-8 or control characters
+            self._attrs["summaryMetrics"] = json.loads(
+                self._attrs["summaryMetrics"],
+                strict=False,
+            )
         self._attrs["systemMetrics"] = (
             json.loads(self._attrs["systemMetrics"])
             if self._attrs.get("systemMetrics")
@@ -2385,6 +2400,18 @@ class Run(Attrs):
         path.insert(2, "runs")
         return self.client.app_url + "/".join(path)
 
+    @property
+    def metadata(self):
+        if self._metadata is None:
+            try:
+                f = self.file("wandb-metadata.json")
+                contents = util.download_file_into_memory(f.url, Api().api_key)
+                self._metadata = json_util.loads(contents)
+            except:  # noqa: E722
+                # file doesn't exist, or can't be downloaded, or can't be parsed
+                pass
+        return self._metadata
+
     @property
     def lastHistoryStep(self):  # noqa: N802
         query = gql(
@@ -2603,8 +2630,6 @@ class QueuedRun:
     def wait_until_running(self):
         if self._run is not None:
             return self._run
-        if self.container_job:
-            raise LaunchError("Container jobs cannot be waited on")
 
         while True:
             # sleep here to hide an ugly warning
@@ -2656,6 +2681,7 @@ class RunQueue:
         self._default_resource_config = _default_resource_config
         self._type = None
         self._items = None
+        self._id = None
 
     @property
     def name(self):
@@ -2687,6 +2713,12 @@ class RunQueue:
             self._get_default_resource_config()
         return self._default_resource_config
 
+    @property
+    def id(self) -> str:
+        if self._id is None:
+            self._get_metadata()
+        return self._id
+
     @property
     def items(self) -> List[QueuedRun]:
         """Up to the first 100 queued runs. Modifying this list will not modify the queue or any enqueued items!"""
@@ -2695,6 +2727,30 @@ class RunQueue:
             self._get_items()
         return self._items
 
+    @normalize_exceptions
+    def delete(self):
+        """Delete the run queue from the wandb backend."""
+        query = gql(
+            """
+            mutation DeleteRunQueue($id: ID!) {
+                deleteRunQueues(input: {queueIDs: [$id]}) {
+                    success
+                    clientMutationId
+                }
+            }
+            """
+        )
+        variable_values = {"id": self.id}
+        res = self._client.execute(query, variable_values)
+        if res["deleteRunQueues"]["success"]:
+            self._id = None
+            self._access = None
+            self._default_resource_config_id = None
+            self._default_resource_config = None
+            self._items = None
+        else:
+            raise CommError(f"Failed to delete run queue {self.name}")
+
     def __repr__(self):
         return f"<RunQueue {self._entity}/{self._name}>"
 
@@ -2705,6 +2761,7 @@ class RunQueue:
             query GetRunQueueMetadata($projectName: String!, $entityName: String!, $runQueue: String!) {
                 project(name: $projectName, entityName: $entityName) {
                     runQueue(name: $runQueue) {
+                        id
                         access
                         defaultResourceConfigID
                     }
@@ -2718,6 +2775,7 @@ class RunQueue:
             "runQueue": self._name,
         }
         res = self._client.execute(query, variable_values)
+        self._id = res["project"]["runQueue"]["id"]
         self._access = res["project"]["runQueue"]["access"]
         self._default_resource_config_id = res["project"]["runQueue"][
             "defaultResourceConfigID"
@@ -4158,9 +4216,8 @@ class RunArtifacts(Paginator):
                     }
                 }
             }
-            %s
             """
-            % wandb.Artifact._GQL_FRAGMENT
+            + wandb.Artifact._get_gql_artifact_fragment()
         )
 
         input_query = gql(
@@ -4186,9 +4243,8 @@ class RunArtifacts(Paginator):
                     }
                 }
             }
-            %s
             """
-            % wandb.Artifact._GQL_FRAGMENT
+            + wandb.Artifact._get_gql_artifact_fragment()
         )
 
         self.run = run
@@ -4425,6 +4481,66 @@ class ArtifactCollection:
         self._attrs = response["project"]["artifactType"]["artifactCollection"]
         return self._attrs
 
+    @normalize_exceptions
+    def is_sequence(self) -> bool:
+        """Return True if this is a sequence."""
+        query = gql(
+            """
+            query FindSequence($entity: String!, $project: String!, $collection: String!, $type: String!) {
+                project(name: $project, entityName: $entity) {
+                    artifactType(name: $type) {
+                        __typename
+                        artifactSequence(name: $collection) {
+                            __typename
+                        }
+                    }
+                }
+            }
+            """
+        )
+        variables = {
+            "entity": self.entity,
+            "project": self.project,
+            "collection": self.name,
+            "type": self.type,
+        }
+        res = self.client.execute(query, variable_values=variables)
+        sequence = res["project"]["artifactType"]["artifactSequence"]
+        return sequence is not None and sequence["__typename"] == "ArtifactSequence"
+
+    @normalize_exceptions
+    def delete(self):
+        """Delete the entire artifact collection."""
+        if self.is_sequence():
+            mutation = gql(
+                """
+                mutation deleteArtifactSequence($id: ID!) {
+                    deleteArtifactSequence(input: {
+                        artifactSequenceID: $id
+                    }) {
+                        artifactCollection {
+                            state
+                        }
+                    }
+                }
+                """
+            )
+        else:
+            mutation = gql(
+                """
+                mutation deleteArtifactPortfolio($id: ID!) {
+                    deleteArtifactPortfolio(input: {
+                        artifactPortfolioID: $id
+                    }) {
+                        artifactCollection {
+                            state
+                        }
+                    }
+                }
+                """
+            )
+        self.client.execute(mutation, variable_values={"id": self.id})
+
     def __repr__(self):
         return f"<ArtifactCollection {self.name} ({self.type})>"
 
@@ -4490,7 +4606,7 @@ class ArtifactVersions(Paginator):
                 artifact_collection_edge_name(
                     server_supports_artifact_collections_gql_edges(client)
                 ),
-                wandb.Artifact._GQL_FRAGMENT,
+                wandb.Artifact._get_gql_artifact_fragment(),
             )
         )
         super().__init__(client, variables, per_page)
@@ -4687,6 +4803,10 @@ class Job:
             code_artifact = self._api.artifact(name=artifact_string, type="code")
         if code_artifact is None:
             raise LaunchError("No code artifact found")
+        if code_artifact.state == ArtifactState.DELETED:
+            raise LaunchError(
+                f"Job {self.name} references deleted code artifact {code_artifact.name}"
+            )
         return code_artifact
 
     def _configure_launch_project_notebook(self, launch_project):
@@ -4752,7 +4872,7 @@ class Job:
         resource_args=None,
         project_queue=None,
     ):
-        from wandb.sdk.launch import launch_add
+        from wandb.sdk.launch import _launch_add
 
         run_config = {}
         for key, item in config.items():
@@ -4772,7 +4892,7 @@ class Job:
             if isinstance(assigned_config_type, InvalidType):
                 raise TypeError(self._input_types.explain(run_config))
 
-        queued_run = launch_add.launch_add(
+        queued_run = _launch_add.launch_add(
             job=self._name,
             config={"overrides": {"run_config": run_config}},
             project=project or self._project,

wandb/apis/reports/_panels.py

@@ -650,7 +650,7 @@ class RunComparer(Panel):
 
 class MediaBrowser(Panel):
     num_columns: Optional[int] = Attr(json_path="spec.config.columnCount")
-    media_keys: Optional[str] = Attr(json_path="spec.config.media_keys")
+    media_keys: Optional[str] = Attr(json_path="spec.config.mediaKeys")
     # Attr(json_path="spec.config.chartTitle")
     # Attr(json_path="spec.config.stepIndex")
     # Attr(json_path="spec.config.mediaIndex")

wandb/apis/reports/blocks.py

@@ -23,4 +23,5 @@ from ._blocks import (
     WeaveBlockArtifact,
     WeaveBlockArtifactVersionedFile,
     WeaveBlockSummaryTable,
+    Twitter,
 )

wandb/apis/reports/report.py

@@ -1,3 +1,4 @@
+import base64
 import inspect
 import json
 import re
@@ -67,13 +68,34 @@ class Report(Base):
 
     @staticmethod
     def _url_to_report_id(url):
+        path_msg = "Path must be `entity/project/reports/report_title--report_id`"
         try:
-            report, *_ = url.split("?")
-            # If the report title ends in trailing space
-            report = report.replace("---", "--")
-            *_, report_id = report.split("--")
+            report_path, *_ = url.split("?")
+            report_path = report_path.replace("---", "--")
+
+            if "--" not in report_path:
+                raise ValueError(path_msg)
+
+            *_, report_id = report_path.split("--")
+            if len(report_id) == 0:
+                raise ValueError("Invalid report id")
+
+            report_id = report_id.strip()
+
+            """
+                Server does not generate IDs with correct padding, so decode with default validate=False.
+                Then re-encode it with correct padding.
+                https://stackoverflow.com/questions/2941995/python-ignore-incorrect-padding-error-when-base64-decoding
+
+                Corresponding core app logic that strips the padding in url
+                https://github.com/wandb/core/blob/b563437c1f3237ec35b1fb388ac14abbab7b4279/frontends/app/src/util/url/shared.ts#L33-L78
+            """
+            report_id = base64.b64encode(base64.b64decode(report_id + "==")).decode(
+                "utf-8"
+            )
+
         except ValueError as e:
-            raise ValueError("Path must be `entity/project/reports/report_id`") from e
+            raise ValueError(path_msg) from e
         else:
             return report_id
 

wandb/cli/cli.py

@@ -34,8 +34,8 @@ from wandb.apis import InternalApi, PublicApi
 from wandb.integration.magic import magic_install
 from wandb.sdk.artifacts.artifacts_cache import get_artifacts_cache
 from wandb.sdk.launch import utils as launch_utils
+from wandb.sdk.launch._launch_add import _launch_add
 from wandb.sdk.launch.errors import ExecutionError, LaunchError
-from wandb.sdk.launch.launch_add import _launch_add
 from wandb.sdk.launch.sweeps import utils as sweep_utils
 from wandb.sdk.launch.sweeps.scheduler import Scheduler
 from wandb.sdk.lib import filesystem
@@ -249,9 +249,6 @@ def login(key, host, cloud, relogin, anonymously, no_offline=False):
 @cli.command(
     context_settings=CONTEXT, help="Run a wandb service", name="service", hidden=True
 )
-@click.option(
-    "--grpc-port", default=None, type=int, help="The host port to bind grpc service."
-)
 @click.option(
     "--sock-port", default=None, type=int, help="The host port to bind socket service."
 )
@@ -260,29 +257,24 @@ def login(key, host, cloud, relogin, anonymously, no_offline=False):
 @click.option("--pid", default=None, type=int, help="The parent process id to monitor.")
 @click.option("--debug", is_flag=True, help="log debug info")
 @click.option("--serve-sock", is_flag=True, help="use socket mode")
-@click.option("--serve-grpc", is_flag=True, help="use grpc mode")
 @display_error
 def service(
-    grpc_port=None,
     sock_port=None,
     port_filename=None,
     address=None,
     pid=None,
     debug=False,
     serve_sock=False,
-    serve_grpc=False,
 ):
     from wandb.sdk.service.server import WandbServer
 
     server = WandbServer(
-        grpc_port=grpc_port,
         sock_port=sock_port,
         port_fname=port_filename,
         address=address,
         pid=pid,
         debug=debug,
         serve_sock=serve_sock,
-        serve_grpc=serve_grpc,
     )
     server.serve()
 
@@ -1250,6 +1242,12 @@ def launch_sweep(
     hidden=True,
     help="Name of the project containing the queue to push to. If none, defaults to entity level queues.",
 )
+@click.option(
+    "--dockerfile",
+    "-D",
+    default=None,
+    help="Path to the Dockerfile used to build the job, relative to the job's root",
+)
 @display_error
 def launch(
     uri,
@@ -1268,6 +1266,7 @@ def launch(
     build,
     repository,
     project_queue,
+    dockerfile,
 ):
     """Start a W&B run from the given URI.
 
@@ -1282,7 +1281,7 @@ def launch(
     logger.info(
         f"=== Launch called with kwargs {locals()} CLI Version: {wandb.__version__}==="
     )
-    from wandb.sdk.launch import launch as wandb_launch
+    from wandb.sdk.launch._launch import _launch
 
     api = _get_cling_api()
     wandb._sentry.configure_scope(process_context="launch_cli")
@@ -1292,6 +1291,11 @@ def launch(
             "Cannot use both --async and --queue with wandb launch, see help for details."
         )
 
+    if queue and docker_image and not project:
+        raise LaunchError(
+            "Cannot use --queue and --docker together without a project. Please specify a project with --project or -p."
+        )
+
     if resource_args is not None:
         resource_args = util.load_json_yaml_dict(resource_args)
         if resource_args is None:
@@ -1321,26 +1325,35 @@ def launch(
 
     run_id = config.get("run_id")
 
+    if dockerfile:
+        if "overrides" in config:
+            config["overrides"]["dockerfile"] = dockerfile
+        else:
+            config["overrides"] = {"dockerfile": dockerfile}
+
     if queue is None:
         # direct launch
         try:
-            wandb_launch.run(
+            run = _launch(
                 api,
                 uri,
                 job,
-                entry_point,
-                git_version,
                 project=project,
                 entity=entity,
                 docker_image=docker_image,
                 name=name,
+                entry_point=entry_point,
+                version=git_version,
                 resource=resource,
                 resource_args=resource_args,
-                config=config,
+                launch_config=config,
                 synchronous=(not run_async),
                 run_id=run_id,
                 repository=repository,
             )
+            if run.get_status().state in ["failed", "stopped", "preempted"]:
+                wandb.termerror("Launched run exited with non-zero status")
+                sys.exit(1)
         except LaunchError as e:
             logger.error("=== %s ===", e)
             wandb._sentry.exception(e)
@@ -1436,11 +1449,11 @@ def launch_agent(
             "--url is not supported in this version, upgrade with: pip install -u wandb"
         )
 
-    from wandb.sdk.launch import launch as wandb_launch
+    import wandb.sdk.launch._launch as _launch
 
     api = _get_cling_api()
     wandb._sentry.configure_scope(process_context="launch_agent")
-    agent_config, api = wandb_launch.resolve_agent_config(
+    agent_config, api = _launch.resolve_agent_config(
         entity, project, max_jobs, queues, config
     )
     if agent_config.get("project") is None:
@@ -1457,7 +1470,7 @@ def launch_agent(
 
     wandb.termlog("Starting launch agent ✨")
     try:
-        wandb_launch.create_and_run_agent(api, agent_config)
+        _launch.create_and_run_agent(api, agent_config)
     except Exception as e:
         wandb._sentry.exception(e)
         raise e
@@ -1497,11 +1510,11 @@ def scheduler(
     ctx,
     sweep_id,
 ):
-    api = _get_cling_api()
+    api = InternalApi()
     if api.api_key is None:
         wandb.termlog("Login to W&B to use the sweep scheduler feature")
         ctx.invoke(login, no_offline=True)
-        api = _get_cling_api(reset=True)
+        api = InternalApi(reset=True)
 
     wandb._sentry.configure_scope(process_context="sweep_scheduler")
     wandb.termlog("Starting a Launch Scheduler 🚀")
@@ -1531,12 +1544,12 @@ def scheduler(
         raise e
 
 
-@cli.group("job")
+@cli.group(help="Commands for managing and viewing W&B jobs")
 def job() -> None:
     pass
 
 
-@job.command("list")
+@job.command("list", help="List jobs in a project")
 @click.option(
     "--project",
     "-p",
@@ -1578,7 +1591,9 @@ def _list(project, entity):
         wandb.termlog(f"{name} -- versions ({len(aliases)}): {aliases_str}")
 
 
-@job.command()
+@job.command(
+    help="Describe a launch job. Provide the launch job in the form of: entity/project/job-name:alias-or-version"
+)
 @click.argument("job")
 def describe(job):
     public_api = PublicApi()
@@ -2058,8 +2073,15 @@ def artifact():
     multiple=True,
     help="An alias to apply to this artifact",
 )
+@click.option("--id", "run_id", help="The run you want to upload to.")
+@click.option(
+    "--resume",
+    is_flag=True,
+    default=None,
+    help="Resume the last run from your current directory.",
+)
 @display_error
-def put(path, name, description, type, alias):
+def put(path, name, description, type, alias, run_id, resume):
     if name is None:
         name = os.path.basename(path)
     public_api = PublicApi()
@@ -2087,32 +2109,21 @@ def put(path, name, description, type, alias):
         raise ClickException("Path argument must be a file or directory")
 
     run = wandb.init(
-        entity=entity, project=project, config={"path": path}, job_type="cli_put"
-    )
-    # We create the artifact manually to get the current version
-    res, _ = api.create_artifact(
-        type,
-        artifact_name,
-        artifact.digest,
-        client_id=artifact._client_id,
-        sequence_client_id=artifact._sequence_client_id,
-        entity_name=entity,
-        project_name=project,
-        run_name=run.id,
-        description=description,
-        aliases=[{"artifactCollectionName": artifact_name, "alias": a} for a in alias],
+        entity=entity,
+        project=project,
+        config={"path": path},
+        job_type="cli_put",
+        id=run_id,
+        resume=resume,
     )
-    artifact_path = artifact_path.split(":")[0] + ":" + res.get("version", "latest")
-    # Re-create the artifact and actually upload any files needed
     run.log_artifact(artifact, aliases=alias)
     artifact.wait()
 
     wandb.termlog(
         "Artifact uploaded, use this artifact in a run by adding:\n", prefix=False
     )
-
     wandb.termlog(
-        f'    artifact = run.use_artifact("{artifact_path}")\n',
+        f'    artifact = run.use_artifact("{artifact.source_qualified_name}")\n',
         prefix=False,
     )
 

wandb/docker/__init__.py

@@ -82,6 +82,22 @@ def is_buildx_installed() -> bool:
     return _buildx_installed
 
 
+def is_docker_installed() -> bool:
+    """Return `True` if docker is installed and working, else `False`."""
+    try:
+        # Run the docker --version command
+        result = subprocess.run(
+            ["docker", "--version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        )
+        if result.returncode == 0:
+            return True
+        else:
+            return False
+    except FileNotFoundError:
+        # If docker command is not found
+        return False
+
+
 def build(
     tags: List[str], file: str, context_path: str, platform: Optional[str] = None
 ) -> str:
@@ -304,6 +320,7 @@ __all__ = [
     "run",
     "image_id",
     "image_id_from_registry",
+    "is_docker_installed",
     "auth_token",
     "parse",
     "parse_repository_tag",

wandb/docker/auth.py

@@ -106,7 +106,7 @@ def resolve_repository_name(repo_name: str) -> Tuple[str, str]:
     index_name, remote_name = split_repo_name(repo_name)
     if index_name[0] == "-" or index_name[-1] == "-":
         raise InvalidRepositoryError(
-            f"Invalid index name ({index_name}). Cannot begin or end with a" " hyphen."
+            f"Invalid index name ({index_name}). Cannot begin or end with a hyphen."
         )
     return resolve_index_name(index_name), remote_name