wandb 0.18.0__py3-none-macosx_10_13_x86_64.whl

wandb/sdk/launch/wandb_reference.py

@@ -0,0 +1,138 @@
+"""Support for parsing W&B URLs (which might be user provided) into constituent parts."""
+
+from dataclasses import dataclass
+from enum import IntEnum
+from typing import Optional
+from urllib.parse import urlparse
+
+PREFIX_HTTP = "http://"
+PREFIX_HTTPS = "https://"
+
+
+class ReferenceType(IntEnum):
+    RUN = 1
+    JOB = 2
+
+
+# Ideally we would not overload the URL paths as we do.
+# TODO: Not sure these are exhaustive, and even if so more special paths might get added.
+#       Would be good to have restrictions that we could check.
+RESERVED_NON_ENTITIES = (
+    "create-team",
+    "fully-connected",
+    "registry",
+    "settings",
+    "subscriptions",
+)
+RESERVED_NON_PROJECTS = (
+    "likes",
+    "projects",
+)
+RESERVED_JOB_PATHS = ("_view",)
+
+
+@dataclass
+class WandbReference:
+    # TODO: This will include port, should we separate that out?
+    host: Optional[str] = None
+
+    entity: Optional[str] = None
+    project: Optional[str] = None
+
+    # Set when we don't know how to parse yet
+    path: Optional[str] = None
+
+    # Reference type will determine what other fields are set
+    ref_type: Optional[ReferenceType] = None
+
+    run_id: Optional[str] = None
+
+    job_name: Optional[str] = None
+    job_alias: str = "latest"  # In addition to an alias can be a version specifier
+
+    def is_bare(self) -> bool:
+        return self.host is None
+
+    def is_job(self) -> bool:
+        return self.ref_type == ReferenceType.JOB
+
+    def is_run(self) -> bool:
+        return self.ref_type == ReferenceType.RUN
+
+    def is_job_or_run(self) -> bool:
+        return self.is_job() or self.is_run()
+
+    def job_reference(self) -> str:
+        assert self.is_job()
+        return f"{self.job_name}:{self.job_alias}"
+
+    def job_reference_scoped(self) -> str:
+        assert self.entity
+        assert self.project
+        unscoped = self.job_reference()
+        return f"{self.entity}/{self.project}/{unscoped}"
+
+    def url_host(self) -> str:
+        return f"{PREFIX_HTTPS}{self.host}" if self.host else ""
+
+    def url_entity(self) -> str:
+        assert self.entity
+        return f"{self.url_host()}/{self.entity}"
+
+    def url_project(self) -> str:
+        assert self.project
+        return f"{self.url_entity()}/{self.project}"
+
+    @staticmethod
+    def parse(uri: str) -> Optional["WandbReference"]:
+        """Attempt to parse a string as a W&B URL."""
+        # TODO: Error if HTTP and host is not localhost?
+        if (
+            not uri.startswith("/")
+            and not uri.startswith(PREFIX_HTTP)
+            and not uri.startswith(PREFIX_HTTPS)
+        ):
+            return None
+
+        ref = WandbReference()
+
+        # This takes care of things like query and fragment
+        parsed = urlparse(uri)
+        if parsed.netloc:
+            ref.host = parsed.netloc
+
+        if not parsed.path.startswith("/"):
+            return ref
+
+        ref.path = parsed.path[1:]
+        parts = ref.path.split("/")
+        if len(parts) > 0:
+            if parts[0] not in RESERVED_NON_ENTITIES:
+                ref.path = None
+                ref.entity = parts[0]
+                if len(parts) > 1:
+                    if parts[1] not in RESERVED_NON_PROJECTS:
+                        ref.project = parts[1]
+                        if len(parts) > 3 and parts[2] == "runs":
+                            ref.ref_type = ReferenceType.RUN
+                            ref.run_id = parts[3]
+                        elif (
+                            len(parts) > 4
+                            and parts[2] == "artifacts"
+                            and parts[3] == "job"
+                        ):
+                            ref.ref_type = ReferenceType.JOB
+                            ref.job_name = parts[4]
+                            if len(parts) > 5 and parts[5] not in RESERVED_JOB_PATHS:
+                                ref.job_alias = parts[5]
+                        # TODO: Right now we are not tracking selection as part of URL state in the Jobs tab.
+                        #       If that changes we'll want to update this.
+
+        return ref
+
+    @staticmethod
+    def is_uri_job_or_run(uri: str) -> bool:
+        ref = WandbReference.parse(uri)
+        if ref and ref.is_job_or_run():
+            return True
+        return False

wandb/sdk/lib/__init__.py

@@ -0,0 +1,5 @@
+from . import lazyloader
+from .disabled import RunDisabled, SummaryDisabled
+from .run_moment import RunMoment
+
+__all__ = ("lazyloader", "RunDisabled", "SummaryDisabled", "RunMoment")

wandb/sdk/lib/_settings_toposort_generate.py

@@ -0,0 +1,159 @@
+import inspect
+import sys
+from typing import Dict, List, Set, Tuple
+
+from wandb.errors import UsageError
+from wandb.sdk.wandb_settings import Settings
+
+if sys.version_info >= (3, 8):
+    from typing import get_type_hints
+else:
+    from typing_extensions import get_type_hints
+
+
+template = """
+__all__ = ("SETTINGS_TOPOLOGICALLY_SORTED", "_Setting")
+
+import sys
+from typing import Tuple
+
+if sys.version_info >= (3, 8):
+    from typing import Final, Literal
+else:
+    from typing_extensions import Final, Literal
+
+
+_Setting = Literal[
+    $settings_literal_list
+]
+
+SETTINGS_TOPOLOGICALLY_SORTED: Final[Tuple[_Setting, ...]] = (
+    $settings_topologically_sorted
+)
+"""
+
+
+class Graph:
+    # A simple class representing an unweighted directed graph
+    # that uses an adjacency list representation.
+    # We use to ensure that we don't have cyclic dependencies in the settings
+    # and that modifications to the settings are applied in the correct order.
+    def __init__(self) -> None:
+        self.adj_list: Dict[str, Set[str]] = {}
+
+    def add_node(self, node: str) -> None:
+        if node not in self.adj_list:
+            self.adj_list[node] = set()
+
+    def add_edge(self, node1: str, node2: str) -> None:
+        self.adj_list[node1].add(node2)
+
+    def get_neighbors(self, node: str) -> Set[str]:
+        return self.adj_list[node]
+
+    # return a list of nodes sorted in topological order
+    def topological_sort_dfs(self) -> List[str]:
+        sorted_copy = {k: sorted(v) for k, v in self.adj_list.items()}
+
+        sorted_nodes: List[str] = []
+        visited_nodes: Set[str] = set()
+        current_nodes: Set[str] = set()
+
+        def visit(n: str) -> None:
+            if n in visited_nodes:
+                return None
+            if n in current_nodes:
+                raise UsageError("Cyclic dependency detected in wandb.Settings")
+
+            current_nodes.add(n)
+            for neighbor in sorted_copy[n]:
+                visit(neighbor)
+
+            current_nodes.remove(n)
+            visited_nodes.add(n)
+            sorted_nodes.append(n)
+
+            return None
+
+        for node in self.adj_list:
+            if node not in visited_nodes:
+                visit(node)
+
+        return sorted_nodes
+
+
+def _get_modification_order(
+    settings: Settings,
+) -> Tuple[Tuple[str, ...], Tuple[str, ...]]:
+    """Return the order in which settings should be modified, based on dependencies."""
+    dependency_graph = Graph()
+
+    props = tuple(get_type_hints(Settings).keys())
+
+    # discover prop dependencies from validator methods and runtime hooks
+
+    prefix = "_validate_"
+    symbols = set(dir(settings))
+    validator_methods = tuple(sorted(m for m in symbols if m.startswith(prefix)))
+
+    # extract dependencies from validator methods
+    for m in validator_methods:
+        setting = m.split(prefix)[1]
+        dependency_graph.add_node(setting)
+        # if the method is not static, inspect its code to find the attributes it depends on
+        if (
+            not isinstance(Settings.__dict__[m], staticmethod)
+            and not isinstance(Settings.__dict__[m], classmethod)
+            and Settings.__dict__[m].__code__.co_argcount > 0
+        ):
+            unbound_closure_vars = inspect.getclosurevars(Settings.__dict__[m]).unbound
+            dependencies = (v for v in unbound_closure_vars if v in props)
+            for d in dependencies:
+                dependency_graph.add_node(d)
+                dependency_graph.add_edge(setting, d)
+
+    # extract dependencies from props' runtime hooks
+    default_props = settings._default_props()
+    for prop, spec in default_props.items():
+        if "hook" not in spec:
+            continue
+
+        dependency_graph.add_node(prop)
+
+        hook = spec["hook"]
+        if callable(hook):
+            hook = [hook]
+
+        for h in hook:
+            unbound_closure_vars = inspect.getclosurevars(h).unbound
+            dependencies = (v for v in unbound_closure_vars if v in props)
+            for d in dependencies:
+                dependency_graph.add_node(d)
+                dependency_graph.add_edge(prop, d)
+
+    modification_order = dependency_graph.topological_sort_dfs()
+    return props, tuple(modification_order)
+
+
+def generate(settings: Settings) -> None:
+    _settings_literal_list, _settings_topologically_sorted = _get_modification_order(
+        settings
+    )
+    settings_literal_list = ", ".join(f'"{s}"' for s in _settings_literal_list)
+    settings_topologically_sorted = ", ".join(
+        f'"{s}"' for s in _settings_topologically_sorted
+    )
+
+    print(
+        template.replace(
+            "$settings_literal_list",
+            settings_literal_list,
+        ).replace(
+            "$settings_topologically_sorted",
+            settings_topologically_sorted,
+        )
+    )
+
+
+if __name__ == "__main__":
+    generate(Settings())

wandb/sdk/lib/_settings_toposort_generated.py

@@ -0,0 +1,249 @@
+# DO NOT EDIT -- GENERATED BY: `generate-tool.py --generate`
+__all__ = ("SETTINGS_TOPOLOGICALLY_SORTED", "_Setting")
+
+import sys
+from typing import Tuple
+
+if sys.version_info >= (3, 8):
+    from typing import Final, Literal
+else:
+    from typing_extensions import Final, Literal
+
+
+_Setting = Literal[
+    "_args",
+    "_aws_lambda",
+    "_cli_only_mode",
+    "_code_path_local",
+    "_colab",
+    "_cuda",
+    "_disable_meta",
+    "_disable_service",
+    "_disable_setproctitle",
+    "_disable_stats",
+    "_disable_update_check",
+    "_disable_viewer",
+    "_disable_machine_info",
+    "_executable",
+    "_extra_http_headers",
+    "_file_stream_max_bytes",
+    "_file_stream_retry_max",
+    "_file_stream_retry_wait_min_seconds",
+    "_file_stream_retry_wait_max_seconds",
+    "_file_stream_timeout_seconds",
+    "_file_transfer_retry_max",
+    "_file_transfer_retry_wait_min_seconds",
+    "_file_transfer_retry_wait_max_seconds",
+    "_file_transfer_timeout_seconds",
+    "_flow_control_custom",
+    "_flow_control_disabled",
+    "_graphql_retry_max",
+    "_graphql_retry_wait_min_seconds",
+    "_graphql_retry_wait_max_seconds",
+    "_graphql_timeout_seconds",
+    "_internal_check_process",
+    "_internal_queue_timeout",
+    "_ipython",
+    "_jupyter",
+    "_jupyter_name",
+    "_jupyter_path",
+    "_jupyter_root",
+    "_kaggle",
+    "_live_policy_rate_limit",
+    "_live_policy_wait_time",
+    "_log_level",
+    "_network_buffer",
+    "_noop",
+    "_notebook",
+    "_offline",
+    "_sync",
+    "_os",
+    "_platform",
+    "_proxies",
+    "_python",
+    "_runqueue_item_id",
+    "_require_legacy_service",
+    "_save_requirements",
+    "_service_transport",
+    "_service_wait",
+    "_shared",
+    "_start_datetime",
+    "_start_time",
+    "_stats_pid",
+    "_stats_sampling_interval",
+    "_stats_sample_rate_seconds",
+    "_stats_samples_to_average",
+    "_stats_join_assets",
+    "_stats_neuron_monitor_config_path",
+    "_stats_open_metrics_endpoints",
+    "_stats_open_metrics_filters",
+    "_stats_disk_paths",
+    "_stats_buffer_size",
+    "_tmp_code_dir",
+    "_tracelog",
+    "_unsaved_keys",
+    "_windows",
+    "allow_val_change",
+    "anonymous",
+    "api_key",
+    "azure_account_url_to_access_key",
+    "base_url",
+    "code_dir",
+    "colab_url",
+    "config_paths",
+    "console",
+    "console_multipart",
+    "credentials_file",
+    "deployment",
+    "disable_code",
+    "disable_git",
+    "disable_hints",
+    "disable_job_creation",
+    "disabled",
+    "docker",
+    "email",
+    "entity",
+    "files_dir",
+    "force",
+    "fork_from",
+    "resume_from",
+    "git_commit",
+    "git_remote",
+    "git_remote_url",
+    "git_root",
+    "heartbeat_seconds",
+    "host",
+    "http_proxy",
+    "https_proxy",
+    "identity_token_file",
+    "ignore_globs",
+    "init_timeout",
+    "is_local",
+    "job_name",
+    "job_source",
+    "label_disable",
+    "launch",
+    "launch_config_path",
+    "log_dir",
+    "log_internal",
+    "log_symlink_internal",
+    "log_symlink_user",
+    "log_user",
+    "login_timeout",
+    "mode",
+    "notebook_name",
+    "program",
+    "program_abspath",
+    "program_relpath",
+    "project",
+    "project_url",
+    "quiet",
+    "reinit",
+    "relogin",
+    "resume",
+    "resume_fname",
+    "resumed",
+    "root_dir",
+    "run_group",
+    "run_id",
+    "run_job_type",
+    "run_mode",
+    "run_name",
+    "run_notes",
+    "run_tags",
+    "run_url",
+    "sagemaker_disable",
+    "save_code",
+    "settings_system",
+    "settings_workspace",
+    "show_colors",
+    "show_emoji",
+    "show_errors",
+    "show_info",
+    "show_warnings",
+    "silent",
+    "start_method",
+    "strict",
+    "summary_errors",
+    "summary_timeout",
+    "summary_warnings",
+    "sweep_id",
+    "sweep_param_path",
+    "sweep_url",
+    "symlink",
+    "sync_dir",
+    "sync_file",
+    "sync_symlink_latest",
+    "table_raise_on_max_row_limit_exceeded",
+    "timespec",
+    "tmp_dir",
+    "username",
+    "wandb_dir",
+]
+
+SETTINGS_TOPOLOGICALLY_SORTED: Final[Tuple[_Setting, ...]] = (
+    "_service_wait",
+    "_stats_sample_rate_seconds",
+    "_stats_samples_to_average",
+    "_stats_sampling_interval",
+    "anonymous",
+    "api_key",
+    "base_url",
+    "console",
+    "job_source",
+    "mode",
+    "project",
+    "run_id",
+    "start_method",
+    "_aws_lambda",
+    "program",
+    "_code_path_local",
+    "_colab",
+    "_disable_machine_info",
+    "_disable_meta",
+    "_disable_stats",
+    "_network_buffer",
+    "_flow_control_disabled",
+    "_flow_control_custom",
+    "_ipython",
+    "_jupyter",
+    "_kaggle",
+    "_noop",
+    "_notebook",
+    "disabled",
+    "_offline",
+    "_shared",
+    "_stats_neuron_monitor_config_path",
+    "run_mode",
+    "_start_datetime",
+    "timespec",
+    "root_dir",
+    "wandb_dir",
+    "tmp_dir",
+    "_tmp_code_dir",
+    "_windows",
+    "colab_url",
+    "is_local",
+    "deployment",
+    "disable_code",
+    "disable_git",
+    "disable_job_creation",
+    "files_dir",
+    "_proxies",
+    "http_proxy",
+    "https_proxy",
+    "log_dir",
+    "log_internal",
+    "log_symlink_internal",
+    "log_symlink_user",
+    "log_user",
+    "project_url",
+    "resume_fname",
+    "run_url",
+    "settings_system",
+    "settings_workspace",
+    "sweep_url",
+    "sync_dir",
+    "sync_file",
+    "sync_symlink_latest",
+)

wandb/sdk/lib/_wburls_generate.py

@@ -0,0 +1,25 @@
+from wburls import wburls  # type: ignore
+
+template = """
+import sys
+
+if sys.version_info >= (3, 8):
+    from typing import Literal
+else:
+    from typing_extensions import Literal
+
+
+URLS = Literal[
+    $literal_list
+]
+"""
+
+
+def generate() -> None:
+    urls = wburls._get_urls()
+    literal_list = ", ".join([f"{key!r}" for key in urls])
+    print(template.replace("$literal_list", literal_list))
+
+
+if __name__ == "__main__":
+    generate()

wandb/sdk/lib/_wburls_generated.py

@@ -0,0 +1,22 @@
+# DO NOT EDIT -- GENERATED BY: `generate-tool.py --generate`
+import sys
+
+if sys.version_info >= (3, 8):
+    from typing import Literal
+else:
+    from typing_extensions import Literal
+
+
+URLS = Literal[
+    "cli_launch",
+    "doc_run",
+    "doc_require",
+    "doc_start_err",
+    "doc_artifacts_guide",
+    "upgrade_server",
+    "multiprocess",
+    "wandb_init",
+    "wandb_server",
+    "wandb_define_metric",
+    "wandb_core",
+]