wandb 0.15.9__py3-none-any.whl → 0.15.11__py3-none-any.whl

wandb/sdk/wandb_settings.py

@@ -32,7 +32,7 @@ from typing import (
     Union,
     no_type_check,
 )
-from urllib.parse import quote, urlencode, urlparse, urlsplit
+from urllib.parse import quote, unquote, urlencode, urlparse, urlsplit
 
 from google.protobuf.wrappers_pb2 import BoolValue, DoubleValue, Int32Value, StringValue
 
@@ -170,13 +170,18 @@ def _get_program() -> Optional[str]:
         return None
 
 
-def _get_program_relpath_from_gitrepo(
-    program: str, _logger: Optional[_EarlyLogger] = None
+def _get_program_relpath(
+    program: str, root: Optional[str] = None, _logger: Optional[_EarlyLogger] = None
 ) -> Optional[str]:
-    repo = GitRepo()
-    root = repo.root
+    if not program:
+        if _logger is not None:
+            _logger.warning("Empty program passed to get_program_relpath")
+        return None
+
+    root = root or os.getcwd()
     if not root:
-        root = os.getcwd()
+        return None
+
     full_path_to_program = os.path.join(
         root, os.path.relpath(os.getcwd(), root), program
     )
@@ -304,12 +309,27 @@ class SettingsData:
     _disable_setproctitle: bool  # Do not use setproctitle on internal process
     _disable_stats: bool  # Do not collect system metrics
     _disable_viewer: bool  # Prevent early viewer query
+    _disable_machine_info: bool  # Disable automatic machine info collection
     _except_exit: bool
     _executable: str
     _extra_http_headers: Mapping[str, str]
-    _file_stream_timeout_seconds: float
+    # file stream retry client configuration
+    _file_stream_retry_max: int  # max number of retries
+    _file_stream_retry_wait_min_seconds: int  # min wait time between retries
+    _file_stream_retry_wait_max_seconds: int  # max wait time between retries
+    _file_stream_timeout_seconds: int  # timeout for individual HTTP requests
+    # file uploader retry client configuration
+    _file_uploader_retry_max: int
+    _file_uploader_retry_wait_min_seconds: int
+    _file_uploader_retry_wait_max_seconds: int
+    _file_uploader_timeout_seconds: int
     _flow_control_custom: bool
     _flow_control_disabled: bool
+    # graphql retry client configuration
+    _graphql_retry_max: int
+    _graphql_retry_wait_min_seconds: int
+    _graphql_retry_wait_max_seconds: int
+    _graphql_timeout_seconds: int
     _internal_check_process: float
     _internal_queue_timeout: float
     _ipython: bool
@@ -328,6 +348,7 @@ class SettingsData:
     _sync: bool
     _os: str
     _platform: str
+    _proxies: Mapping[str, str]  # dedicated global proxy servers [scheme -> url]
     _python: str
     _runqueue_item_id: str
     _require_nexus: bool
@@ -346,6 +367,7 @@ class SettingsData:
     # - {"metric regex pattern, including endpoint name as prefix": {"label": "label value regex pattern"}}
     # - ("metric regex pattern 1", "metric regex pattern 2", ...)
     _stats_open_metrics_filters: Union[Sequence[str], Mapping[str, Mapping[str, str]]]
+    _stats_disk_paths: Sequence[str]  # paths to monitor disk usage
     _tmp_code_dir: str
     _tracelog: str
     _unsaved_keys: Sequence[str]
@@ -356,6 +378,7 @@ class SettingsData:
     azure_account_url_to_access_key: Dict[str, str]
     base_url: str  # The base url for the wandb api
     code_dir: str
+    colab_url: str
     config_paths: Sequence[str]
     console: str
     deployment: str
@@ -394,7 +417,8 @@ class SettingsData:
     notebook_name: str
     problem: str
     program: str
-    program_relpath: Optional[str]
+    program_abspath: str
+    program_relpath: str
     project: str
     project_url: str
     quiet: bool
@@ -587,29 +611,65 @@ class Settings(SettingsData):
         Note that key names must be the same as the class attribute names.
         """
         props: Dict[str, Dict[str, Any]] = dict(
+            _async_upload_concurrency_limit={
+                "preprocessor": int,
+                "validator": self._validate__async_upload_concurrency_limit,
+            },
             _aws_lambda={
                 "hook": lambda _: is_aws_lambda(),
                 "auto_hook": True,
             },
-            _async_upload_concurrency_limit={
-                "preprocessor": int,
-                "validator": self._validate__async_upload_concurrency_limit,
+            _colab={
+                "hook": lambda _: "google.colab" in sys.modules,
+                "auto_hook": True,
+            },
+            _disable_machine_info={
+                "value": False,
+                "preprocessor": _str_as_bool,
+            },
+            _disable_meta={
+                "value": False,
+                "preprocessor": _str_as_bool,
+                "hook": lambda x: self._disable_machine_info or x,
             },
-            _disable_meta={"preprocessor": _str_as_bool},
             _disable_service={
                 "value": False,
                 "preprocessor": _str_as_bool,
                 "is_policy": True,
             },
             _disable_setproctitle={"value": False, "preprocessor": _str_as_bool},
-            _disable_stats={"preprocessor": _str_as_bool},
+            _disable_stats={
+                "value": False,
+                "preprocessor": _str_as_bool,
+                "hook": lambda x: self._disable_machine_info or x,
+            },
             _disable_viewer={"preprocessor": _str_as_bool},
             _extra_http_headers={"preprocessor": _str_as_json},
-            _network_buffer={"preprocessor": int},
-            _colab={
-                "hook": lambda _: "google.colab" in sys.modules,
+            # Retry filestream requests for 2 hours before dropping chunk (how do we recover?)
+            # retry_count = seconds_in_2_hours / max_retry_time + num_retries_until_max_60_sec
+            #             = 7200 / 60 + ceil(log2(60/2))
+            #             = 120 + 5
+            _file_stream_retry_max={"value": 125, "preprocessor": int},
+            _file_stream_retry_wait_min_seconds={"value": 2, "preprocessor": int},
+            _file_stream_retry_wait_max_seconds={"value": 60, "preprocessor": int},
+            # A 3 minute timeout for all filestream post requests
+            _file_stream_timeout_seconds={"value": 180, "preprocessor": int},
+            _file_uploader_retry_max={"value": 10, "preprocessor": int},
+            _file_uploader_retry_wait_min_seconds={"value": 2, "preprocessor": int},
+            _file_uploader_retry_wait_max_seconds={"value": 60, "preprocessor": int},
+            _file_uploader_timeout_seconds={"value": 0, "preprocessor": int},
+            _flow_control_disabled={
+                "hook": lambda _: self._network_buffer == 0,
+                "auto_hook": True,
+            },
+            _flow_control_custom={
+                "hook": lambda _: bool(self._network_buffer),
                 "auto_hook": True,
             },
+            _graphql_retry_max={"value": 10, "preprocessor": int},
+            _graphql_retry_wait_min_seconds={"value": 2, "preprocessor": int},
+            _graphql_retry_wait_max_seconds={"value": 60, "preprocessor": int},
+            _graphql_timeout_seconds={"value": 30.0, "preprocessor": int},
             _internal_check_process={"value": 8, "preprocessor": float},
             _internal_queue_timeout={"value": 2, "preprocessor": float},
             _ipython={
@@ -622,6 +682,7 @@ class Settings(SettingsData):
             },
             _kaggle={"hook": lambda _: util._is_likely_kaggle(), "auto_hook": True},
             _log_level={"value": logging.DEBUG},
+            _network_buffer={"preprocessor": int},
             _noop={"hook": lambda _: self.mode == "disabled", "auto_hook": True},
             _notebook={
                 "hook": lambda _: self._ipython
@@ -638,20 +699,10 @@ class Settings(SettingsData):
                 ),
                 "auto_hook": True,
             },
-            _file_stream_timeout_seconds={
-                "value": 60,
-                "preprocessor": float,
-            },
-            _flow_control_disabled={
-                "hook": lambda _: self._network_buffer == 0,
-                "auto_hook": True,
-            },
-            _flow_control_custom={
-                "hook": lambda _: bool(self._network_buffer),
-                "auto_hook": True,
-            },
-            _sync={"value": False},
             _platform={"value": util.get_platform_name()},
+            _proxies={
+                "preprocessor": _str_as_json,
+            },
             _require_nexus={"value": False, "preprocessor": _str_as_bool},
             _save_requirements={"value": True, "preprocessor": _str_as_bool},
             _service_wait={
@@ -682,6 +733,11 @@ class Settings(SettingsData):
                 "value": (".*",),
                 "preprocessor": _str_as_json,
             },
+            _stats_disk_paths={
+                "value": ("/",),
+                "preprocessor": _str_as_json,
+            },
+            _sync={"value": False},
             _tmp_code_dir={
                 "value": "code",
                 "hook": lambda x: self._path_convert(self.tmp_dir, x),
@@ -697,6 +753,10 @@ class Settings(SettingsData):
                 "preprocessor": lambda x: str(x).strip().rstrip("/"),
                 "validator": self._validate_base_url,
             },
+            colab_url={
+                "hook": lambda _: self._get_colab_url(),
+                "auto_hook": True,
+            },
             config_paths={"preprocessor": _str_as_tuple},
             console={
                 "value": "auto",
@@ -708,10 +768,22 @@ class Settings(SettingsData):
                 "hook": lambda _: "local" if self.is_local else "cloud",
                 "auto_hook": True,
             },
-            disable_code={"preprocessor": _str_as_bool},
+            disable_code={
+                "value": False,
+                "preprocessor": _str_as_bool,
+                "hook": lambda x: self._disable_machine_info or x,
+            },
             disable_hints={"preprocessor": _str_as_bool},
-            disable_git={"preprocessor": _str_as_bool},
-            disable_job_creation={"value": False, "preprocessor": _str_as_bool},
+            disable_git={
+                "value": False,
+                "preprocessor": _str_as_bool,
+                "hook": lambda x: self._disable_machine_info or x,
+            },
+            disable_job_creation={
+                "value": False,
+                "preprocessor": _str_as_bool,
+                "hook": lambda x: self._disable_machine_info or x,
+            },
             disabled={"value": False, "preprocessor": _str_as_bool},
             files_dir={
                 "value": "files",
@@ -726,7 +798,7 @@ class Settings(SettingsData):
                 "value": tuple(),
                 "preprocessor": lambda x: tuple(x) if not isinstance(x, tuple) else x,
             },
-            init_timeout={"value": 60, "preprocessor": lambda x: float(x)},
+            init_timeout={"value": 90, "preprocessor": lambda x: float(x)},
             is_local={
                 "hook": (
                     lambda _: self.base_url != "https://api.wandb.ai"
@@ -764,6 +836,9 @@ class Settings(SettingsData):
             login_timeout={"preprocessor": lambda x: float(x)},
             mode={"value": "online", "validator": self._validate_mode},
             problem={"value": "fatal", "validator": self._validate_problem},
+            program={
+                "hook": lambda x: self._get_program(x),
+            },
             project={"validator": self._validate_project},
             project_url={"hook": lambda _: self._project_url(), "auto_hook": True},
             quiet={"preprocessor": _str_as_bool},
@@ -1134,6 +1209,32 @@ class Settings(SettingsData):
                 console = "redirect"
         return console
 
+    def _get_colab_url(self) -> Optional[str]:
+        if not self._colab:
+            return None
+        if self._jupyter_path and self._jupyter_path.startswith("fileId="):
+            unescaped = unquote(self._jupyter_path)
+            return "https://colab.research.google.com/notebook#" + unescaped
+        return None
+
+    def _get_program(self, program: Optional[str]) -> Optional[str]:
+        if program is not None and program != "<python with no main file>":
+            return program
+
+        if not self._jupyter:
+            return program
+
+        if self.notebook_name:
+            return self.notebook_name
+
+        if not self._jupyter_path:
+            return program
+
+        if self._jupyter_path.startswith("fileId="):
+            return self._jupyter_name
+        else:
+            return self._jupyter_path
+
     def _get_url_query_string(self) -> str:
         # TODO(settings) use `wandb_setting` (if self.anonymous != "true":)
         if Api().settings().get("anonymous") != "true":
@@ -1545,6 +1646,9 @@ class Settings(SettingsData):
             "WANDB_NOTES": "run_notes",
             "WANDB_TAGS": "run_tags",
             "WANDB_JOB_TYPE": "run_job_type",
+            "WANDB_HTTP_TIMEOUT": "_graphql_timeout_seconds",
+            "WANDB_FILE_PUSHER_TIMEOUT": "_file_uploader_timeout_seconds",
+            "WANDB_USER_EMAIL": "email",
         }
         env = dict()
         for setting, value in environ.items():
@@ -1665,10 +1769,18 @@ class Settings(SettingsData):
         settings: Dict[str, Union[bool, str, None]] = dict()
         program = self.program or _get_program()
         if program is not None:
-            program_relpath = self.program_relpath or _get_program_relpath_from_gitrepo(
-                program, _logger=_logger
+            repo = GitRepo()
+            root = repo.root or os.getcwd()
+
+            program_relpath = self.program_relpath or _get_program_relpath(
+                program, repo.root, _logger=_logger
             )
             settings["program_relpath"] = program_relpath
+            program_abspath = os.path.abspath(
+                os.path.join(root, os.path.relpath(os.getcwd(), root), program)
+            )
+            if os.path.exists(program_abspath):
+                settings["program_abspath"] = program_abspath
         else:
             program = "<python with no main file>"
 
@@ -1731,6 +1843,7 @@ class Settings(SettingsData):
             job_type="run_job_type",
             notes="run_notes",
             dir="root_dir",
+            sweep_id="sweep_id",
         )
         init_settings = {
             param_map.get(k, k): v for k, v in init_settings.items() if v is not None

wandb/testing/relay.py

@@ -74,6 +74,18 @@ class DeliberateHTTPError(Exception):
         return f"DeliberateHTTPError({self.message!r}, {self.status_code!r})"
 
 
+@dataclasses.dataclass
+class RunAttrs:
+    """Simple data class for run attributes."""
+
+    name: str
+    display_name: str
+    description: str
+    sweep_name: str
+    project: Dict[str, Any]
+    config: Dict[str, Any]
+
+
 class Context:
     """A container used to store the snooped state/data of a test.
 
@@ -223,6 +235,20 @@ class Context:
         run_stats = self.events[mask_run]
         return run_stats
 
+    def get_run_attrs(self, run_id: str) -> Optional[RunAttrs]:
+        run_entry = self._entries.get(run_id)
+        if not run_entry:
+            return None
+
+        return RunAttrs(
+            name=run_entry["name"],
+            display_name=run_entry["displayName"],
+            description=run_entry["description"],
+            sweep_name=run_entry["sweepName"],
+            project=run_entry["project"],
+            config=run_entry["config"],
+        )
+
     # todo: add getter (by run_id) utilities for other properties
 
 
@@ -317,8 +343,8 @@ class QueryResolver:
         query = "CreateRunFiles" in request_data.get("query", "")
         if query:
             run_name = request_data["variables"]["run"]
-            files = (
-                response_data.get("data", {}).get("createRunFiles", {}).get("files", {})
+            files = ((response_data.get("data") or {}).get("createRunFiles") or {}).get(
+                "files", {}
             )
             post_processed_data = {
                 "name": run_name,
@@ -526,6 +552,7 @@ class RelayServer:
         base_url: str,
         inject: Optional[List[InjectedResponse]] = None,
         control: Optional[RelayControlProtocol] = None,
+        verbose: bool = False,
     ) -> None:
         # todo for the future:
         #  - consider switching from Flask to Quart
@@ -581,6 +608,7 @@ class RelayServer:
 
         # useful when debugging:
         # self.after_request_fn = self.app.after_request(self.after_request_fn)
+        self.verbose = verbose
 
     @staticmethod
     def handle_http_exception(e):
@@ -632,11 +660,25 @@ class RelayServer:
             json=request.get_json(),
         ).prepare()
 
+        if self.verbose:
+            print("*****************")
+            print("RELAY REQUEST:")
+            print(prepared_relayed_request.url)
+            print(prepared_relayed_request.method)
+            print(prepared_relayed_request.headers)
+            print(prepared_relayed_request.body)
+            print("*****************")
+
         for injected_response in self.inject:
             # where are we in the application pattern?
             should_apply = injected_response.application_pattern.should_apply()
             # check if an injected response matches the request
             if injected_response == prepared_relayed_request:
+                if self.verbose:
+                    print("*****************")
+                    print("INJECTING RESPONSE:")
+                    print(injected_response.to_dict())
+                    print("*****************")
                 # rotate the injection pattern
                 injected_response.application_pattern.next()
                 if should_apply:
@@ -693,19 +735,21 @@ class RelayServer:
         request = flask.request
         with Timer() as timer:
             relayed_response = self.relay(request)
-        # print("*****************")
-        # print("GRAPHQL REQUEST:")
-        # print(request.get_json())
-        # print("GRAPHQL RESPONSE:")
-        # print(relayed_response.status_code, relayed_response.json())
-        # print("*****************")
+        if self.verbose:
+            print("*****************")
+            print("GRAPHQL REQUEST:")
+            print(request.get_json())
+            print("GRAPHQL RESPONSE:")
+            print(relayed_response.status_code, relayed_response.json())
+            print("*****************")
         # snoop work to extract the context
         self.snoop_context(request, relayed_response, timer.elapsed)
-        # print("*****************")
-        # print("SNOOPED CONTEXT:")
-        # print(self.context.entries)
-        # print(len(self.context.raw_data))
-        # print("*****************")
+        if self.verbose:
+            print("*****************")
+            print("SNOOPED CONTEXT:")
+            print(self.context.entries)
+            print(len(self.context.raw_data))
+            print("*****************")
 
         return relayed_response.json()
 
@@ -713,16 +757,17 @@ class RelayServer:
         request = flask.request
         with Timer() as timer:
             relayed_response = self.relay(request)
-        # print("*****************")
-        # print("FILE STREAM REQUEST:")
-        # print("********PATH*********")
-        # print(path)
-        # print("********ENDPATH*********")
-        # print(request.get_json())
-        # print("FILE STREAM RESPONSE:")
-        # print(relayed_response)
-        # print(relayed_response.status_code, relayed_response.json())
-        # print("*****************")
+        if self.verbose:
+            print("*****************")
+            print("FILE STREAM REQUEST:")
+            print("********PATH*********")
+            print(path)
+            print("********ENDPATH*********")
+            print(request.get_json())
+            print("FILE STREAM RESPONSE:")
+            print(relayed_response)
+            print(relayed_response.status_code, relayed_response.json())
+            print("*****************")
 
         self.snoop_context(request, relayed_response, timer.elapsed, path=path)
 
@@ -732,12 +777,13 @@ class RelayServer:
         request = flask.request
         with Timer() as timer:
             relayed_response = self.relay(request)
-        # print("*****************")
-        # print("STORAGE REQUEST:")
-        # print(request.get_json())
-        # print("STORAGE RESPONSE:")
-        # print(relayed_response.status_code, relayed_response.json())
-        # print("*****************")
+        if self.verbose:
+            print("*****************")
+            print("STORAGE REQUEST:")
+            print(request.get_json())
+            print("STORAGE RESPONSE:")
+            print(relayed_response.status_code, relayed_response.json())
+            print("*****************")
 
         self.snoop_context(request, relayed_response, timer.elapsed)
 
@@ -747,15 +793,16 @@ class RelayServer:
         request = flask.request
         with Timer() as timer:
             relayed_response = self.relay(request)
-        # print("*****************")
-        # print("STORAGE FILE REQUEST:")
-        # print("********PATH*********")
-        # print(path)
-        # print("********ENDPATH*********")
-        # print(request.get_json())
-        # print("STORAGE FILE RESPONSE:")
-        # print(relayed_response.json())
-        # print("*****************")
+        if self.verbose:
+            print("*****************")
+            print("STORAGE FILE REQUEST:")
+            print("********PATH*********")
+            print(path)
+            print("********ENDPATH*********")
+            print(request.get_json())
+            print("STORAGE FILE RESPONSE:")
+            print(relayed_response.json())
+            print("*****************")
 
         self.snoop_context(request, relayed_response, timer.elapsed, path=path)