tensorbored 2.21.0rc1769983804__py3-none-any.whl

tensorbored/context.py

@@ -0,0 +1,129 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Request-scoped context."""
+
+from tensorbored import auth as auth_lib
+
+# A `RequestContext` value is stored on WSGI environments under this key.
+_WSGI_KEY = "tensorbored.request_context"
+
+
+class RequestContext:
+    """Container of request-scoped values.
+
+    This context is for cross-cutting concerns: authentication,
+    authorization, auditing, internationalization, logging, and so on.
+    It is not simply for passing commonly used parameters to functions.
+
+    `RequestContext` values are to be treated as immutable.
+
+    Fields:
+      auth: An `AuthContext`, which may be empty but is never `None`.
+      remote_ip: An `ipaddress.IPv4Address` or `ipaddress.IPv6Address` or None.
+        Best guess of the IP Address of the end user.
+      x_forwarded_for: A tuple of `ipaddress.IPv4Address` or `ipaddress.IPv6Address`,
+        which may be empty but is never None. This should be parsed value of X-Forwarded-For
+        HTTP header from the request.
+      client_feature_flags: A dict of string to arbitrary type. These represent
+        feature flag key/value pairs sent by the client application. Usage of
+        client_feature_flags should know the name of the feature flag key and
+        should know and validate the type of the value.
+    """
+
+    def __init__(
+        self,
+        auth=None,
+        remote_ip=None,
+        x_forwarded_for=None,
+        client_feature_flags=None,
+    ):
+        """Create a request context.
+
+        The argument list is sorted and may be extended in the future;
+        therefore, callers must pass only named arguments to this
+        initializer.
+
+        Args:
+          See "Fields" on class docstring. All arguments are optional
+          and will be replaced with default values if appropriate.
+        """
+        self._auth = auth if auth is not None else auth_lib.AuthContext.empty()
+        self._remote_ip = remote_ip
+        self._x_forwarded_for = x_forwarded_for or ()
+        self._client_feature_flags = client_feature_flags or {}
+
+    @property
+    def auth(self):
+        return self._auth
+
+    @property
+    def remote_ip(self):
+        return self._remote_ip
+
+    @property
+    def x_forwarded_for(self):
+        return self._x_forwarded_for
+
+    @property
+    def client_feature_flags(self):
+        return self._client_feature_flags
+
+    def replace(self, **kwargs):
+        """Create a copy of this context with updated key-value pairs.
+
+        Analogous to `namedtuple._replace`. For example, to create a new
+        request context like `ctx` but with auth context `auth`, call
+        `ctx.replace(auth=auth)`.
+
+        Args:
+          As to `__init__`.
+
+        Returns:
+          A new context like this one but with the specified updates.
+        """
+        kwargs.setdefault("auth", self.auth)
+        kwargs.setdefault("remote_ip", self.remote_ip)
+        kwargs.setdefault("x_forwarded_for", self.x_forwarded_for)
+        kwargs.setdefault("client_feature_flags", self.client_feature_flags)
+        return type(self)(**kwargs)
+
+
+def from_environ(environ):
+    """Get a `RequestContext` from a WSGI environment.
+
+    See also `set_in_environ`.
+
+    Args:
+      environ: A WSGI environment (see PEP 3333).
+
+    Returns:
+      The `RequestContext` stored in the WSGI environment, or an empty
+      `RequestContext` if none is stored.
+    """
+    result = environ.get(_WSGI_KEY)
+    return result if result is not None else RequestContext()
+
+
+def set_in_environ(environ, ctx):
+    """Set the `RequestContext` in a WSGI environment.
+
+    After `set_in_environ(e, ctx)`, `from_environ(e) is ctx`. The input
+    environment is mutated.
+
+    Args:
+      environ: A WSGI environment to update.
+      ctx: A new `RequestContext` value.
+    """
+    environ[_WSGI_KEY] = ctx

tensorbored/data/grpc_provider.py

@@ -0,0 +1,365 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""A data provider that talks to a gRPC server."""
+
+import collections
+import contextlib
+
+import grpc
+
+from tensorbored.util import tensor_util
+from tensorbored.util import timing
+from tensorbored import errors
+from tensorbored.data import provider
+from tensorbored.data.proto import data_provider_pb2
+from tensorbored.data.proto import data_provider_pb2_grpc
+
+
+def make_stub(channel):
+    """Wraps a gRPC channel with a service stub."""
+    return data_provider_pb2_grpc.TensorBoardDataProviderStub(channel)
+
+
+class GrpcDataProvider(provider.DataProvider):
+    """Data provider that talks over gRPC."""
+
+    def __init__(self, addr, stub):
+        """Initializes a GrpcDataProvider.
+
+        Args:
+          addr: String address of the remote peer. Used cosmetically for
+            data location.
+          stub: `data_provider_pb2_grpc.TensorBoardDataProviderStub`
+            value. See `make_stub` to construct one from a channel.
+        """
+        self._addr = addr
+        self._stub = stub
+
+    def __str__(self):
+        return "GrpcDataProvider(addr=%r)" % self._addr
+
+    def experiment_metadata(self, ctx, *, experiment_id):
+        req = data_provider_pb2.GetExperimentRequest()
+        req.experiment_id = experiment_id
+        with _translate_grpc_error():
+            res = self._stub.GetExperiment(req)
+        res = provider.ExperimentMetadata(
+            data_location=res.data_location,
+            experiment_name=res.name,
+            experiment_description=res.description,
+            creation_time=_timestamp_proto_to_float(res.creation_time),
+        )
+        return res
+
+    def list_plugins(self, ctx, *, experiment_id):
+        req = data_provider_pb2.ListPluginsRequest()
+        req.experiment_id = experiment_id
+        with _translate_grpc_error():
+            res = self._stub.ListPlugins(req)
+        return [p.name for p in res.plugins]
+
+    def list_runs(self, ctx, *, experiment_id):
+        req = data_provider_pb2.ListRunsRequest()
+        req.experiment_id = experiment_id
+        with _translate_grpc_error():
+            res = self._stub.ListRuns(req)
+        return [
+            provider.Run(
+                run_id=run.name,
+                run_name=run.name,
+                start_time=run.start_time,
+            )
+            for run in res.runs
+        ]
+
+    @timing.log_latency
+    def list_scalars(
+        self, ctx, *, experiment_id, plugin_name, run_tag_filter=None
+    ):
+        with timing.log_latency("build request"):
+            req = data_provider_pb2.ListScalarsRequest()
+            req.experiment_id = experiment_id
+            req.plugin_filter.plugin_name = plugin_name
+            _populate_rtf(run_tag_filter, req.run_tag_filter)
+        with timing.log_latency("_stub.ListScalars"):
+            with _translate_grpc_error():
+                res = self._stub.ListScalars(req)
+        with timing.log_latency("build result"):
+            result = {}
+            for run_entry in res.runs:
+                tags = {}
+                result[run_entry.run_name] = tags
+                for tag_entry in run_entry.tags:
+                    time_series = tag_entry.metadata
+                    tags[tag_entry.tag_name] = provider.ScalarTimeSeries(
+                        max_step=time_series.max_step,
+                        max_wall_time=time_series.max_wall_time,
+                        plugin_content=time_series.summary_metadata.plugin_data.content,
+                        description=time_series.summary_metadata.summary_description,
+                        display_name=time_series.summary_metadata.display_name,
+                    )
+            return result
+
+    @timing.log_latency
+    def read_scalars(
+        self,
+        ctx,
+        *,
+        experiment_id,
+        plugin_name,
+        downsample=None,
+        run_tag_filter=None,
+    ):
+        with timing.log_latency("build request"):
+            req = data_provider_pb2.ReadScalarsRequest()
+            req.experiment_id = experiment_id
+            req.plugin_filter.plugin_name = plugin_name
+            _populate_rtf(run_tag_filter, req.run_tag_filter)
+            req.downsample.num_points = downsample
+        with timing.log_latency("_stub.ReadScalars"):
+            with _translate_grpc_error():
+                res = self._stub.ReadScalars(req)
+        with timing.log_latency("build result"):
+            result = {}
+            for run_entry in res.runs:
+                tags = {}
+                result[run_entry.run_name] = tags
+                for tag_entry in run_entry.tags:
+                    series = []
+                    tags[tag_entry.tag_name] = series
+                    d = tag_entry.data
+                    for step, wt, value in zip(d.step, d.wall_time, d.value):
+                        point = provider.ScalarDatum(
+                            step=step,
+                            wall_time=wt,
+                            value=value,
+                        )
+                        series.append(point)
+            return result
+
+    @timing.log_latency
+    def read_last_scalars(
+        self,
+        ctx,
+        *,
+        experiment_id,
+        plugin_name,
+        run_tag_filter=None,
+    ):
+        with timing.log_latency("build request"):
+            req = data_provider_pb2.ReadScalarsRequest()
+            req.experiment_id = experiment_id
+            req.plugin_filter.plugin_name = plugin_name
+            _populate_rtf(run_tag_filter, req.run_tag_filter)
+            # `ReadScalars` always includes the most recent datum, therefore
+            # downsampling to one means fetching the latest value.
+            req.downsample.num_points = 1
+        with timing.log_latency("_stub.ReadScalars"):
+            with _translate_grpc_error():
+                res = self._stub.ReadScalars(req)
+        with timing.log_latency("build result"):
+            result = collections.defaultdict(dict)
+            for run_entry in res.runs:
+                run_name = run_entry.run_name
+                for tag_entry in run_entry.tags:
+                    d = tag_entry.data
+                    # There should be no more than one datum in
+                    # `tag_entry.data` since downsample was set to 1.
+                    for step, wt, value in zip(d.step, d.wall_time, d.value):
+                        result[run_name][tag_entry.tag_name] = (
+                            provider.ScalarDatum(
+                                step=step,
+                                wall_time=wt,
+                                value=value,
+                            )
+                        )
+            return result
+
+    @timing.log_latency
+    def list_tensors(
+        self, ctx, *, experiment_id, plugin_name, run_tag_filter=None
+    ):
+        with timing.log_latency("build request"):
+            req = data_provider_pb2.ListTensorsRequest()
+            req.experiment_id = experiment_id
+            req.plugin_filter.plugin_name = plugin_name
+            _populate_rtf(run_tag_filter, req.run_tag_filter)
+        with timing.log_latency("_stub.ListTensors"):
+            with _translate_grpc_error():
+                res = self._stub.ListTensors(req)
+        with timing.log_latency("build result"):
+            result = {}
+            for run_entry in res.runs:
+                tags = {}
+                result[run_entry.run_name] = tags
+                for tag_entry in run_entry.tags:
+                    time_series = tag_entry.metadata
+                    tags[tag_entry.tag_name] = provider.TensorTimeSeries(
+                        max_step=time_series.max_step,
+                        max_wall_time=time_series.max_wall_time,
+                        plugin_content=time_series.summary_metadata.plugin_data.content,
+                        description=time_series.summary_metadata.summary_description,
+                        display_name=time_series.summary_metadata.display_name,
+                    )
+            return result
+
+    @timing.log_latency
+    def read_tensors(
+        self,
+        ctx,
+        *,
+        experiment_id,
+        plugin_name,
+        downsample=None,
+        run_tag_filter=None,
+    ):
+        with timing.log_latency("build request"):
+            req = data_provider_pb2.ReadTensorsRequest()
+            req.experiment_id = experiment_id
+            req.plugin_filter.plugin_name = plugin_name
+            _populate_rtf(run_tag_filter, req.run_tag_filter)
+            req.downsample.num_points = downsample
+        with timing.log_latency("_stub.ReadTensors"):
+            with _translate_grpc_error():
+                res = self._stub.ReadTensors(req)
+        with timing.log_latency("build result"):
+            result = {}
+            for run_entry in res.runs:
+                tags = {}
+                result[run_entry.run_name] = tags
+                for tag_entry in run_entry.tags:
+                    series = []
+                    tags[tag_entry.tag_name] = series
+                    d = tag_entry.data
+                    for step, wt, value in zip(d.step, d.wall_time, d.value):
+                        point = provider.TensorDatum(
+                            step=step,
+                            wall_time=wt,
+                            numpy=tensor_util.make_ndarray(value),
+                        )
+                        series.append(point)
+            return result
+
+    @timing.log_latency
+    def list_blob_sequences(
+        self, ctx, experiment_id, plugin_name, run_tag_filter=None
+    ):
+        with timing.log_latency("build request"):
+            req = data_provider_pb2.ListBlobSequencesRequest()
+            req.experiment_id = experiment_id
+            req.plugin_filter.plugin_name = plugin_name
+            _populate_rtf(run_tag_filter, req.run_tag_filter)
+        with timing.log_latency("_stub.ListBlobSequences"):
+            with _translate_grpc_error():
+                res = self._stub.ListBlobSequences(req)
+        with timing.log_latency("build result"):
+            result = {}
+            for run_entry in res.runs:
+                tags = {}
+                result[run_entry.run_name] = tags
+                for tag_entry in run_entry.tags:
+                    time_series = tag_entry.metadata
+                    tags[tag_entry.tag_name] = provider.BlobSequenceTimeSeries(
+                        max_step=time_series.max_step,
+                        max_wall_time=time_series.max_wall_time,
+                        max_length=time_series.max_length,
+                        plugin_content=time_series.summary_metadata.plugin_data.content,
+                        description=time_series.summary_metadata.summary_description,
+                        display_name=time_series.summary_metadata.display_name,
+                    )
+            return result
+
+    @timing.log_latency
+    def read_blob_sequences(
+        self,
+        ctx,
+        experiment_id,
+        plugin_name,
+        downsample=None,
+        run_tag_filter=None,
+    ):
+        with timing.log_latency("build request"):
+            req = data_provider_pb2.ReadBlobSequencesRequest()
+            req.experiment_id = experiment_id
+            req.plugin_filter.plugin_name = plugin_name
+            _populate_rtf(run_tag_filter, req.run_tag_filter)
+            req.downsample.num_points = downsample
+        with timing.log_latency("_stub.ReadBlobSequences"):
+            with _translate_grpc_error():
+                res = self._stub.ReadBlobSequences(req)
+        with timing.log_latency("build result"):
+            result = {}
+            for run_entry in res.runs:
+                tags = {}
+                result[run_entry.run_name] = tags
+                for tag_entry in run_entry.tags:
+                    series = []
+                    tags[tag_entry.tag_name] = series
+                    d = tag_entry.data
+                    for step, wt, blob_sequence in zip(
+                        d.step, d.wall_time, d.values
+                    ):
+                        values = []
+                        for ref in blob_sequence.blob_refs:
+                            values.append(
+                                provider.BlobReference(
+                                    blob_key=ref.blob_key, url=ref.url or None
+                                )
+                            )
+                        point = provider.BlobSequenceDatum(
+                            step=step, wall_time=wt, values=tuple(values)
+                        )
+                        series.append(point)
+            return result
+
+    @timing.log_latency
+    def read_blob(self, ctx, blob_key):
+        with timing.log_latency("build request"):
+            req = data_provider_pb2.ReadBlobRequest()
+            req.blob_key = blob_key
+        with timing.log_latency("list(_stub.ReadBlob)"):
+            with _translate_grpc_error():
+                responses = list(self._stub.ReadBlob(req))
+        with timing.log_latency("build result"):
+            return b"".join(res.data for res in responses)
+
+
+@contextlib.contextmanager
+def _translate_grpc_error():
+    try:
+        yield
+    except grpc.RpcError as e:
+        if e.code() == grpc.StatusCode.INVALID_ARGUMENT:
+            raise errors.InvalidArgumentError(e.details())
+        if e.code() == grpc.StatusCode.NOT_FOUND:
+            raise errors.NotFoundError(e.details())
+        if e.code() == grpc.StatusCode.PERMISSION_DENIED:
+            raise errors.PermissionDeniedError(e.details())
+        raise
+
+
+def _populate_rtf(run_tag_filter, rtf_proto):
+    """Copies `run_tag_filter` into `rtf_proto`."""
+    if run_tag_filter is None:
+        return
+    if run_tag_filter.runs is not None:
+        rtf_proto.runs.names[:] = sorted(run_tag_filter.runs)
+    if run_tag_filter.tags is not None:
+        rtf_proto.tags.names[:] = sorted(run_tag_filter.tags)
+
+
+def _timestamp_proto_to_float(ts):
+    """Converts `timestamp_pb2.Timestamp` to float seconds since epoch."""
+    return ts.ToNanoseconds() / 1e9

tensorbored/data/ingester.py

@@ -0,0 +1,46 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Abstraction for data ingestion logic."""
+
+import abc
+
+
+class DataIngester(metaclass=abc.ABCMeta):
+    """Link between a data source and a data provider.
+
+    A data ingester starts a reload operation in the background and
+    provides a data provider as a view.
+    """
+
+    @property
+    @abc.abstractmethod
+    def data_provider(self):
+        """Returns a `DataProvider`.
+
+        It may be an error to dereference this before `start` is called.
+        """
+        pass
+
+    @abc.abstractmethod
+    def start(self):
+        """Starts ingesting data.
+
+        This may start a background thread or process, and will return
+        once communication with that task is established. It won't block
+        forever as data is reloaded.
+
+        Must only be called once.
+        """
+        pass