tensorbored 2.21.0rc1769983804__py3-none-any.whl

tensorbored/plugins/hparams/metadata.py

@@ -0,0 +1,125 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Constants used in the HParams plugin."""
+
+from tensorbored.compat.proto import summary_pb2
+from tensorbored.compat.proto import types_pb2
+from tensorbored.plugins.hparams import error
+from tensorbored.plugins.hparams import plugin_data_pb2
+from tensorbored.util import tensor_util
+
+PLUGIN_NAME = "hparams"
+PLUGIN_DATA_VERSION = 0
+
+# Tensor value for use in summaries that really only need to store
+# metadata. A length-0 float vector is of minimal serialized length
+# (6 bytes) among valid tensors. Cache this: computing it takes on the
+# order of tens of microseconds.
+NULL_TENSOR = tensor_util.make_tensor_proto(
+    [], dtype=types_pb2.DT_FLOAT, shape=(0,)
+)
+
+EXPERIMENT_TAG = "_hparams_/experiment"
+SESSION_START_INFO_TAG = "_hparams_/session_start_info"
+SESSION_END_INFO_TAG = "_hparams_/session_end_info"
+
+
+def create_summary_metadata(hparams_plugin_data_pb):
+    """Returns a summary metadata for the HParams plugin.
+
+    Returns a summary_pb2.SummaryMetadata holding a copy of the given
+    HParamsPluginData message in its plugin_data.content field.
+    Sets the version field of the hparams_plugin_data_pb copy to
+    PLUGIN_DATA_VERSION.
+
+    Args:
+      hparams_plugin_data_pb: the HParamsPluginData protobuffer to use.
+    """
+    if not isinstance(
+        hparams_plugin_data_pb, plugin_data_pb2.HParamsPluginData
+    ):
+        raise TypeError(
+            "Needed an instance of plugin_data_pb2.HParamsPluginData."
+            " Got: %s" % type(hparams_plugin_data_pb)
+        )
+    content = plugin_data_pb2.HParamsPluginData()
+    content.CopyFrom(hparams_plugin_data_pb)
+    content.version = PLUGIN_DATA_VERSION
+    return summary_pb2.SummaryMetadata(
+        plugin_data=summary_pb2.SummaryMetadata.PluginData(
+            plugin_name=PLUGIN_NAME, content=content.SerializeToString()
+        )
+    )
+
+
+def parse_experiment_plugin_data(content):
+    """Returns the experiment from HParam's
+    SummaryMetadata.plugin_data.content.
+
+    Raises HParamsError if the content doesn't have 'experiment' set or
+    this file is incompatible with the version of the metadata stored.
+
+    Args:
+      content: The SummaryMetadata.plugin_data.content to use.
+    """
+    return _parse_plugin_data_as(content, "experiment")
+
+
+def parse_session_start_info_plugin_data(content):
+    """Returns session_start_info from the plugin_data.content.
+
+    Raises HParamsError if the content doesn't have 'session_start_info' set or
+    this file is incompatible with the version of the metadata stored.
+
+    Args:
+      content: The SummaryMetadata.plugin_data.content to use.
+    """
+    return _parse_plugin_data_as(content, "session_start_info")
+
+
+def parse_session_end_info_plugin_data(content):
+    """Returns session_end_info from the plugin_data.content.
+
+    Raises HParamsError if the content doesn't have 'session_end_info' set or
+    this file is incompatible with the version of the metadata stored.
+
+    Args:
+      content: The SummaryMetadata.plugin_data.content to use.
+    """
+    return _parse_plugin_data_as(content, "session_end_info")
+
+
+def _parse_plugin_data_as(content, data_oneof_field):
+    """Returns a data oneof's field from plugin_data.content.
+
+    Raises HParamsError if the content doesn't have 'data_oneof_field' set or
+    this file is incompatible with the version of the metadata stored.
+
+    Args:
+      content: The SummaryMetadata.plugin_data.content to use.
+      data_oneof_field: string. The name of the data oneof field to return.
+    """
+    plugin_data = plugin_data_pb2.HParamsPluginData.FromString(content)
+    if plugin_data.version != PLUGIN_DATA_VERSION:
+        raise error.HParamsError(
+            "Only supports plugin_data version: %s; found: %s in: %s"
+            % (PLUGIN_DATA_VERSION, plugin_data.version, plugin_data)
+        )
+    if not plugin_data.HasField(data_oneof_field):
+        raise error.HParamsError(
+            "Expected plugin_data.%s to be set. Got: %s"
+            % (data_oneof_field, plugin_data)
+        )
+    return getattr(plugin_data, data_oneof_field)

tensorbored/plugins/hparams/metrics.py

@@ -0,0 +1,41 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Functions for dealing with metrics."""
+
+import os
+
+
+from tensorbored.plugins.hparams import api_pb2
+
+
+def run_tag_from_session_and_metric(session_name, metric_name):
+    """Returns a (run,tag) tuple storing the evaluations of the specified
+    metric.
+
+    Args:
+      session_name: str.
+      metric_name: MetricName protobuffer.
+    Returns: (run, tag) tuple.
+    """
+    assert isinstance(session_name, str)
+    assert isinstance(metric_name, api_pb2.MetricName)
+    # os.path.join() will append a final slash if the group is empty; it seems
+    # like multiplexer.Tensors won't recognize paths that end with a '/' so
+    # we remove the final '/' in that case.
+    run = os.path.join(session_name, metric_name.group)
+    if run.endswith("/"):
+        run = run[:-1]
+    tag = metric_name.tag
+    return run, tag

tensorbored/plugins/hparams/plugin_data_pb2.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: tensorbored/plugins/hparams/plugin_data.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from tensorbored.plugins.hparams import api_pb2 as tensorbored_dot_plugins_dot_hparams_dot_api__pb2
+from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n-tensorbored/plugins/hparams/plugin_data.proto\x12\x13tensorbored.hparams\x1a%tensorbored/plugins/hparams/api.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xe9\x01\n\x11HParamsPluginData\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x35\n\nexperiment\x18\x02 \x01(\x0b\x32\x1f.tensorbored.hparams.ExperimentH\x00\x12\x43\n\x12session_start_info\x18\x03 \x01(\x0b\x32%.tensorbored.hparams.SessionStartInfoH\x00\x12?\n\x10session_end_info\x18\x04 \x01(\x0b\x32#.tensorbored.hparams.SessionEndInfoH\x00\x42\x06\n\x04\x64\x61ta\"\xf4\x01\n\x10SessionStartInfo\x12\x43\n\x07hparams\x18\x01 \x03(\x0b\x32\x32.tensorbored.hparams.SessionStartInfo.HparamsEntry\x12\x11\n\tmodel_uri\x18\x02 \x01(\t\x12\x13\n\x0bmonitor_url\x18\x03 \x01(\t\x12\x12\n\ngroup_name\x18\x04 \x01(\t\x12\x17\n\x0fstart_time_secs\x18\x05 \x01(\x01\x1a\x46\n\x0cHparamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01\"T\n\x0eSessionEndInfo\x12+\n\x06status\x18\x01 \x01(\x0e\x32\x1b.tensorbored.hparams.Status\x12\x15\n\rend_time_secs\x18\x02 \x01(\x01\x62\x06proto3')
+
+
+
+_HPARAMSPLUGINDATA = DESCRIPTOR.message_types_by_name['HParamsPluginData']
+_SESSIONSTARTINFO = DESCRIPTOR.message_types_by_name['SessionStartInfo']
+_SESSIONSTARTINFO_HPARAMSENTRY = _SESSIONSTARTINFO.nested_types_by_name['HparamsEntry']
+_SESSIONENDINFO = DESCRIPTOR.message_types_by_name['SessionEndInfo']
+HParamsPluginData = _reflection.GeneratedProtocolMessageType('HParamsPluginData', (_message.Message,), {
+  'DESCRIPTOR' : _HPARAMSPLUGINDATA,
+  '__module__' : 'tensorbored.plugins.hparams.plugin_data_pb2'
+  # @@protoc_insertion_point(class_scope:tensorbored.hparams.HParamsPluginData)
+  })
+_sym_db.RegisterMessage(HParamsPluginData)
+
+SessionStartInfo = _reflection.GeneratedProtocolMessageType('SessionStartInfo', (_message.Message,), {
+
+  'HparamsEntry' : _reflection.GeneratedProtocolMessageType('HparamsEntry', (_message.Message,), {
+    'DESCRIPTOR' : _SESSIONSTARTINFO_HPARAMSENTRY,
+    '__module__' : 'tensorbored.plugins.hparams.plugin_data_pb2'
+    # @@protoc_insertion_point(class_scope:tensorbored.hparams.SessionStartInfo.HparamsEntry)
+    })
+  ,
+  'DESCRIPTOR' : _SESSIONSTARTINFO,
+  '__module__' : 'tensorbored.plugins.hparams.plugin_data_pb2'
+  # @@protoc_insertion_point(class_scope:tensorbored.hparams.SessionStartInfo)
+  })
+_sym_db.RegisterMessage(SessionStartInfo)
+_sym_db.RegisterMessage(SessionStartInfo.HparamsEntry)
+
+SessionEndInfo = _reflection.GeneratedProtocolMessageType('SessionEndInfo', (_message.Message,), {
+  'DESCRIPTOR' : _SESSIONENDINFO,
+  '__module__' : 'tensorbored.plugins.hparams.plugin_data_pb2'
+  # @@protoc_insertion_point(class_scope:tensorbored.hparams.SessionEndInfo)
+  })
+_sym_db.RegisterMessage(SessionEndInfo)
+
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+  DESCRIPTOR._options = None
+  _SESSIONSTARTINFO_HPARAMSENTRY._options = None
+  _SESSIONSTARTINFO_HPARAMSENTRY._serialized_options = b'8\001'
+  _HPARAMSPLUGINDATA._serialized_start=140
+  _HPARAMSPLUGINDATA._serialized_end=373
+  _SESSIONSTARTINFO._serialized_start=376
+  _SESSIONSTARTINFO._serialized_end=620
+  _SESSIONSTARTINFO_HPARAMSENTRY._serialized_start=550
+  _SESSIONSTARTINFO_HPARAMSENTRY._serialized_end=620
+  _SESSIONENDINFO._serialized_start=622
+  _SESSIONENDINFO._serialized_end=706
+# @@protoc_insertion_point(module_scope)

tensorbored/plugins/hparams/summary.py

@@ -0,0 +1,205 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Summary creation methods for the HParams plugin.
+
+Typical usage for exporting summaries in a hyperparameters-tuning experiment:
+1. Create the experiment (once) by calling experiment_pb() and exporting
+   the resulting summary into a top-level (empty) run.
+2. In each training session in the experiment, call session_start_pb() before
+   the session starts, exporting the resulting summary into a uniquely named
+   run for the session, say <session_name>.
+3. Train the model in the session, exporting each metric as a scalar summary
+   in runs of the form <session_name>/<sub_dir>, where <sub_dir> can be empty a
+   (in which case the run is just the <session_name>) and depends on the
+   metric. The name of such a metric is a (group, tag) pair given by
+   (<sub_dir>, tag) where tag is the tag of the scalar summary.
+   When calling experiment_pb in step 1, you'll need to pass all the metric
+   names used in the experiemnt.
+4. When the session completes, call session_end_pb() and export the resulting
+   summary into the same session run <session_name>.
+"""
+
+import time
+
+
+import tensorflow as tf
+
+from tensorbored.plugins.hparams import api_pb2
+from tensorbored.plugins.hparams import metadata
+from tensorbored.plugins.hparams import plugin_data_pb2
+
+
+def experiment_pb(
+    hparam_infos, metric_infos, user="", description="", time_created_secs=None
+):
+    """Creates a summary that defines a hyperparameter-tuning experiment.
+
+    Args:
+      hparam_infos: Array of api_pb2.HParamInfo messages. Describes the
+          hyperparameters used in the experiment.
+      metric_infos: Array of api_pb2.MetricInfo messages. Describes the metrics
+          used in the experiment. See the documentation at the top of this file
+          for how to populate this.
+      user: String. An id for the user running the experiment
+      description: String. A description for the experiment. May contain markdown.
+      time_created_secs: float. The time the experiment is created in seconds
+      since the UNIX epoch. If None uses the current time.
+
+    Returns:
+      A summary protobuffer containing the experiment definition.
+    """
+    if time_created_secs is None:
+        time_created_secs = time.time()
+    experiment = api_pb2.Experiment(
+        description=description,
+        user=user,
+        time_created_secs=time_created_secs,
+        hparam_infos=hparam_infos,
+        metric_infos=metric_infos,
+    )
+    return _summary(
+        metadata.EXPERIMENT_TAG,
+        plugin_data_pb2.HParamsPluginData(experiment=experiment),
+    )
+
+
+def session_start_pb(
+    hparams, model_uri="", monitor_url="", group_name="", start_time_secs=None
+):
+    """Constructs a SessionStartInfo protobuffer.
+
+    Creates a summary that contains a training session metadata information.
+    One such summary per training session should be created. Each should have
+    a different run.
+
+    Args:
+      hparams: A dictionary with string keys. Describes the hyperparameter values
+               used in the session, mapping each hyperparameter name to its value.
+               Supported value types are  `bool`, `int`, `float`, `str`, `list`,
+               `tuple`.
+               The type of value must correspond to the type of hyperparameter
+               (defined in the corresponding api_pb2.HParamInfo member of the
+               Experiment protobuf) as follows:
+
+                +-----------------+---------------------------------+
+                |Hyperparameter   | Allowed (Python) value types    |
+                |type             |                                 |
+                +-----------------+---------------------------------+
+                |DATA_TYPE_BOOL   | bool                            |
+                |DATA_TYPE_FLOAT64| int, float                      |
+                |DATA_TYPE_STRING | str, tuple, list   |
+                +-----------------+---------------------------------+
+
+               Tuple and list instances will be converted to their string
+               representation.
+      model_uri: See the comment for the field with the same name of
+                 plugin_data_pb2.SessionStartInfo.
+      monitor_url: See the comment for the field with the same name of
+                   plugin_data_pb2.SessionStartInfo.
+      group_name:  See the comment for the field with the same name of
+                   plugin_data_pb2.SessionStartInfo.
+      start_time_secs: float. The time to use as the session start time.
+                       Represented as seconds since the UNIX epoch. If None uses
+                       the current time.
+    Returns:
+      The summary protobuffer mentioned above.
+    """
+    if start_time_secs is None:
+        start_time_secs = time.time()
+    session_start_info = plugin_data_pb2.SessionStartInfo(
+        model_uri=model_uri,
+        monitor_url=monitor_url,
+        group_name=group_name,
+        start_time_secs=start_time_secs,
+    )
+    for hp_name, hp_val in hparams.items():
+        # Boolean typed values need to be checked before integers since in Python
+        # isinstance(True/False, int) returns True.
+        if isinstance(hp_val, bool):
+            session_start_info.hparams[hp_name].bool_value = hp_val
+        elif isinstance(hp_val, (float, int)):
+            session_start_info.hparams[hp_name].number_value = hp_val
+        elif isinstance(hp_val, str):
+            session_start_info.hparams[hp_name].string_value = hp_val
+        elif isinstance(hp_val, (list, tuple)):
+            session_start_info.hparams[hp_name].string_value = str(hp_val)
+        else:
+            raise TypeError(
+                "hparams[%s]=%s has type: %s which is not supported"
+                % (hp_name, hp_val, type(hp_val))
+            )
+    return _summary(
+        metadata.SESSION_START_INFO_TAG,
+        plugin_data_pb2.HParamsPluginData(
+            session_start_info=session_start_info
+        ),
+    )
+
+
+def session_end_pb(status, end_time_secs=None):
+    """Constructs a SessionEndInfo protobuffer.
+
+    Creates a summary that contains status information for a completed
+    training session. Should be exported after the training session is completed.
+    One such summary per training session should be created. Each should have
+    a different run.
+
+    Args:
+      status: A tensorboard.hparams.Status enumeration value denoting the
+          status of the session.
+      end_time_secs: float. The time to use as the session end time. Represented
+          as seconds since the unix epoch. If None uses the current time.
+
+    Returns:
+      The summary protobuffer mentioned above.
+    """
+    if end_time_secs is None:
+        end_time_secs = time.time()
+
+    session_end_info = plugin_data_pb2.SessionEndInfo(
+        status=status, end_time_secs=end_time_secs
+    )
+    return _summary(
+        metadata.SESSION_END_INFO_TAG,
+        plugin_data_pb2.HParamsPluginData(session_end_info=session_end_info),
+    )
+
+
+def _summary(tag, hparams_plugin_data):
+    """Returns a summary holding the given HParamsPluginData message.
+
+    Helper function.
+
+    Args:
+      tag: string. The tag to use.
+      hparams_plugin_data: The HParamsPluginData message to use.
+    """
+    summary = tf.compat.v1.Summary()
+    tb_metadata = metadata.create_summary_metadata(hparams_plugin_data)
+    raw_metadata = tb_metadata.SerializeToString()
+    tf_metadata = tf.compat.v1.SummaryMetadata.FromString(raw_metadata)
+    summary.value.add(
+        tag=tag,
+        metadata=tf_metadata,
+        tensor=_TF_NULL_TENSOR,
+    )
+    return summary
+
+
+# Like `metadata.NULL_TENSOR`, but with the TensorFlow version of the
+# proto. Slight kludge needed to expose the `TensorProto` type.
+_TF_NULL_TENSOR = type(tf.make_tensor_proto(0)).FromString(
+    metadata.NULL_TENSOR.SerializeToString()
+)