ml-dash 0.0.11__py3-none-any.whl → 0.5.9__py3-none-any.whl

ml_dash/schema/files/images.py

@@ -1,27 +0,0 @@
-from os.path import split
-from graphene import ObjectType, relay, String
-from ml_dash import schema
-
-
-class File(ObjectType):
-    class Meta:
-        interfaces = relay.Node,
-
-    name = String(description='name of the directory')
-
-    # description = String(description='string serialized data')
-    # experiments = List(lambda: schema.Experiments)
-
-    @classmethod
-    def get_node(cls, info, id):
-        return get_file(id)
-
-
-class FileConnection(relay.Connection):
-    class Meta:
-        node = File
-
-
-def get_file(id):
-    # path = os.path.join(Args.logdir, id[1:])
-    return File(id=id, name=split(id[1:])[1])

ml_dash/schema/files/metrics.py

@@ -1,64 +0,0 @@
-from os.path import split, realpath, join
-from graphene import relay, ObjectType, String, List, JSONString, Int
-from graphene.types.generic import GenericScalar
-from graphql_relay import from_global_id
-from ml_dash.config import Args
-from ml_dash.schema.files.file_helpers import find_files, read_records, read_dataframe
-
-
-class Metrics(ObjectType):
-    class Meta:
-        interfaces = relay.Node,
-
-    path = String(description="path to the file")
-
-    def resolve_path(self, info):
-        return self.id
-
-    keys = List(String, description="list of keys for the metrics")
-
-    # value = List(GenericScalar, description="the raw value")
-    value = GenericScalar(description="The value of the metrics file",
-                          keys=List(String),
-                          k=Int(required=False),
-                          last=Int(required=False),
-                          window=Int(required=False))
-
-    def resolve_keys(self, info):
-        df = read_dataframe(join(Args.logdir, self.id[1:]))
-        keys = df.dropna().keys()
-        return list(keys)
-
-    # todo: add more complex queries.
-    def resolve_value(self, info, keys=None, k=None, last=None, window=None):
-        path = join(Args.logdir, self.id[1:])
-        realpath(path)
-        _ = read_dataframe(path)
-        if keys:
-            df = _[keys].dropna()
-            return {k: df[k].values.tolist() for k in keys}
-        else:
-            df = _.dropna()
-            return {k: v.values.tolist() for k, v in df.items()}
-
-    @classmethod
-    def get_node(cls, info, id):
-        return Metrics(id)
-
-
-class MetricsConnection(relay.Connection):
-    class Meta:
-        node = Metrics
-
-
-def get_metrics(id):
-    # note: this is where the key finding happens.
-    return Metrics(id=id)
-
-
-def find_metrics(cwd, **kwargs):
-    from ml_dash.config import Args
-    _cwd = realpath(join(Args.logdir, cwd[1:]))
-    parameter_files = find_files(_cwd, "**/metrics.pkl", **kwargs)
-    for p in parameter_files:
-        yield Metrics(id=join(cwd, p['path']))

ml_dash/schema/files/parameters.py

@@ -1,50 +0,0 @@
-from functools import reduce
-from os.path import split, join
-from graphene import ObjectType, relay, String, List
-from graphene.types.generic import GenericScalar
-from ml_dash import schema
-from ml_dash.config import Args
-from ml_dash.schema.files.file_helpers import read_json
-from ml_dash.schema.helpers import assign, dot_keys, dot_flatten
-
-
-class Parameters(ObjectType):
-    class Meta:
-        interfaces = relay.Node,
-
-    _path = String(description="The true path to the parameter file. Internal use only")
-    keys = List(String, description="list of parameter keys")
-    value = GenericScalar(description="the json value for the parameters")
-    raw = GenericScalar(description="the raw data object for the parameters")
-    flat = GenericScalar(description="the raw data object for the parameters")
-
-    def resolve_keys(self, info):
-        value = reduce(assign, read_json(join(Args.logdir, self.id[1:])) or [{}])
-        return dot_keys(value)
-
-    def resolve_value(self, info, **kwargs):
-        return reduce(assign, read_json(join(Args.logdir, self.id[1:])) or [{}])
-
-    def resolve_raw(self, info, **kwargs):
-        return read_json(join(Args.logdir, self.id[1:]))
-
-    def resolve_flat(self, info, **kwargs):
-        value = reduce(assign, read_json(join(Args.logdir, self.id[1:])) or [{}])
-        return dot_flatten(value)
-
-    # description = String(description='string serialized data')
-    # experiments = List(lambda: schema.Experiments)
-
-    @classmethod
-    def get_node(cls, info, id):
-        return get_parameters(id)
-
-
-class ParameterConnection(relay.Connection):
-    class Meta:
-        node = Parameters
-
-
-def get_parameters(id):
-    # path = os.path.join(Args.logdir, id[1:])
-    return Parameters(id=id)

ml_dash/schema/files/series.py

@@ -1,235 +0,0 @@
-from os.path import split, realpath, join, isabs
-from graphene import relay, ObjectType, String, List, JSONString, ID, Enum, Date, Time, DateTime, Int, Float, Union
-from graphene.types.generic import GenericScalar
-from ml_dash.config import Args
-from ml_dash.schema.files.file_helpers import find_files, read_records, read_dataframe
-import numpy as np
-
-
-class Series(ObjectType):
-    class Meta:
-        interfaces = relay.Node,
-
-    path = String(description="the file path for the configuration file")
-
-    prefix = String(description='path prefix for the metrics files')
-    metrics_files = List(ID, description="List of metrics file IDs that we use to aggregate this series")
-
-    _df = GenericScalar(description='the processed dataframe object that aggregates all metrics files.')
-
-    window = Float(description="the window for the rolling average")
-
-    label = String(description="the lable for the series")
-    x_key = String(description="key for the x")
-    y_key = String(description="key for the y axis")
-    y_keys = List(String, description="tuple of keys for the y axis")
-
-    # stype = SeriesTyes(description="the type of series data")
-
-    # resolved from dataset
-    x_data = GenericScalar(description="x data")
-    y_mean = GenericScalar(description="y data from the mean of the window")
-    # y_mode = GenericScalar(description="y data as from mode of the window")
-    y_median = GenericScalar(description="y data as from mode of the window")
-    y_min = GenericScalar(description="min in each bin")
-    y_max = GenericScalar(description="max in each bin")
-    y_25 = GenericScalar(description="quarter quantile")
-    y_75 = GenericScalar(description="3/4th quantile")
-    y_95 = GenericScalar(description="95th quantile")
-    y_05 = GenericScalar(description="5th quantile")
-    y_count = GenericScalar(description="the number of datapoints used to compute each tick")
-
-    # todo: start time
-
-    # todo: need to move the keys out, so that we can dropnan on the joint table.
-    #   Otherwise the different data columns would not necessarily be the same length.
-    def resolve_x_data(self, info):
-        # note: new in 0.24.1.
-        #  ~> df.value.dtype does NOT work for categorical data.
-        _ = self._df['__x'].to_numpy()
-        if np.issubdtype(_.dtype, np.datetime64):
-            return (_.astype(int) / 1000).tolist()
-        elif np.issubdtype(_.dtype, np.timedelta64):
-            return (_.astype(int) / 1000).tolist()
-        return _.tolist()
-
-    def resolve_y_mean(self, info):
-        if self.y_key is not None:
-            return self._df[self.y_key]['mean'].to_numpy().tolist()
-        return {k: self._df[k]['mean'].to_numpy().tolist() for k in self.y_keys}
-
-    # def resolve_y_mode(self, info):
-    #     if self.y_key is not None:
-    #         return self._df[self.y_key]['mode'].to_numpy().tolist()
-    #     return {k: self._df[k]['mode'].to_numpy().tolist() for k in self.y_keys}
-
-    def resolve_y_min(self, info):
-        if self.y_key is not None:
-            return self._df[self.y_key]['min'].to_numpy().tolist()
-        return {k: self._df[k]['min'].to_numpy().tolist() for k in self.y_keys}
-
-    def resolve_y_max(self, info):
-        if self.y_key is not None:
-            return self._df[self.y_key]['max'].to_numpy().tolist()
-        return {k: self._df[k]['max'].to_numpy().tolist() for k in self.y_keys}
-
-    def resolve_y_median(self, info):
-        if self.y_key is not None:
-            return self._df[self.y_key]['50%'].to_numpy().tolist()
-        return {k: self._df[k]['50%'].to_numpy().tolist() for k in self.y_keys}
-
-    def resolve_y_25(self, info):
-        if self.y_key is not None:
-            return self._df[self.y_key]['25%'].to_numpy().tolist()
-        return {k: self._df[k]['25%'].to_numpy().tolist() for k in self.y_keys}
-
-    def resolve_y_75(self, info):
-        if self.y_key is not None:
-            return self._df[self.y_key]['75%'].to_numpy().tolist()
-        return {k: self._df[k]['75%'].to_numpy().tolist() for k in self.y_keys}
-
-    def resolve_y_95(self, info):
-        if self.y_key is not None:
-            return self._df[self.y_key]['95%'].to_numpy().tolist()
-        return {k: self._df[k]['95%'].to_numpy().tolist() for k in self.y_keys}
-
-    def resolve_y_05(self, info):
-        if self.y_key is not None:
-            return self._df[self.y_key]['5%'].to_numpy().tolist()
-        return {k: self._df[k]['5%'].to_numpy().tolist() for k in self.y_keys}
-
-    def resolve_y_count(self, info):
-        if self.y_key is not None:
-            return self._df[self.y_key]['count'].to_numpy().tolist()
-        return {k: self._df[k]['count'].to_numpy().tolist() for k in self.y_keys}
-
-    @classmethod
-    def get_node(cls, info, id):
-        return Series(id)
-
-
-def get_series(metrics_files=tuple(),
-               prefix=None,
-               head=None,
-               tail=None,
-               x_low=None,
-               x_high=None,
-               x_edge=None,  # OneOf('start', 'after', 'mid', 'mode')
-               k=None,
-               x_align=None,  # OneOf(int, 'left', 'right')
-               x_key=None,
-               y_key=None,
-               y_keys=None,
-               label=None):
-    assert not y_key or not y_keys, "yKey and yKeys can not be trueful at the same time"
-    assert y_key or y_keys, "yKey and yKeys can not be both falseful."
-    assert head is None or tail is None, "head and tail can not be trueful at the same time"
-    if not prefix:
-        for id in metrics_files:
-            assert isabs(id), f"metricFile need to be absolute path is prefix is {prefix}. It is {id} instead."
-
-    ids = [join(prefix or "", id) for id in metrics_files]
-    dfs = [read_dataframe(join(Args.logdir, _id[1:])) for _id in ids]
-
-    y_keys = y_keys or [y_key]
-    join_keys = [x_key, *y_keys]
-    join_keys = list(set([k for k in join_keys if k is not None]))
-
-    joined = []
-    for df in dfs:
-        if df is None:
-            continue
-        if x_key is not None:
-            df.set_index(x_key)
-            if x_align is None:
-                pass
-            elif x_align == "start":
-                df[x_key] -= df[x_key][0]
-            elif x_align == "end":
-                df[x_key] -= df[x_key][-1]
-            else:
-                df[x_key] -= x_align
-        else:
-            df = df[y_keys]
-
-        if tail is not None:
-            df = df.tail(tail)
-        if head is not None:
-            df = df.head(head)
-        inds = True
-        if x_low is not None:
-            inds &= df.index >= x_low
-        if x_high is not None:
-            inds &= df.index <= x_high
-        if inds is not True:
-            df = df.loc[inds]
-
-        # todo: only dropna if we are not using ranges. <need to test>
-        try:
-            if head is None and tail is None:
-                joined.append(df[join_keys].dropna())
-            else:
-                joined.append(df[join_keys])
-        except KeyError as e:
-            raise KeyError(
-                f"{join_keys} contain keys that is not in the dataframe. "
-                f"Keys available include {df.keys()}") from e
-
-    if not joined:  # No dataframe, return `null`.
-        return None
-
-    import pandas as pd
-    all = pd.concat(joined)
-
-    if x_key:
-        all = all.set_index(x_key)
-
-    all.rank(method='first')
-
-    if k is not None:
-        bins = pd.qcut(all.index, k, duplicates='drop')
-        grouped = all.groupby(bins)
-    else:
-        grouped = all.groupby(level=0)
-
-    df = pd.merge(grouped[y_keys].agg(['count', 'mean', 'min', 'max']).reset_index(),
-                  grouped[y_keys].describe(percentiles=[0.25, 0.75, 0.5, 0.05, 0.95]))
-
-    if k is not None:
-        if x_edge == "right" or x_edge is None:
-            df['__x'] = df['index'].apply(lambda r: r.right)
-        elif x_edge == "left":
-            df['__x'] = df['index'].apply(lambda r: r.left)
-        elif x_edge == "mean":
-            df['__x'] = df['index'].apply(lambda r: 0.5 * (r.left + r.right))
-        # todo: use mode of each bin
-        else:
-            raise KeyError(f"x_edge {[x_edge]} should be OneOf['start', 'after', 'mid', 'mode']")
-    else:
-        df['__x'] = df.index
-
-    return Series(metrics_files,
-                  _df=df.sort_values(by="__x"),
-                  metrics_files=metrics_files,
-                  prefix=prefix,
-                  x_key=x_key,
-                  y_key=y_key,
-                  y_keys=y_keys,
-                  label=label,
-                  )
-
-
-SeriesArguments = dict(
-    metrics_files=List(String, required=True),
-    prefix=String(description="prefix to the metricFiles.", required=False),
-    head=Int(description="the number of datapoints (for each metrics file) to take from the head-end"),
-    tail=Int(description="the number of datapoints (for each metrics file) to take from the tail-end"),
-    x_low=Float(description="the (inclusive) lower end of the x column"),
-    x_high=Float(description="the (inclusive) higher end of the x column"),
-    k=Int(required=False, description='the number of datapoints to return.'),
-    x_align=String(description="a number (anchor point), 'start', 'end'"),
-    x_key=String(),
-    y_key=String(description="You can leave the xKey, but the yKey is required."),
-    y_keys=List(String, description="Alternatively you can pass a list of keys to yKey*s*."),
-    label=String(),
-)

ml_dash/schema/files/videos.py

@@ -1,27 +0,0 @@
-from os.path import split
-from graphene import ObjectType, relay, String
-from ml_dash import schema
-
-
-class File(ObjectType):
-    class Meta:
-        interfaces = relay.Node,
-
-    name = String(description='name of the directory')
-
-    # description = String(description='string serialized data')
-    # experiments = List(lambda: schema.Experiments)
-
-    @classmethod
-    def get_node(cls, info, id):
-        return get_file(id)
-
-
-class FileConnection(relay.Connection):
-    class Meta:
-        node = File
-
-
-def get_file(id):
-    # path = os.path.join(Args.logdir, id[1:])
-    return File(id=id, name=split(id[1:])[1])

ml_dash/schema/helpers.py

@@ -1,66 +0,0 @@
-from typing import List, Any
-
-
-def assign(d1, d2):
-    if not d2:
-        return d1
-    for k, v in d2.items():
-        if isinstance(d1.get(k, None), dict):
-            d1[k] = assign(d1[k], v)
-        else:
-            d1[k] = v
-    return d1
-
-
-if __name__ == "__main__":
-    object1 = {"a": 1, "b": 2, "c": 3}
-    object2 = assign({"c": 4, "d": 5}, object1)
-    assert object2['c'] == 3
-    assert object2['d'] == 5
-
-
-def idot_keys(d, strict=True):
-    for k, v in d.items():
-        if isinstance(v, dict):
-            if not strict:
-                yield k
-            for _ in idot_keys(v, strict):
-                yield k + "." + _
-        else:
-            yield k
-
-
-def dot_keys(d, strict=True):
-    return [*idot_keys(d, strict)]
-
-
-if __name__ == "__main__":
-    object = {"a": 1, "b": 2, "c": 3, "child": {"a": 3, "grandchild": {'d': 8}}}
-    assert dot_keys(object) == ['a', 'b', 'c', 'child.a', 'child.grandchild.d']
-    assert dot_keys(object, strict=False) == ['a', 'b', 'c', 'child', 'child.a', 'child.grandchild',
-                                              'child.grandchild.d']
-
-
-def idot_flatten(d, ancestors: List[Any] = tuple()):
-    """
-    returns a flattened dictionary with the keys of the nexted dictionaries converted into dot-separated keys.
-
-    :param d: map
-    :return: flat map
-    """
-    for k, v in d.items():
-        if isinstance(v, dict):
-            for _k, _v in idot_flatten(v):
-                yield k + "." + _k, _v
-        else:
-            yield k, v
-
-
-def dot_flatten(d):
-    # note: dictionaries are ordered by default in python 3.7.
-    return dict(idot_flatten(d))
-
-
-if __name__ == "__main__":
-    object = {"a": 1, "b": 2, "c": 3, "child": {"a": 3, "grandchild": {'d': 8}}}
-    assert list(dot_flatten(object).keys()) == ['a', 'b', 'c', 'child.a', 'child.grandchild.d']

ml_dash/schema/projects.py

@@ -1,65 +0,0 @@
-from os import listdir
-from os.path import isfile, join, split
-
-from graphene import ObjectType, relay, String, List
-from ml_dash import schema
-
-
-class Project(ObjectType):
-    class Meta:
-        interfaces = relay.Node,
-
-    name = String(description='name of the project')
-
-    # description = String(description='string serialized data')
-    # experiments = List(lambda: schema.Experiments)
-
-    experiments = relay.ConnectionField(lambda: schema.experiments.ExperimentConnection)
-
-    def resolve_experiments(self, info, **kwargs):
-        return schema.experiments.find_experiments(cwd=self.id)
-
-    directories = relay.ConnectionField(lambda: schema.directories.DirectoryConnection)
-    files = relay.ConnectionField(lambda: schema.files.FileConnection)
-
-    def resolve_directories(self, info, **kwargs):
-        from ml_dash.config import Args
-        root_dir = join(Args.logdir, self.id[1:])
-        return [schema.Directory(id=join(self.id, _), name=_)
-                for _ in listdir(root_dir) if not isfile(join(root_dir, _))]
-
-    def resolve_files(self, info, **kwargs):
-        from ml_dash.config import Args
-        root_dir = join(Args.logdir, self.id[1:])
-        return [schema.Directory(id=join(self.id, _), name=_)
-                for _ in listdir(root_dir) if isfile(join(root_dir, _))]
-
-    # def resolve_experiments(self, info, **kargs):
-    #     from ml_dash.config import Args
-    #     root_dir = join(Args.logdir, self.id[1:])
-    #
-    #     return [schema.Directory(id=join(self.id, _), name=_)
-    #             for _ in listdir(root_dir) if isfile(join(root_dir, _))]
-
-    @classmethod
-    def get_node(cls, info, id):
-        return get_project(id)
-
-
-class ProjectConnection(relay.Connection):
-    class Meta:
-        node = Project
-
-
-def get_projects(username):
-    import os
-    from ml_dash.config import Args
-    user_root = join(Args.logdir, username)
-    return [Project(name=_, id=join('/', username, _))
-            for _ in os.listdir(user_root) if not isfile(_)]
-
-
-def get_project(id):
-    from ml_dash.config import Args
-    path = join(Args.logdir, id[1:])
-    return Project(id=id, name=split(id[1:])[1], _path=path)

ml_dash/schema/schema_helpers.py

@@ -1,19 +0,0 @@
-def bind(fn):
-    """
-    Binds the function to the class.
-
-    :param fn:
-    :return: bound_fn
-    """
-    return lambda _, *args, **kwargs: fn(*args, **kwargs)
-
-
-def bind_args(fn):
-    """
-    Binds args after info.
-
-    :param fn:
-    :return: bound_fn
-    """
-    return lambda _, info, *args, **kwargs: fn(*args, **kwargs)
-

ml_dash/schema/users.py

@@ -1,33 +0,0 @@
-from os.path import isfile
-from graphene import ObjectType, relay, String
-from ml_dash import schema
-
-
-class User(ObjectType):
-    class Meta:
-        interfaces = relay.Node,
-
-    @classmethod
-    def get_node(_, info, id):
-        print(info, id)
-        return get_user(id)
-
-    username = String(description='string serialized data')
-    name = String(description='string serialized data')
-
-    projects = relay.ConnectionField(lambda: schema.projects.ProjectConnection)
-
-    def resolve_projects(self, info, **kwargs):
-        return schema.projects.get_projects(self.username)
-
-    # teams = List(lambda: schema.Team)
-
-
-def get_users(ids=None):
-    import os
-    from ml_dash.config import Args
-    return [User(username=_, name="Ge Yang") for _ in os.listdir(Args.logdir) if not isfile(_)]
-
-
-def get_user(username):
-    return User(username=username, name="Ge Yang", id=username)

ml_dash/sse.py

@@ -1,18 +0,0 @@
-# SSE "protocol" is described here: http://mzl.la/UPFyxY
-class ServerSentEvent(object):
-
-    def __init__(self, data):
-        self.data = data
-        self.event = None
-        self.id = None
-        self.desc_map = {
-            self.data: "data",
-            self.event: "event",
-            self.id: "id"
-        }
-
-    def __str__(self):
-        if not self.data:
-            return ""
-        lines = [f"{v}: {k}" for k, v in self.desc_map.items() if k]
-        return "%s\n\n" % "\n".join(lines)

ml_dash-0.0.11.dist-info/METADATA

@@ -1,67 +0,0 @@
-Metadata-Version: 2.1
-Name: ml-dash
-Version: 0.0.11
-Summary: A Beautiful Visualization Dashboard For Machine Learning
-Home-page: https://github.com/episodeyang/ml_dash
-Author: Ge Yang
-Author-email: yangge1987@gmail.com
-License: UNKNOWN
-Keywords: ml_logger,ml-logger,ml dash,ml-dash,ml_dashdashboard,machine learning,vis_server,logging,debug,debugging
-Platform: UNKNOWN
-Classifier: Development Status :: 4 - Beta
-Classifier: Intended Audience :: Science/Research
-Classifier: Programming Language :: Python :: 3
-Requires-Dist: typing
-Requires-Dist: numpy
-Requires-Dist: termcolor
-Requires-Dist: params-proto
-Requires-Dist: cloudpickle
-Requires-Dist: uvloop (==0.8.1)
-Requires-Dist: requests
-Requires-Dist: requests-futures
-Requires-Dist: hachiko
-Requires-Dist: sanic
-Requires-Dist: sanic-cors
-Requires-Dist: sanic-graphql
-Requires-Dist: graphene
-Requires-Dist: graphql-core
-Requires-Dist: graphql-relay
-Requires-Dist: dill
-Requires-Dist: ruamel.yaml
-
-ML-Dash, A Beautiful Visualization Dashboard for Machine Learning
-=================================================================
-
-`Downloads <http://pepy.tech/project/ml-dash>`__
-
-ML-dash replaces visdom and tensorboard. It is the single real-time job
-visualization dashboard for machine learning.
-
-**Parallel Coordinates** **Aggregating Over Multiple Runs** **Create
-Movies out of images**
-
-Usage
------
-
-To **install** ``ml_dash``, do:
-
-.. code-block:: bash
-
-   pip install ml-dash
-
-**Note: the server accepts requests from ``localhost`` only, by
-default.** In order to
-
-.. code-block:: bash
-
-   python -m ml_dash.main --log-dir=<your-log-dir> --host=0.0.0.0 --port=<your-port-number> --workers=4
-
-It is the easiest if you setup a long-lived instrument server with a
-public ip for yourself or the entire lab.
-
-Implementation Notes
-~~~~~~~~~~~~~~~~~~~~
-
-See `https://github.com/episodeyang/ml_dash/blob/master/notes/README.md <https://github.com/episodeyang/ml_dash/blob/master/notes/README.md>`__
-
-