ml-dash 0.0.11__py3-none-any.whl → 0.5.9__py3-none-any.whl

ml_dash/schema/__init__.py

@@ -1,110 +0,0 @@
-from graphene import relay, ObjectType, Float, Schema, List, String, Field
-from ml_dash.schema.files.series import Series, get_series, SeriesArguments
-from ml_dash.schema.files.metrics import Metrics, get_metrics
-from ml_dash.schema.schema_helpers import bind, bind_args
-from ml_dash.schema.users import User, get_users, get_user
-from ml_dash.schema.projects import Project
-from ml_dash.schema.directories import Directory, get_directory
-from ml_dash.schema.files import File, FileConnection, MutateTextFile, MutateJSONFile, MutateYamlFile, \
-    DeleteFile, DeleteDirectory, glob_files
-# MutateJSONFile, MutateYamlFile
-from ml_dash.schema.experiments import Experiment
-
-
-# class Experiment(graphene.ObjectType):
-#     class Meta:
-#         interfaces = relay.Node,
-#
-#     parameter_keys = graphene.List(description="keys in the parameter file")
-#     metric_keys = graphene.List(description="the x data")
-#     video_keys = graphene.List(description="the x data")
-#     img_keys = graphene.List(description="the x data")
-#     diff_keys = graphene.List(description="the x data")
-#     log_keys = graphene.List(description="the x data")
-#     view_config = ""
-#
-# class TimeSeries(graphene.ObjectType):
-#     class Meta:
-#         interfaces = relay.Node,
-#
-#     x_data = graphene.List(description="the x data")
-#     y_data = graphene.List(description="the y data")
-#     serialized = graphene.String(description='string serialized data')
-#
-#
-# class TimeSeriesWithStd(graphene.ObjectType):
-#     class Meta:
-#         interfaces = relay.Node,
-#
-#     x_data = graphene.List(description="the x data")
-#     y_data = graphene.List(description="the y data")
-#     std_data = graphene.List(description="the standard deviation data")
-#     quantile_25_data = graphene.List(description="the standard deviation data")
-#     quantile_50_data = graphene.List(description="the standard deviation data")
-#     quantile_75_data = graphene.List(description="the standard deviation data")
-#     quantile_100_data = graphene.List(description="the standard deviation data")
-#     mode_data = graphene.List(description="the standard deviation data")
-#     mean_data = graphene.List(description="the standard deviation data")
-#     serialized = graphene.String(description='string serialized data')
-#
-#
-# class LineChart(graphene.ObjectType):
-#     class Meta:
-#         interfaces = relay.Node,
-#
-#     key = graphene.String(description="The path to the metrics file (including metrics.pkl)")
-#     x_key = graphene.String(description="key for the x axis")
-#     x_label = graphene.String(description="label for the x axis")
-#     y_key = graphene.String(description="key for the y axis")
-#     y_label = graphene.String(description="label for the x axis")
-
-
-class EditText(relay.ClientIDMutation):
-    class Input:
-        text = String(required=True, description='updated content for the text file')
-
-    text = String(description="the updated content for the text file")
-
-    @classmethod
-    def mutate_and_get_payload(cls, root, info, text, ):
-        return dict(text=text)
-
-
-class Query(ObjectType):
-    node = relay.Node.Field()
-    # context?
-    # todo: files
-    # todo: series
-
-    users = Field(List(User), resolver=bind_args(get_users))
-    user = Field(User, username=String(), resolver=bind_args(get_user))
-    series = Field(Series, resolver=bind_args(get_series), **SeriesArguments)
-
-    project = relay.Node.Field(Project)
-    metrics = relay.Node.Field(Metrics)
-    directory = relay.Node.Field(Directory)
-
-    glob = Field(List(File),
-                            cwd=String(required=True),
-                            query=String(),
-                            resolver=bind_args(glob_files))
-
-
-class Mutation(ObjectType):
-    # todo: create_file
-    # done: edit_file
-    # done: remove_file
-    # todo: move_file
-    # todo: copy_file
-
-    # do we need to have separate deleteDirectory? (look up relay client-side macros)
-
-    delete_file = DeleteFile.Field()
-    delete_directory = DeleteDirectory.Field()
-    # update_text = EditText.Field()
-    update_text = MutateTextFile.Field()
-    update_json = MutateJSONFile.Field()
-    update_yaml = MutateYamlFile.Field()
-
-
-schema = Schema(query=Query, mutation=Mutation)

ml_dash/schema/archive.py

@@ -1,165 +0,0 @@
-from graphene import relay, ObjectType, Float, Schema, AbstractType, List, String, Union, Field
-
-
-# class Team(ObjectType):
-#     class Meta:
-#         interfaces = relay.Node,
-#
-#     name = String(description='string serialized data')
-#     description = String(description='string serialized data')
-#     users = List(lambda: User)
-#     projects = List(lambda: Project)
-#
-#
-# class Binder(ObjectType):
-#     class Meta:
-#         interfaces = relay.Node,
-#
-#     name = String(description="keys in the parameter file")
-#     children = List("Binder", description="child binders")
-#     files = List(lambda: FileAndDirectory, description="keys in the parameter file")
-#     # date_created
-#     # date_modified
-
-
-class Directory(ObjectType):
-    class Meta:
-        interfaces = relay.Node,
-
-    name = String(description='string serialized data')
-    description = String(description='string serialized data')
-    children = List(lambda: FileAndDirectory)
-
-
-# File Types
-class File(AbstractType):
-    class Meta:
-        interfaces = relay.Node,
-
-    name = String(description='string serialized data')
-    description = String(description='string serialized data')
-
-
-class Parameters(ObjectType, File):
-    pass
-
-
-class Metrics(ObjectType, File):
-    """this is just the file type."""
-    pass
-
-
-class Image(ObjectType, File):
-    pass
-
-
-class Video(ObjectType, File):
-    pass
-
-
-class TextFile(ObjectType, File):
-    pass
-
-
-class BinaryFile(ObjectType, File):
-    pass
-
-
-class JsonFile(ObjectType, File):
-    pass
-
-
-class YamlFile(ObjectType, File):
-    pass
-
-
-class FileAndDirectory(Union):
-    class Meta:
-        types = (Directory, File, Parameters, Metrics, TextFile, BinaryFile, JsonFile)
-
-
-class ExperimentView(ObjectType):
-    class Meta:
-        interfaces = relay.Node,
-
-    charts = List(lambda: Chart)
-
-
-class Point(ObjectType):
-    class Meta:
-        interfaces = relay.Node,
-
-    x = Float()
-    y = Float()
-
-
-class LineSeries(ObjectType):
-    class Meta:
-        interfaces = relay.Node,
-        description = "simple line series"
-
-    data = List(lambda: Point)
-    label = String(description="label for the time series")
-
-
-class StdSeries(ObjectType):
-    class Meta:
-        interfaces = relay.Node,
-        description = "line series with standard deviation"
-
-    data = List(lambda: Point)
-    label = String(description="label for the series")
-
-
-class QuantileSeries(ObjectType):
-    class Meta:
-        interfaces = relay.Node,
-        description = "line series with quantile ticks"
-
-    data = List(lambda: Point)
-    label = String(description="label for the time series")
-
-
-class Series(Union):
-    class Meta:
-        types = (LineSeries, StdSeries, QuantileSeries)
-
-
-class LineChart(ObjectType):
-    class Meta:
-        interfaces = relay.Node,
-
-    time_series = List(lambda: Series)
-
-    title = String(description="title for the chart")
-    x_label = String(description="label for the x axis")
-    y_label = String(description="label for the y axis")
-    x_low = String(description="lower range for the x axis")
-    x_high = String(description="higher range for x axis")
-    y_low = String(description="lower range for y axis")
-    y_high = String(description="higher range for y axis")
-
-
-class ParameterDomain(ObjectType):
-    class Meta:
-        interfaces = relay.Node,
-
-    name = String()
-    domain_low = Float()
-    domain_high = Float()
-
-
-class ParallelCoordinates(ObjectType):
-    class Meta:
-        interfaces = relay.Node,
-
-    # todo: {key, value}
-    data = List(ObjectType)
-    domains = List(ParameterDomain)
-
-
-class Chart(Union):
-    """this is also a file type."""
-
-    class Meta:
-        types = LineChart, ParallelCoordinates

ml_dash/schema/directories.py

@@ -1,59 +0,0 @@
-from os import listdir
-from os.path import isfile, join, split
-from graphene import ObjectType, relay, String, Field
-from ml_dash import schema
-
-
-class Directory(ObjectType):
-    class Meta:
-        interfaces = relay.Node,
-
-    name = String(description='name of the directory')
-    path = String(description='absolute path of the directory')
-
-    readme = Field(lambda: schema.files.File)
-
-    def resolve_readme(self, info, *args, **kwargs):
-        # note: keep it simple, just use README for now.
-        readmes = schema.files.find_files_by_query(cwd=self.id, query="README.md")
-        return readmes[0] if readmes else None
-
-    dash_configs = relay.ConnectionField(lambda: schema.files.FileConnection)
-
-    def resolve_dash_configs(self, info, *args, **kwargs):
-        return schema.files.find_files_by_query(cwd=self.id, query="*.dashcfg")
-
-    experiments = relay.ConnectionField(lambda: schema.experiments.ExperimentConnection)
-
-    def resolve_experiments(self, info, **kwargs):
-        return schema.experiments.find_experiments(cwd=self.id)
-
-    directories = relay.ConnectionField(lambda: schema.directories.DirectoryConnection)
-
-    def resolve_directories(self, info, **kwargs):
-        from ml_dash.config import Args
-        root_dir = join(Args.logdir, self.id[1:])
-        return [get_directory(join(self.id, _))
-                for _ in listdir(root_dir) if not isfile(join(root_dir, _))]
-
-    files = relay.ConnectionField(lambda: schema.files.FileConnection)
-
-    def resolve_files(self, info, **kwargs):
-        from ml_dash.config import Args
-        root_dir = join(Args.logdir, self.id[1:])
-        return [schema.Directory(id=join(self.id, _), name=_)
-                for _ in listdir(root_dir) if isfile(join(root_dir, _))]
-
-    @classmethod
-    def get_node(cls, info, id):
-        return get_directory(id)
-
-
-class DirectoryConnection(relay.Connection):
-    class Meta:
-        node = Directory
-
-
-def get_directory(id):
-    _id = id.rstrip('/')
-    return Directory(id=_id, name=split(_id[1:])[-1], path=_id)

ml_dash/schema/experiments.py

@@ -1,65 +0,0 @@
-from os import listdir
-from os.path import isfile, join, basename, realpath, isabs
-
-from graphene import ObjectType, relay, String, Field
-from ml_dash import schema
-from ml_dash.schema import files
-from ml_dash.schema.files.file_helpers import find_files
-from ml_dash.schema.files.metrics import find_metrics
-
-
-class Experiment(ObjectType):
-    class Meta:
-        interfaces = relay.Node,
-
-    name = String(description='name of the directory')
-    path = String(description="path to the experiment")
-    parameters = Field(lambda: files.parameters.Parameters, )
-    metrics = Field(lambda: files.metrics.Metrics)
-
-    def resolve_parameters(self, info):
-        return files.parameters.get_parameters(self.parameters)
-
-    def resolve_metrics(self, info):
-        for m in find_metrics(self.id):
-            return m
-        return None
-
-    directories = relay.ConnectionField(lambda: schema.directories.DirectoryConnection)
-    files = relay.ConnectionField(lambda: schema.files.FileConnection)
-
-    def resolve_directories(self, info, **kwargs):
-        from ml_dash.config import Args
-        root_dir = join(Args.logdir, self.id[1:])
-        return [schema.directories.get_directory(join(self.id, _))
-                for _ in listdir(root_dir) if not isfile(join(root_dir, _))]
-
-    def resolve_files(self, info, **kwargs):
-        from ml_dash.config import Args
-        root_dir = join(Args.logdir, self.id[1:])
-        return [schema.files.File(id=join(self.id, _), name=_)
-                for _ in listdir(root_dir) if isfile(join(root_dir, _))]
-
-    @classmethod
-    def get_node(cls, info, id):
-        return Experiment(id=id)
-
-
-class ExperimentConnection(relay.Connection):
-    class Meta:
-        node = Experiment
-
-
-def find_experiments(cwd, **kwargs):
-    from ml_dash.config import Args
-    assert isabs(cwd), "the current work directory need to be an absolute path."
-    _cwd = realpath(join(Args.logdir, cwd[1:])).rstrip('/')
-    parameter_files = find_files(_cwd, "**/parameters.pkl", **kwargs)
-    return [
-        # note: not sure about the name.
-        Experiment(id=join(cwd.rstrip('/'), p['dir']),
-                   name=basename(p['dir']) or ".",
-                   path=join(cwd.rstrip('/'), p['dir']),
-                   parameters=join(cwd.rstrip('/'), p['path']), )
-        for p in parameter_files
-    ]

ml_dash/schema/files/__init__.py

@@ -1,204 +0,0 @@
-import os
-from os.path import split, isabs, realpath, join, basename, dirname
-from graphene import ObjectType, relay, String, Int, Mutation, ID, Field, Node, Boolean
-from graphene.types.generic import GenericScalar
-from graphql_relay import from_global_id
-from ml_dash.schema.files.file_helpers import find_files
-
-from . import parameters, metrics
-
-
-class File(ObjectType):
-    class Meta:
-        interfaces = relay.Node,
-
-    name = String(description='name of the directory')
-    stem = String(description="stem of the file name")
-
-    def resolve_stem(self, info, ):
-        return self.name.split("/")[-1].split('.')[0]
-
-    path = String(description='path to the file')
-    rel_path = String(description='relative path to the file')
-    text = String(description='text content of the file', start=Int(required=False, default_value=0),
-                  stop=Int(required=False, default_value=None))
-
-    def resolve_text(self, info, start=0, stop=None):
-        from ml_dash.config import Args
-        with open(join(Args.logdir, self.id[1:]), "r") as f:
-            lines = list(f)[start: stop]
-            return "".join(lines)
-
-    json = GenericScalar(description="the json content of the file")
-
-    def resolve_json(self, info):
-        import json
-        try:
-            with open(self.id, 'r') as f:
-                return json.load(f)
-        except FileNotFoundError:
-            return None
-
-    yaml = GenericScalar(description="the content of the file using yaml")
-
-    def resolve_yaml(self, info):
-        import ruamel.yaml
-        if ruamel.yaml.version_info < (0, 15):
-            yaml = ruamel.yaml
-            load_fn = yaml.safe_load
-        else:
-            from ruamel.yaml import YAML
-            yaml = YAML()
-            yaml.explict_start = True
-            load_fn = yaml.load
-
-        from ml_dash.config import Args
-        with open(join(Args.logdir, self.id[1:]), "r") as f:
-            return load_fn('\n'.join(f))
-
-    @classmethod
-    def get_node(cls, info, id):
-        return get_file(id)
-
-
-class FileConnection(relay.Connection):
-    class Meta:
-        node = File
-
-
-def get_file(id):
-    # path = os.path.join(Args.logdir, id[1:])
-    return File(id=id, name=split(id[1:])[1])
-
-
-def find_files_by_query(cwd, query="**/*.*", **kwargs):
-    from ml_dash.config import Args
-    assert isabs(cwd), "the current work directory need to be an absolute path."
-    _cwd = realpath(join(Args.logdir, cwd[1:])).rstrip('/')
-    parameter_files = find_files(_cwd, query)
-    return [
-        # note: not sure about the name.
-        File(id=join(cwd.rstrip('/'), p['path']),
-             name=basename(p['path']),
-             path=join(cwd.rstrip('/'), p['path']),
-             rel_path=p['path'], )
-        for p in parameter_files
-    ]
-
-
-def glob_files(cwd, query="*.*"):
-    return find_files_by_query(cwd=cwd, query=query)
-
-
-def save_text_to_file(path, text):
-    from ml_dash.config import Args
-    assert isabs(path), "the path has to be absolute path."
-    _path = join(Args.logdir, path[1:])
-    with open(_path, "w") as f:
-        f.write(text)
-    return get_file(path)
-
-
-def save_json_to_file(path, data):
-    from ml_dash.config import Args
-    assert isabs(path), "the path has to be absolute path."
-    _path = join(Args.logdir, path[1:])
-    # note: assume all text format
-    with open(_path, "w+") as f:
-        import json
-        _ = json.dumps(data, sortKeys=True, indent=2)
-        f.write(_)
-    return get_file(path)
-
-
-def remove_file(path):
-    """remove does not work with directories"""
-    from ml_dash.config import Args
-    assert isabs(path), "the path has to be absolute path."
-    _path = join(Args.logdir, path[1:])
-    os.remove(_path)
-
-
-def remove_directory(path):
-    """rmtree does not work with files"""
-    import shutil
-    from ml_dash.config import Args
-    assert isabs(path), "the path has to be absolute path."
-    _path = join(Args.logdir, path[1:])
-    shutil.rmtree(_path)
-
-
-def save_yaml_to_file(path, data):
-    raise NotImplementedError
-
-
-class MutateTextFile(relay.ClientIDMutation):
-    class Input:
-        id = ID()
-        text = String(required=True)
-
-    file = Field(File)
-
-    @classmethod
-    def mutate_and_get_payload(cls, root, info, id, text, client_mutation_id):
-        _type, id = from_global_id(id)
-        return MutateTextFile(file=save_text_to_file(id, text))
-
-
-class MutateYamlFile(relay.ClientIDMutation):
-    class Arguments:
-        data = String()
-
-    file = Field(File)
-
-    @classmethod
-    def mutate_and_get_payload(self, root, info, data, client_mutation_id):
-        _type, id = from_global_id(client_mutation_id)
-        return MutateYamlFile(file=save_yaml_to_file(id, data))
-
-
-class MutateJSONFile(relay.ClientIDMutation):
-    class Arguments:
-        id = ID()
-        data = String()
-
-    file = Field(File)
-
-    @classmethod
-    def mutate_and_get_payload(self, root, info, id, data, client_mutation_id):
-        _type, id = from_global_id(client_mutation_id)
-        return MutateJSONFile(file=save_json_to_file(id, data))
-
-
-class DeleteFile(relay.ClientIDMutation):
-    class Input:
-        id = ID()
-
-    ok = Boolean()
-    id = ID()
-
-    @classmethod
-    def mutate_and_get_payload(cls, root, info, id, client_mutation_id):
-        _type, path = from_global_id(id)
-        try:
-            remove_file(path)
-            return DeleteFile(ok=True, id=id)
-        except FileNotFoundError:
-            return DeleteFile(ok=False)
-
-
-class DeleteDirectory(relay.ClientIDMutation):
-    class Input:
-        id = ID()
-
-    ok = Boolean()
-    id = ID()
-
-    @classmethod
-    def mutate_and_get_payload(cls, root, info, id, client_mutation_id):
-        _type, path = from_global_id(id)
-        try:
-            remove_directory(path)
-            return DeleteDirectory(ok=True, id=id)
-        except FileNotFoundError:
-            return DeleteDirectory(ok=False)

ml_dash/schema/files/file_helpers.py

@@ -1,79 +0,0 @@
-from glob import iglob
-from os import stat
-from os.path import basename, join, realpath, dirname
-
-from ml_dash.file_handlers import cwdContext
-
-
-def file_stat(file_path):
-    # note: this when looped over is very slow. Fine for a small list of files though.
-    stat_res = stat(file_path)
-    sz = stat_res.st_size
-    return dict(
-        name=basename(file_path),
-        path=file_path,
-        dir=dirname(file_path),
-        time_modified=stat_res.st_mtime,
-        time_created=stat_res.st_ctime,
-        # type=ft,
-        size=sz,
-    )
-
-
-def find_files(cwd, query, start=None, stop=None):
-    """
-    find files by iGlob.
-
-    :param cwd: the context folder for the glob, excluded from returned path list.
-    :param query: glob query
-    :param start: starting index for iGlob.
-    :param stop: ending index for iGlob
-    :return:
-    """
-    from itertools import islice
-    with cwdContext(cwd):
-        file_paths = list(islice(iglob(query, recursive=True), start or 0, stop or 200))
-        files = [file_stat(_) for _ in file_paths]
-        return files
-
-
-def read_dataframe(path, k=200):
-    from ml_logger.helpers import load_pickle_as_dataframe
-    try:
-        return load_pickle_as_dataframe(path, k)
-    except FileNotFoundError:
-        return None
-
-
-def read_records(path, k=200):
-    from ml_logger.helpers import load_pickle_as_dataframe
-    df = load_pickle_as_dataframe(path, k)
-    return df.to_json(orient="records")
-
-
-def read_log(path, k=200):
-    from ml_logger.helpers import load_pickle_as_dataframe
-    df = load_pickle_as_dataframe(path, k)
-    return df.to_json(orient="records")
-
-
-def read_json(path):
-    from ml_logger.helpers import load_from_pickle
-    data = [_ for _ in load_from_pickle(path)]
-    return data
-
-
-def read_text(path, start, stop):
-    from itertools import islice
-    with open(path, 'r') as f:
-        text = ''.join([l for l in islice(f, start, stop)])
-    return text
-
-
-def read_binary():
-    raise NotImplementedError()
-    # todo: check the file handling here. Does this use correct
-    #  mimeType for text files?
-    # res = await response.file(path)
-    # if as_attachment:
-    #     res.headers['Content-Disposition'] = 'attachment'