ml-dash 0.0.11__py3-none-any.whl → 0.5.9__py3-none-any.whl

ml_dash-0.5.9.dist-info/METADATA

@@ -0,0 +1,244 @@
+Metadata-Version: 2.3
+Name: ml-dash
+Version: 0.5.9
+Summary: ML experiment tracking and data storage
+Keywords: machine-learning,experiment-tracking,mlops,data-storage
+Author: Ge Yang, Tom Tao
+License: MIT License
+         
+         Copyright (c) 2025 Ge Yang, Tom Tao
+         
+         Permission is hereby granted, free of charge, to any person obtaining a copy
+         of this software and associated documentation files (the "Software"), to deal
+         in the Software without restriction, including without limitation the rights
+         to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+         copies of the Software, and to permit persons to whom the Software is
+         furnished to do so, subject to the following conditions:
+         
+         The above copyright notice and this permission notice shall be included in all
+         copies or substantial portions of the Software.
+         
+         THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+         IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+         FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+         AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+         LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+         OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+         SOFTWARE.
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: pyjwt>=2.8.0
+Requires-Dist: imageio>=2.31.0
+Requires-Dist: imageio-ffmpeg>=0.4.9
+Requires-Dist: scikit-image>=0.21.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: pytest>=8.0.0 ; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23.0 ; extra == 'dev'
+Requires-Dist: sphinx>=7.2.0 ; extra == 'dev'
+Requires-Dist: furo>=2024.0.0 ; extra == 'dev'
+Requires-Dist: sphinx-autodoc-typehints>=2.0.0 ; extra == 'dev'
+Requires-Dist: sphinx-autobuild>=2024.0.0 ; extra == 'dev'
+Requires-Dist: sphinx-copybutton>=0.5.0 ; extra == 'dev'
+Requires-Dist: sphinx-design>=0.5.0 ; extra == 'dev'
+Requires-Dist: sphinx-tabs>=3.4.0 ; extra == 'dev'
+Requires-Dist: sphinxcontrib-mermaid>=0.9.0 ; extra == 'dev'
+Requires-Dist: sphinxext-opengraph>=0.9.0 ; extra == 'dev'
+Requires-Dist: myst-parser>=2.0.0 ; extra == 'dev'
+Requires-Dist: linkify-it-py>=2.0.0 ; extra == 'dev'
+Requires-Dist: ruff>=0.3.0 ; extra == 'dev'
+Requires-Dist: mypy>=1.9.0 ; extra == 'dev'
+Requires-Python: >=3.9
+Provides-Extra: dev
+Description-Content-Type: text/markdown
+
+# ML-Dash
+
+A simple and flexible SDK for ML experiment metricing and data storage.
+
+## Features
+
+- **Three Usage Styles**: Decorator, context manager, or direct instantiation
+- **Dual Operation Modes**: Remote (API server) or local (filesystem)
+- **Auto-creation**: Automatically creates namespace, project, and folder hierarchy
+- **Upsert Behavior**: Updates existing experiments or creates new ones
+- **Experiment Lifecycle**: Automatic status tracking (RUNNING, COMPLETED, FAILED, CANCELLED)
+- **Organized File Storage**: Prefix-based file organization with unique snowflake IDs
+- **Rich Metadata**: Tags, bindrs, descriptions, and custom metadata support
+- **Simple API**: Minimal configuration, maximum flexibility
+
+## Installation
+
+<table>
+<tr>
+<td>Using uv (recommended)</td>
+<td>Using pip</td>
+</tr>
+<tr>
+<td>
+
+```bash
+uv add ml-dash
+```
+
+</td>
+<td>
+
+```bash
+pip install ml-dash
+```
+
+</td>
+</tr>
+</table>
+
+## Getting Started
+
+### Remote Mode (with API Server)
+
+```python
+from ml_dash import Experiment
+
+with Experiment(
+    name="my-experiment",
+    project="my-project",
+    remote="https://api.dash.ml",
+    api_key="your-jwt-token"
+) as experiment:
+    print(f"Experiment ID: {experiment.id}")
+```
+
+### Local Mode (Filesystem)
+
+```python
+from ml_dash import Experiment
+
+with Experiment(
+    name="my-experiment",
+    project="my-project",
+    local_path=".ml-dash"
+) as experiment:
+    pass  # Your code here
+```
+
+See [examples/](examples/) for more complete examples.
+
+## Development Setup
+
+### Installing Dev Dependencies
+
+To contribute to ML-Dash or run tests, install the development dependencies:
+
+<table>
+<tr>
+<td>Using uv (recommended)</td>
+<td>Using pip</td>
+</tr>
+<tr>
+<td>
+
+```bash
+uv sync --extra dev
+```
+
+</td>
+<td>
+
+```bash
+pip install -e ".[dev]"
+```
+
+</td>
+</tr>
+</table>
+
+This installs:
+- `pytest>=8.0.0` - Testing framework
+- `pytest-asyncio>=0.23.0` - Async test support
+- `sphinx>=7.2.0` - Documentation builder
+- `sphinx-rtd-theme>=2.0.0` - Read the Docs theme
+- `sphinx-autobuild>=2024.0.0` - Live preview for documentation
+- `myst-parser>=2.0.0` - Markdown support for Sphinx
+- `ruff>=0.3.0` - Linter and formatter
+- `mypy>=1.9.0` - Type checker
+
+### Running Tests
+
+<table>
+<tr>
+<td>Using uv</td>
+<td>Using pytest directly</td>
+</tr>
+<tr>
+<td>
+
+```bash
+uv run pytest
+```
+
+</td>
+<td>
+
+```bash
+pytest
+```
+
+</td>
+</tr>
+</table>
+
+### Building Documentation
+
+Documentation is built using Sphinx with Read the Docs theme.
+
+<table>
+<tr>
+<td>Build docs</td>
+<td>Live preview</td>
+<td>Clean build</td>
+</tr>
+<tr>
+<td>
+
+```bash
+uv run python -m sphinx -b html docs docs/_build/html
+```
+
+</td>
+<td>
+
+```bash
+uv run sphinx-autobuild docs docs/_build/html
+```
+
+</td>
+<td>
+
+```bash
+rm -rf docs/_build
+```
+
+</td>
+</tr>
+</table>
+
+The live preview command starts a local server and automatically rebuilds when files change.
+
+Alternatively, you can use the Makefile from within the docs directory:
+
+```bash
+cd docs
+make html          # Build HTML documentation
+make clean         # Clean build files
+```
+
+For maintainers, to build and publish a new release: `uv build && uv publish`

ml_dash-0.5.9.dist-info/RECORD

@@ -0,0 +1,20 @@
+ml_dash/__init__.py,sha256=o_LrWVJBY_VkUGhSBs5wdb_NqEsHD1AK9HGsjZGxHxQ,1414
+ml_dash/auto_start.py,sha256=c3XcXFpZdvjtWauEoK5043Gw9k0L_5IDq4fdiB2ha88,959
+ml_dash/cli.py,sha256=lyWVVhmsflSXQt2UCDb8IqC-mSRQwwlB2l1qEIYBUb8,1743
+ml_dash/cli_commands/__init__.py,sha256=bjAmV7MsW-bhtW_4SnLJ0Cfkt9h82vMDC8ebW1Ke8KE,38
+ml_dash/cli_commands/download.py,sha256=TomyUFwelqfQHfh60K7rCyCwEZVp1CkMToogprgC64Q,29614
+ml_dash/cli_commands/list.py,sha256=Cx9yWsTV5HPaevYpQ6BugCEr5z_4bhxQ0T51OXExuTU,10900
+ml_dash/cli_commands/upload.py,sha256=jo6FVdbuokTz64rjvOEWWhLBzlh2gM0Ru4TRNv9hX60,47943
+ml_dash/client.py,sha256=31C2Kb3KULwhrb3UlpCFY7HDA3-kvj3XVmWUvXEvQHY,27993
+ml_dash/config.py,sha256=iQbHCu4lM_Sg8YadyEXSJ6Ht9yKIJHN26L7L-rMH4gE,3112
+ml_dash/experiment.py,sha256=K36HkHJb_O2-vdaPPOCq74_2nZtfiLaS0o7qhTntD8Q,30646
+ml_dash/files.py,sha256=JptjoxGJiXJ-nkj6C7vDhw-cgJRCB0cHt_SIUJG665o,23024
+ml_dash/log.py,sha256=0yXaNnFwYeBI3tRLHX3kkqWRpg0MbSGwmgjnOfsElCk,5350
+ml_dash/metric.py,sha256=c0Zl0wEufmQuVfwIMvrORLwqe92Iaf0PfKRgmlgQWzQ,10343
+ml_dash/params.py,sha256=xaByDSVar4D1pZqxTANkMPeZTL5-V7ewJe5TXfPLhMQ,5980
+ml_dash/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ml_dash/storage.py,sha256=1NLkIOZeFv0e97zieEX0tJT3qHTuM8UN6hvsFRQ6TTo,38941
+ml_dash-0.5.9.dist-info/WHEEL,sha256=z-mOpxbJHqy3cq6SvUThBZdaLGFZzdZPtgWLcP2NKjQ,79
+ml_dash-0.5.9.dist-info/entry_points.txt,sha256=dYs2EHX1uRNO7AQGNnVaJJpgiy0Z9q7tiy4fHSyaf3Q,46
+ml_dash-0.5.9.dist-info/METADATA,sha256=Acfle-Q8_jExDsli8yuZ8EMP1knWSkPtONo1FYB1sM4,6175
+ml_dash-0.5.9.dist-info/RECORD,,

ml_dash-0.5.9.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.9.15
+Root-Is-Purelib: true
+Tag: py3-none-any

ml_dash-0.5.9.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+ml-dash = ml_dash.cli:main
+

ml_dash/app.py

@@ -1,33 +0,0 @@
-import os
-from sanic import Sanic
-from sanic.exceptions import FileNotFound
-from sanic.response import file
-
-# gets current directory
-BASE = os.path.realpath(__file__)
-print(BASE)
-print(os.path.dirname(BASE))
-
-build_path = os.path.join(os.path.dirname(BASE), "../app-build")
-print(build_path)
-
-app = Sanic()
-# serve js file for webpack
-app.static('/', build_path)
-
-
-# app.static('/main.js', './app-build/main.js', name='main.js')
-
-@app.route('/')
-@app.exception(FileNotFound)
-async def index(request, exception=None):
-    print('hey ====', [exception])
-    return await file(build_path + '/index.html')
-
-
-if __name__ == '__main__':
-    app.run(
-        host='0.0.0.0',
-        port=int(os.environ.get('PORT', 3002)),
-        workers=int(os.environ.get('WEB_CONCURRENCY', 1)),
-        debug=bool(os.environ.get('DEBUG', '')))

ml_dash/file_events.py

@@ -1,71 +0,0 @@
-from hachiko.hachiko import AIOEventHandler, AIOWatchdog
-from asyncio import coroutine, Queue, sleep
-from sanic import response
-from sanic.exceptions import RequestTimeout
-
-from ml_dash.file_utils import path_match
-from termcolor import cprint
-
-from . import config
-import json
-
-subscriptions = []
-watcher = None
-
-
-class Handler(AIOEventHandler):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-    @coroutine
-    async def on_any_event(self, event):
-        _event = dict(src_path=event.src_path, event_type=event.event_type, is_directory=event.is_directory)
-        for que in subscriptions:
-            await que.put(_event)
-            # self._loop.create_task(que.put(event))
-
-
-def setup_watch_queue(app, loop):
-    print('setting up watch queue')
-    start_watcher()
-    cprint('watcher setup complete!', "green")
-
-
-def start_watcher():
-    global watcher
-
-    handler = Handler()
-    print('starting file watcher...')
-    watcher = AIOWatchdog(config.Args.logdir, event_handler=handler)
-    watcher.start()
-    print('watcher start is complete')
-
-
-import os
-
-
-# server does not have access to a disconnect event.
-# currently subscriptions only grows.
-# Will add timeout based cleanup after.
-async def file_events(request, file_path="", query="*"):
-    q = Queue()
-    subscriptions.append(q)
-
-    async def streaming_fn(response):
-        try:
-            while True:
-                print('subscription que started')
-                file_event = await q.get()
-                src_path = file_event['src_path']
-                if src_path.startswith(os.path.join(config.Args.logdir, file_path)) and path_match(file_path, query):
-                    file_event['src_path'] = src_path[len(config.Args.logdir):]
-                print("=>>", file_event)
-                response.write(f"data: {json.dumps(file_event)}\r\n\r\n".encode())
-                sleep(0.1)
-        # todo: this timeout doesn't really work.
-        # todo: also add handling of stream is terminated logic (separate from above).
-        except RequestTimeout:
-            subscriptions.remove(q)
-
-    return response.stream(streaming_fn, content_type="text/event-stream")
-    # subscriptions.remove(q)

ml_dash/file_handlers.py

@@ -1,141 +0,0 @@
-import os
-import stat
-from glob import iglob
-from shutil import rmtree
-from sanic import response
-
-from . import config
-
-
-def get_type(mode):
-    if stat.S_ISDIR(mode) or stat.S_ISLNK(mode):
-        type = 'dir'
-    else:
-        type = 'file'
-    return type
-
-
-async def remove_path(request, file_path=""):
-    print(file_path)
-    path = os.path.join(config.Args.logdir, file_path)
-    if os.path.isdir(path):
-        rmtree(path)
-        res = response.text("ok", status=204)
-    elif os.path.isfile(path):
-        os.remove(path)
-        res = response.text("ok", status=204)
-    else:
-        res = response.text('Not found', status=404)
-    return res
-
-
-from contextlib import contextmanager
-
-
-@contextmanager
-def cwdContext(path):
-    owd = os.getcwd()
-    os.chdir(path)
-    try:
-        yield
-    finally:
-        os.chdir(owd)
-
-
-async def batch_get_path(request):
-    try:
-        data = request.json
-
-        file_paths = data['paths']
-        options = data['options']
-
-        batch_res_data = dict()
-
-        if options.get('json', False):
-            for path in file_paths:
-                from ml_logger.helpers import load_from_pickle
-                batch_res_data[path] = [_ for _ in load_from_pickle(path)]
-
-            res = response.json(batch_res_data, status=200, content_type='application/json')
-            return res
-
-    except Exception as e:
-        print('Exception: ', e)
-        res = response.text('Internal Error' + str(e), status=502)
-        return res
-
-
-async def get_path(request, file_path=""):
-    print(file_path)
-
-    as_records = request.args.get('records')
-    as_json = request.args.get('json')
-    as_log = request.args.get('log')
-    as_attachment = int(request.args.get('download', '0'))
-    is_recursive = request.args.get('recursive')
-    show_hidden = request.args.get('hidden')
-    query = request.args.get('query', "*").strip()
-
-    _start = request.args.get('start', None)
-    _stop = request.args.get('stop', None)
-    start = None if _start is None else int(_start)
-    stop = None if _stop is None else int(_stop)
-
-    reservoir_k = int(request.args.get('reservoir', '200'))
-
-    # limit for the search itself.
-    search_limit = 500
-
-    path = os.path.join(config.Args.logdir, file_path)
-    print("=============>", [query], [path], os.path.isdir(path))
-
-    if os.path.isdir(path):
-        from itertools import islice
-        with cwdContext(path):
-            print(os.getcwd(), query, is_recursive)
-            file_paths = list(islice(iglob(query, recursive=is_recursive), start or 0, stop or 200))
-            files = map(file_stat, file_paths)
-            res = response.json(files, status=200)
-    elif os.path.isfile(path):
-        if as_records:
-            from ml_logger.helpers import load_pickle_as_dataframe
-            df = load_pickle_as_dataframe(path, reservoir_k)
-            res = response.text(df.to_json(orient="records"), status=200, content_type='application/json')
-        elif as_log:
-            from ml_logger.helpers import load_pickle_as_dataframe
-            df = load_pickle_as_dataframe(path, reservoir_k)
-            res = response.text(df.to_json(orient="records"), status=200, content_type='application/json')
-        elif as_json:
-            from ml_logger.helpers import load_from_pickle
-            data = [_ for _ in load_from_pickle(path)]
-            res = response.json(data, status=200, content_type='application/json')
-        elif type(start) is int or type(stop) is int:
-            from itertools import islice
-            with open(path, 'r') as f:
-                text = ''.join([l for l in islice(f, start, stop)])
-            res = response.text(text, status=200)
-        else:
-            # todo: check the file handling here. Does this use correct
-            #  mimeType for text files?
-            res = await response.file(path)
-            if as_attachment:
-                res.headers['Content-Disposition'] = 'attachment'
-    else:
-        res = response.text('Not found', status=404)
-    return res
-
-
-# use glob! LOL
-def file_stat(file_path):
-    # this looped over is very slow. Fine for a small list of files though.
-    stat_res = os.stat(file_path)
-    ft = get_type(stat_res.st_mode)
-    sz = stat_res.st_size
-    return dict(
-        name=os.path.basename(file_path),
-        path=file_path,
-        mtime=stat_res.st_mtime,
-        ctime=stat_res.st_ctime,
-        type=ft,
-        size=sz,
-    )

ml_dash/file_utils.py

@@ -1,5 +0,0 @@
-def path_match(query, pattern):
-    import glob, re
-    regex = fnmatch.translate(pattern)
-    reobj = re.compile(regex)
-    return reobj.match(query)

ml_dash/file_watcher.py

@@ -1,30 +0,0 @@
-# from . import config
-import asyncio
-from hachiko.hachiko import AIOWatchdog
-
-
-class Handler:
-    def dispatch(self, *args, **kwargs):
-        print(args, kwargs)
-
-@asyncio.coroutine
-def watch_fs(path):
-    watch = AIOWatchdog(path, event_handler=Handler())
-    watch.start()
-    while True:
-        yield from asyncio.sleep(10)
-    watch.stop()
-
-
-
-if __name__ == "__main__":
-    # asyncio.get_event_loop().run_until_complete(watch_fs("/Users/ge/machine_learning/berkeley-playground/ins-runs"))
-    # asyncio.get_event_loop().run_until_complete(watch_fs("."))
-    path = "."
-    watch = AIOWatchdog(path, event_handler=Handler())
-    watch.start()
-    import time
-    print('watch is setup')
-    while True:
-        time.sleep(10)
-

ml_dash/main.py

@@ -1,60 +0,0 @@
-from ml_dash.schema import schema
-from sanic_graphql import GraphQLView
-
-from .file_events import file_events, setup_watch_queue
-from .file_handlers import get_path, remove_path, batch_get_path
-
-from sanic import Sanic
-from sanic_cors import CORS
-
-app = Sanic(__name__)
-# CORS(app)
-CORS(app, resources={r"/*": {"origins": "*"}}, automatic_options=True)
-
-# @app.listener('before_server_start')
-# def init_graphql(app, loop):
-#     app.add_route(GraphQLView.as_view(schema=schema, executor=AsyncioExecutor(loop=loop)), '/graphql')
-
-# new graphQL endpoints
-app.add_route(GraphQLView.as_view(schema=schema, graphiql=True), '/graphql',
-              methods=['GET', 'POST', 'FETCH', 'OPTIONS'])
-app.add_route(GraphQLView.as_view(schema=schema, batch=True), '/graphql/batch',
-              methods=['GET', 'POST', 'FETCH', 'OPTIONS'])
-
-# # Serving static app
-# app.add_route(get_path, '/*', methods=['GET', 'OPTIONS'])
-
-# old RPC endpoints
-app.add_route(get_path, '/files/', methods=['GET', 'OPTIONS'])
-app.add_route(get_path, '/files/<file_path:path>', methods=['GET', 'OPTIONS'])
-app.add_route(batch_get_path, '/batch-files', methods=['GET', 'OPTIONS'])
-app.add_route(remove_path, '/files/<file_path:path>', methods=['DELETE'])
-app.add_route(file_events, '/file-events', methods=['GET', 'OPTIONS'])
-app.add_route(file_events, '/file-events/<file_path:path>', methods=['GET', 'OPTIONS'])
-app.listener('before_server_start')(setup_watch_queue)
-
-
-# app.add_task(start_watcher)
-
-
-def run(logdir=None, **kwargs):
-    from . import config
-    from termcolor import cprint
-
-    if logdir:
-        config.Args.logdir = logdir
-
-    cprint("launched server with config:", "green")
-    cprint("Args:", 'yellow')
-    print(vars(config.Args))
-    cprint("Sanic Server Args:", 'yellow')
-    print(vars(config.ServerArgs))
-
-    config.ServerArgs.update(**kwargs)
-    app.run(**vars(config.ServerArgs))
-
-
-if __name__ == "__main__":
-    # see: https://sanic.readthedocs.io/en/latest/sanic/deploying.html
-    # call this as `python -m ml_logger.main`
-    run()

ml_dash/mime_types.py

@@ -1,20 +0,0 @@
-ignored = ['.bzr', '$RECYCLE.BIN', '.DAV', '.DS_Store', '.git', '.hg', '.htaccess', '.htpasswd', '.Spotlight-V100',
-           '.svn', '__MACOSX', 'ehthumbs.db', 'robots.txt', 'Thumbs.db', 'thumbs.tps']
-datatypes = {'audio': 'm4a,mp3,oga,ogg,webma,wav',
-             'archive': '7z,zip,rar,gz,tar',
-             'image': 'gif,ico,jpe,jpeg,jpg,png,svg,webp',
-             'pdf': 'pdf',
-             'quicktime': '3g2,3gp,3gp2,3gpp,mov,qt',
-             'source': 'atom,bat,bash,c,cmd,coffee,css,hml,js,json,java,less,markdown,md,php,pl,py,rb,rss,sass,scpt,swift,scss,sh,xml,yml,plist',
-             'text': 'txt',
-             'video': 'mp4,m4v,ogv,webm',
-             'website': 'htm,html,mhtm,mhtml,xhtm,xhtml'}
-icontypes = {'fa-music': 'm4a,mp3,oga,ogg,webma,wav',
-             'fa-archive': '7z,zip,rar,gz,tar',
-             'fa-picture-o': 'gif,ico,jpe,jpeg,jpg,png,svg,webp',
-             'fa-file-text': 'pdf',
-             'fa-film': '3g2,3gp,3gp2,3gpp,mov,qt',
-             'fa-code': 'atom,plist,bat,bash,c,cmd,coffee,css,hml,js,json,java,less,markdown,md,php,pl,py,rb,rss,sass,scpt,swift,scss,sh,xml,yml',
-             'fa-file-text-o': 'txt',
-             'fa-film': 'mp4,m4v,ogv,webm',
-             'fa-globe': 'htm,html,mhtm,mhtml,xhtm,xhtml'}