pluto-ml 0.0.2__py3-none-any.whl

mlop/__init__.py

@@ -0,0 +1,17 @@
+"""
+Backward compatibility shim for mlop -> pluto migration.
+This module is deprecated. Use `import pluto` instead.
+"""
+
+import warnings
+
+warnings.warn(
+    "The 'mlop' package is deprecated and will be removed in a future release. "
+    "Please use 'import pluto' instead.",
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+# Re-export everything from pluto
+from pluto import *  # noqa: E402, F401, F403
+from pluto import __all__, __version__  # noqa: E402, F401

mlop/__main__.py

@@ -0,0 +1,14 @@
+"""Deprecated CLI entry point."""
+
+import warnings
+
+warnings.warn(
+    "The 'mlop' command is deprecated. Use 'pluto' instead.",
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+from pluto.__main__ import main  # noqa: E402
+
+if __name__ == '__main__':
+    main()

mlop/compat/__init__.py

@@ -0,0 +1,15 @@
+"""
+Backward compatibility shim for mlop.compat -> pluto.compat migration.
+This module is deprecated. Use `import pluto.compat` instead.
+"""
+
+import warnings
+
+warnings.warn(
+    "The 'mlop.compat' module is deprecated. "
+    "Please use 'import pluto.compat' instead.",
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+from pluto.compat import *  # noqa: E402, F401, F403

mlop/compat/lightning.py

@@ -0,0 +1,15 @@
+"""
+Backward compatibility shim for mlop.compat.lightning.
+This module is deprecated. Use `import pluto.compat.lightning` instead.
+"""
+
+import warnings
+
+warnings.warn(
+    "The 'mlop.compat.lightning' module is deprecated. "
+    "Please use 'import pluto.compat.lightning' instead.",
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+from pluto.compat.lightning import *  # noqa: E402, F401, F403

mlop/compat/neptune.py

@@ -0,0 +1,15 @@
+"""
+Backward compatibility shim for mlop.compat.neptune -> pluto.compat.neptune migration.
+This module is deprecated. Use `import pluto.compat.neptune` instead.
+"""
+
+import warnings
+
+warnings.warn(
+    "The 'mlop.compat.neptune' module is deprecated. "
+    "Please use 'import pluto.compat.neptune' instead.",
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+from pluto.compat.neptune import *  # noqa: E402, F401, F403

mlop/compat/torch.py

@@ -0,0 +1,15 @@
+"""
+Backward compatibility shim for mlop.compat.torch -> pluto.compat.torch migration.
+This module is deprecated. Use `import pluto.compat.torch` instead.
+"""
+
+import warnings
+
+warnings.warn(
+    "The 'mlop.compat.torch' module is deprecated. "
+    "Please use 'import pluto.compat.torch' instead.",
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+from pluto.compat.torch import *  # noqa: E402, F401, F403

mlop/compat/transformers.py

@@ -0,0 +1,15 @@
+"""
+Backward compatibility shim for mlop.compat.transformers -> pluto.compat.transformers.
+This module is deprecated. Use `import pluto.compat.transformers` instead.
+"""
+
+import warnings
+
+warnings.warn(
+    "The 'mlop.compat.transformers' module is deprecated. "
+    "Please use 'import pluto.compat.transformers' instead.",
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+from pluto.compat.transformers import *  # noqa: E402, F401, F403

pluto/__init__.py

@@ -0,0 +1,73 @@
+import os
+import subprocess
+from typing import Any, Callable, List, Optional
+
+from .auth import login, logout
+from .data import Data, Graph, Histogram, Table
+from .file import Artifact, Audio, File, Image, Text, Video
+from .init import finish, init
+from .sets import Settings, setup
+from .sys import System
+
+_hooks: List[Any] = []
+ops: Optional[List[Any]] = None
+log: Optional[Callable[..., Any]] = None
+watch: Optional[Callable[..., Any]] = None
+alert: Optional[Callable[..., Any]] = None
+
+__all__ = (
+    'Data',
+    'Graph',
+    'Histogram',
+    'Table',
+    'File',
+    'Artifact',
+    'Text',
+    'Image',
+    'Audio',
+    'Video',
+    'System',
+    'Settings',
+    'alert',
+    'init',
+    'login',
+    'logout',
+    'watch',
+    'finish',
+    'setup',
+)
+
+__version__ = '0.0.2'
+
+
+# Replaced with the current commit when building the wheels.
+_PLUTO_COMMIT_SHA = 'e8c13ee426ab5c9a428bcb2285db1a552836a653'
+
+
+def _get_git_commit():
+    if 'PLUTO_COMMIT_SHA' not in _PLUTO_COMMIT_SHA:
+        # This is a release build, so we don't need to get the commit hash from
+        # git, as it's already been set.
+        return _PLUTO_COMMIT_SHA
+
+    # This is a development build (pip install -e .), so we need to get the
+    # commit hash from git.
+    try:
+        cwd = os.path.dirname(__file__)
+        commit_hash = subprocess.check_output(
+            ['git', 'rev-parse', 'HEAD'],
+            cwd=cwd,
+            universal_newlines=True,
+            stderr=subprocess.DEVNULL,
+        ).strip()
+        changes = subprocess.check_output(
+            ['git', 'status', '--porcelain'],
+            cwd=cwd,
+            universal_newlines=True,
+            stderr=subprocess.DEVNULL,
+        ).strip()
+        if changes:
+            commit_hash += '-dirty'
+        return commit_hash
+    except Exception:  # pylint: disable=broad-except
+        return _PLUTO_COMMIT_SHA

pluto/__main__.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+import argparse
+import sys
+
+from . import __version__, _get_git_commit
+from .auth import login, logout
+
+
+def main():
+    parser = argparse.ArgumentParser(description='pluto')
+    parser.add_argument(
+        '-v',
+        '--version',
+        action='store_true',
+        help='show the installed pluto version',
+    )
+    parser.add_argument(
+        '-c',
+        '--commit',
+        action='store_true',
+        help='show the current git commit hash',
+    )
+    subparsers = parser.add_subparsers(dest='command', help='commands')
+
+    p_login = subparsers.add_parser('login', help='login to pluto')
+    p_login.add_argument('key', nargs='?', help='login key')
+    subparsers.add_parser('logout', help='logout from pluto')
+
+    args = parser.parse_args()
+
+    if args.version:
+        print(__version__)
+        return
+
+    if args.commit:
+        print(_get_git_commit())
+        return
+
+    if args.command == 'login':
+        if args.key:
+            login(settings={'_auth': args.key})
+        else:
+            login()
+    elif args.command == 'logout':
+        logout()
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

pluto/api.py

@@ -0,0 +1,261 @@
+import json
+import logging
+import re
+import signal
+from datetime import datetime
+
+from .util import clean_dict, find_node
+
+logger = logging.getLogger(f"{__name__.split('.')[0]}")
+tag = 'API'
+
+STATUS = {
+    -1: 'RUNNING',
+    0: 'COMPLETED',
+    1: 'FAILED',
+    signal.SIGINT.value: 'TERMINATED',  # "INTERRUPTED",
+}
+
+ABBR = {
+    'pct': 'percentage',
+    'net': 'network',
+    'mem': 'memory',
+    'recv': 'received',
+    'bytes_': 'bytes.',
+}
+
+
+def make_compat_trigger_v1(settings):
+    return json.dumps(
+        {
+            'runId': settings._op_id,
+        }
+    ).encode()
+
+
+def make_compat_start_v1(config, settings, info, tags=None):
+    return json.dumps(
+        {
+            # "runId": settings._op_id,
+            'runName': settings._op_name,
+            'projectName': settings.project,
+            'config': json.dumps(config) if config is not None else None,
+            'loggerSettings': json.dumps(clean_dict(settings.to_dict())),
+            'systemMetadata': json.dumps(info) if info is not None else None,
+            'tags': tags if tags else None,
+            'createdAt': settings.compat.get('createdAt'),
+            'updatedAt': settings.compat.get('updatedAt'),
+        }
+    ).encode()
+
+
+def make_compat_status_v1(settings, trace=None):
+    return json.dumps(
+        {
+            'runId': settings._op_id,
+            'status': STATUS[settings._op_status],
+            # "metadata": json.dumps(settings.meta),
+            'statusMetadata': json.dumps(trace) if trace is not None else None,
+        }
+    ).encode()
+
+
+def make_compat_update_tags_v1(settings, tags):
+    return json.dumps(
+        {
+            'runId': settings._op_id,
+            'tags': tags,
+        }
+    ).encode()
+
+
+def make_compat_update_config_v1(settings, config):
+    return json.dumps(
+        {
+            'runId': settings._op_id,
+            'config': json.dumps(config) if config else None,
+        }
+    ).encode()
+
+
+def make_compat_meta_v1(meta, dtype, settings):
+    return json.dumps(
+        {
+            'runId': settings._op_id,
+            # "runName": settings._op_name,
+            # "projectName": settings.project,
+            'logType': dtype.upper() if dtype != 'num' else 'METRIC',
+            'logName': meta,  # TODO: better aggregate
+        }
+    ).encode()
+
+
+def make_compat_monitor_v1(data):
+    if not ABBR:
+        return data
+    pattern = re.compile('|'.join(map(re.escape, ABBR.keys())))
+    return {pattern.sub(lambda m: ABBR[m.group(0)], k): v for k, v in data.items()}
+
+
+def make_compat_num_v1(data, timestamp, step):
+    line = [
+        json.dumps(
+            {
+                'time': int(timestamp * 1000),  # convert to ms
+                'step': int(step),
+                'data': data,
+            }
+        )
+    ]
+    return ('\n'.join(line) + '\n').encode('utf-8')
+
+
+def make_compat_data_v1(data, timestamp, step):
+    lines = []
+    for k, dl in data.items():
+        for d in dl:
+            c = json.dumps(d.to_dict())
+            lines.append(
+                json.dumps(
+                    {
+                        'time': int(timestamp * 1000),  # convert to ms
+                        'data': c,
+                        'dataType': type(d).__name__.upper(),
+                        'logName': k,
+                        'step': step,
+                    }
+                )
+            )
+    return ('\n'.join(lines) + '\n').encode('utf-8')
+
+
+def make_compat_file_v1(file, timestamp, step):
+    batch = []
+    for k, fl in file.items():
+        for f in fl:
+            i = {
+                'fileName': f'{f._name}{f._ext}',
+                'fileSize': f._stat.st_size,
+                'fileType': f._ext[1:],
+                'time': int(f._stat.st_mtime * 1000),
+                'logName': k,
+                'step': step,
+            }
+            batch.append(i)
+    return json.dumps({'files': batch}).encode()
+
+
+def make_compat_storage_v1(f, fl):
+    # workaround for lack of file ident on server side
+    for i in fl:
+        if next(iter(i.keys())) == f'{f._name}{f._ext}':
+            return next(iter(i.values()))
+    return None
+
+
+def make_compat_message_v1(level, message, timestamp, step):
+    # TODO: server side int log level support
+    line = [
+        json.dumps(
+            {
+                'time': int(timestamp * 1000),  # convert to ms
+                'message': message,
+                'lineNumber': step,
+                'logType': logging._levelToName.get(level),
+            }
+        )
+    ]
+    return ('\n'.join(line) + '\n').encode('utf-8')
+
+
+def make_compat_graph_v1(settings, name, nodes):
+    return json.dumps(
+        {'runId': settings._op_id, 'graph': {'format': name, 'nodes': nodes}}
+    ).encode()
+
+
+def make_compat_graph_nodes_v1(d, ref, dep=0, p='', r={}):
+    if 'name' not in d:
+        d['name'] = ''
+        name = '.'
+    elif p == '.':
+        name = str(d['name'])
+    else:
+        name = f"{p}.{d['name']}"
+
+    if 'id' in d:
+        n = d.copy()
+        n = {k: v for k, v in n.items() if k not in ['id', 'nodes', 'name']}
+        r.update({name: n})
+        r[name]['depth'] = dep
+
+        if ref:
+            rd = find_node(ref, d['id'], key='nodes')
+            if rd:
+                rn = rd.copy()
+                rn = {k: v for k, v in rn.items() if k not in ['id', 'nodes']}
+                r[name].update(rn)
+            else:
+                logger.debug(
+                    f'{tag}: {n} not found in reference dictionary '
+                    f'when processing {name}'
+                )
+            r[name]['node_type'] = (
+                r[name]['node_type'].upper() if r[name].get('node_type') else 'UNKNOWN'
+            )
+
+    if 'nodes' in d:
+        for c in d['nodes']:
+            make_compat_graph_nodes_v1(d=c, ref=ref, dep=dep + 1, p=name, r=r)
+
+    return r
+
+
+def make_compat_alert_v1(settings, t, m, n, level, url, **kwargs):
+    return json.dumps(
+        {
+            'runId': settings._op_id,
+            'alert': {
+                'timestamp': int(t * 1000),
+                'level': level,
+                'title': n,
+                'body': m,
+                'email': kwargs.get('email', None),
+                'url': kwargs.get('url', None),
+            },
+        }
+    ).encode()
+
+
+def make_compat_webhook_v1(timestamp, level, title, message, step, url):
+    timestamp_str = datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')
+    return json.dumps(
+        {
+            'username': __name__.split('.')[0],
+            'content': f'{level}: {title}',
+            'embeds': [
+                {
+                    'description': message,
+                    'footer': {'text': ('Step: ' f'{step} at ' f'{timestamp_str}')},
+                }
+            ],
+            # slack
+            'text': f'{level}: {title}',
+            'blocks': [
+                {
+                    'type': 'section',
+                    'text': {
+                        'type': 'mrkdwn',
+                        'text': (
+                            f'`{level}` *{title}:* {message}\n\n'
+                            + f'_<{url}|Check out live updates for this run>_\n'
+                            + f'*Local Time:* <!date^{int(timestamp)}^'
+                            + '{date_short_pretty} {time_secs}'
+                            + '|Time>\n'
+                            + f'*Step:* {step}\n'
+                        ),
+                    },
+                }
+            ],
+        }
+    ).encode()

pluto/auth.py

@@ -0,0 +1,102 @@
+import getpass
+import logging
+import sys
+import webbrowser
+
+import httpx
+import keyring
+
+from .log import setup_logger, teardown_logger
+from .sets import get_console, setup
+from .util import ANSI, import_lib, print_url
+
+tlogger = logging.getLogger('auth')
+tag = 'Authentication'
+
+
+def login(settings=None, retry=False):
+    settings = setup(settings)
+    setup_logger(settings=settings, logger=tlogger)
+    try:
+        assert sys.platform == 'darwin'
+        auth = keyring.get_password(f'{settings.tag}', f'{settings.tag}')
+    except (keyring.errors.NoKeyringError, AssertionError):  # fallback
+        keyring.set_keyring(import_lib('keyrings.alt.file').PlaintextKeyring())
+        auth = keyring.get_password(f'{settings.tag}', f'{settings.tag}')
+    if settings._auth is None:
+        if auth == '':
+            keyring.delete_password(f'{settings.tag}', f'{settings.tag}')
+        elif auth is not None:
+            settings._auth = auth
+    if settings._auth == '':
+        tlogger.critical(
+            '%s: authentication failed: the provided token cannot be empty', tag
+        )
+        settings._auth = '_key'
+    client = httpx.Client(
+        verify=True if not settings.insecure_disable_ssl else False,
+        proxy=settings.http_proxy or settings.https_proxy or None,
+    )
+    try:
+        r = client.post(
+            url=settings.url_login,
+            headers={
+                'Authorization': f'Bearer {settings._auth}',
+            },
+        )
+    except Exception as e:
+        tlogger.warning(f'{tag}: server not reachable; reason: {e}')
+        settings._auth = '_key'
+    try:
+        tlogger.info(f"{tag}: logged in as {r.json()['organization']['slug']}")
+        keyring.set_password(f'{settings.tag}', f'{settings.tag}', f'{settings._auth}')
+        teardown_logger(tlogger)
+    except Exception as e:
+        if retry:
+            tlogger.warning('%s: authentication failed; reason: %s', tag, e)
+        hint1 = (
+            f'{ANSI.cyan}- Please copy the API key provided in the web portal '
+            'and paste it below'
+        )
+        hint2 = f'- You can alternatively manually open {print_url(settings.url_token)}'
+        hint3 = f'{ANSI.green}- You may exit at any time by pressing CTRL+C / ⌃+C'
+        tlogger.info(
+            f'{tag}: initializing authentication\n\n {hint1}\n\n {hint2}\n\n {hint3}\n'
+        )
+        if (
+            hasattr(settings._sys, 'monitor') and settings._sys.monitor() == {}
+        ):  # migrate mode
+            return
+        else:
+            webbrowser.open(url=settings.url_token)
+        if get_console() == 'jupyter':
+            settings._auth = getpass.getpass(prompt='Enter API key: ')
+        else:
+            settings._auth = input(f'{ANSI.yellow}Enter API key: ')
+        try:
+            keyring.set_password(
+                f'{settings.tag}', f'{settings.tag}', f'{settings._auth}'
+            )
+        except Exception as e:
+            tlogger.critical(
+                '%s: failed to save key to system keyring service: %s', tag, e
+            )
+        teardown_logger(tlogger)
+        login(settings=settings, retry=True)
+
+
+def logout(settings=None):
+    settings = setup(settings)
+    setup_logger(settings=settings, logger=tlogger)
+    try:
+        assert sys.platform == 'darwin'
+        keyring.delete_password(f'{settings.tag}', f'{settings.tag}')
+    except (keyring.errors.NoKeyringError, AssertionError):
+        keyring.set_keyring(import_lib('keyrings.alt.file').PlaintextKeyring())
+        keyring.delete_password(f'{settings.tag}', f'{settings.tag}')
+    except Exception as e:
+        tlogger.warning(
+            '%s: failed to delete key from system keyring service: %s', tag, e
+        )
+    tlogger.info(f'{tag}: logged out')
+    teardown_logger(tlogger)