scope 0.3.2__tar.gz → 0.4.0__tar.gz

scope-0.4.0/PKG-INFO

@@ -0,0 +1,66 @@
+Metadata-Version: 2.1
+Name: scope
+Version: 0.4.0
+Summary: Metrics logging and analysis
+Home-page: http://github.com/danijar/scope
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Description-Content-Type: text/markdown
+License-File: LICENSE
+
+# 🔬 Scope
+
+Scalable metrics logging and analysis.
+
+## Features
+
+- 🚀 **Scalable:** Quickly log and view petabytes of metrics, thousands of keys, and large videos.
+- 🎞️ **Formats:** Log and view scalars, text, images, and videos. Easy to extend with custom formats.
+- 🧑🏻‍🔬 **Productivity:** Metrics viewer with focus on power users and full keyboard support.
+- ☁️ **Cloud support:** Directly write to and read from Cloud storage via pathlib interface.
+- 🍃 **Lightweight:** The writer and reader measure only ~400 lines of Python code.
+- 🧱 **Reliability:** Unit tested and used across diverse research projects.
+
+## Usage
+
+### Installation
+
+```sh
+pip install scope
+```
+
+### Writing
+
+```python
+import scope
+
+writer = scope.Writer(logdir)
+for step in range(3)
+  video = np.zeros((100, 640, 360, 3), np.uint8)
+  writer.add(step, {'foo': 42, 'bar': video, 'baz': 'Hello World'})
+writer.flush()
+```
+
+### Viewing
+
+```sh
+python -m scope.viewer --basedir ... --port 8000
+```
+
+### Reading
+
+```python
+import scope
+
+reader = scope.Reader(logdir)
+print(reader.keys())               # ('foo', 'bar', 'baz')
+
+print(reader.length('foo'))        # 3
+steps, values = reader['foo']
+print(steps)                       # np.array([0, 1, 2], np.int64)
+print(values)                      # np.array([42, 42, 42], np.float64)
+
+steps, filenames = reader['bar']
+reader.load('bar', filenames[-1])  # np.zeros((100, 640, 360, 3), np.uint8)
+```

scope-0.4.0/requirements.txt

@@ -0,0 +1,6 @@
+av
+elements>=3.15.11
+fastapi
+numpy
+pillow
+uvicorn[standard]

{scope-0.3.2 → scope-0.4.0}/scope/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '0.3.2'
+__version__ = '0.4.0'
 
 from .reader import Reader
 from .writer import Writer

{scope-0.3.2 → scope-0.4.0}/scope/reader.py

@@ -18,7 +18,7 @@ class Reader:
     formats = formats or FORMATS
     if isinstance(logdir, str):
       logdir = pathlib.Path(logdir)
-    self.logdir = logdir
+    self.logdir = logdir / 'scope'
     self.fmts = {x.extension: x for x in formats}
     self.cols = {}
     for child in sorted(logdir.glob('*')):

{scope-0.3.2 → scope-0.4.0}/scope/writer.py

@@ -33,7 +33,7 @@ class Writer:
     formats = formats or FORMATS
     if isinstance(logdir, str):
       logdir = pathlib.Path(logdir)
-    self.logdir = logdir
+    self.logdir = logdir / 'scope'
     self.logdir.mkdir(parents=True, exist_ok=True)
     self.workers = workers
     self.rng = np.random.default_rng(seed=None)

{scope-0.3.2 → scope-0.4.0}/setup.py

@@ -1,6 +1,11 @@
+import os
 import pathlib
 import re
+import subprocess
+
 import setuptools
+from distutils.command.sdist import sdist as _sdist
+from setuptools.command.develop import develop as _develop
 
 
 def parse_requirements(filename):
@@ -16,6 +21,24 @@ def parse_version(filename):
   return version
 
 
+def build_viewer():
+  orig = os.getcwd()
+  try:
+    os.chdir(pathlib.Path(__file__).parent / 'viewer')
+    subprocess.check_call(['npm', 'install'])
+    subprocess.check_call(['npm', 'run', 'build'])
+  finally:
+    os.chdir(orig)
+
+
+def patch(cmd):
+  class Patched(cmd):
+    def run(self):
+      build_viewer()
+      cmd.run(self)
+  return Patched
+
+
 setuptools.setup(
     name='scope',
     version=parse_version('scope/__init__.py'),
@@ -23,9 +46,20 @@ setuptools.setup(
     url='http://github.com/danijar/scope',
     long_description=pathlib.Path('README.md').read_text(),
     long_description_content_type='text/markdown',
-    packages=['scope'],
-    include_package_data=True,
     install_requires=parse_requirements('requirements.txt'),
+    packages=['scope', 'scope.viewer'],
+    package_dir={
+      'scope': 'scope',
+      'scope.viewer': 'viewer',
+    },
+    package_data={
+        'scope.viewer': ['dist/*'],
+    },
+    include_package_data=True,
+    cmdclass={
+        'sdist': patch(_sdist),
+        'develop': patch(_develop),
+    },
     classifiers=[
         'Intended Audience :: Science/Research',
         'License :: OSI Approved :: MIT License',

scope-0.4.0/viewer/__main__.py

@@ -0,0 +1,24 @@
+import os
+import pathlib
+import sys
+
+import elements
+import uvicorn
+
+
+args, sys.argv[1:] = elements.Flags(
+    port=int(os.environ.get('SCOPE_SERVER_PORT', 8000)),
+    workers=32,
+    debug=False,
+).parse_known()
+
+sys.argv.append(f'--debug={args.debug}')
+sys.path.insert(0, str(pathlib.Path(__file__).parent))
+
+uvicorn.run(
+    'server:app',
+    host='0.0.0.0',
+    port=args.port,
+    reload=args.debug,
+    workers=None if args.debug else args.workers,
+)

scope-0.4.0/viewer/dist/index.html

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8">
+    <link rel="icon" href="/favicon.png">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Scope</title>
+    <script type="module" crossorigin src="/assets/index-BZlTxr6y.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/index-DlFPK5at.css">
+  </head>
+  <body>
+    <div id="app"></div>
+  </body>
+</html>

scope-0.4.0/viewer/filesystems.py

@@ -0,0 +1,89 @@
+import elements
+import io
+import os
+import subprocess
+
+
+class Local:
+
+  def list(self, path):
+    return os.listdir(path)
+
+  def size(self, path):
+    return os.path.getsize(path)
+
+  def read(self, path):
+    with open(path, 'rb') as f:
+      return f.read()
+
+  def open(self, path, seek=0, limit=None):
+    f = open(path, 'rb')
+    f.seek(seek)
+    return f
+
+
+class Elements:
+
+  def list(self, path):
+    paths = elements.Path(path).glob('*')
+    return [str(x) for x in paths]
+
+  def size(self, path):
+    return elements.Path(path).size
+
+  def read(self, path):
+    return elements.Path(path).read_bytes()
+
+  def open(self, path, seek=0, limit=None):
+    f = elements.Path(path).open('rb')
+    f.seek(seek)
+    return f
+
+
+class Fileutil:
+
+  def __init__(
+      self,
+      ls='fileutil ls {}',
+      cat='fileutil cat {}',
+      size="fileutil ls -l {} | tr -s ' ' | cut -d' ' -f5",
+      catrange='fileutil cat -input_startpos={} -input_endpos={} {}',
+  ):
+    self._ls = ls
+    self._cat = cat
+    self._size = size
+    self._catrange = catrange
+
+  def list(self, path):
+    try:
+      output = self._sh(self._ls.format(path))
+      return [x.rstrip('/') for x in output.decode('utf-8').splitlines()]
+    except RuntimeError as e:
+      print(e)
+      return []
+
+  def size(self, path):
+    output = self._sh(self._size.format(path))
+    return int(output.decode('utf-8').strip('\n'))
+
+  def read(self, path):
+    return self._sh(self._cat.format(path))
+
+  def open(self, path, seek=0, limit=None):
+    limit = limit or self._size(path)
+    buffer = self._sh(self._catrange.format(seek, limit, path))
+    return io.BytesIO(buffer)
+
+  def _sh(self, cmd):
+    if '|' in cmd:
+      process = subprocess.Popen(
+          cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+      out, err = process.communicate()
+      if process.returncode:
+        raise RuntimeError((process.returncode, out, err))
+      return out
+    else:
+      try:
+        return subprocess.check_output(cmd.split(), shell=False)
+      except subprocess.CalledProcessError as e:
+        raise RuntimeError(f'Error in subprocess: {e}')

scope-0.4.0/viewer/server.py

@@ -0,0 +1,178 @@
+import concurrent.futures
+import functools
+import os
+import pathlib
+import struct
+
+import elements
+import fastapi
+import fastapi.responses
+import fastapi.staticfiles
+
+import filesystems
+
+
+args = elements.Flags(
+    basedir=os.environ.get('SCOPE_ROOT', ''),
+    fs=os.environ.get('SCOPE_FS', 'elements'),
+    maxdepth=2,
+    debug=False,
+).parse()
+print(args)
+assert args.basedir, args.basedir
+basedir = args.basedir.rstrip('/')
+
+
+fs = dict(
+  elements=filesystems.Elements,
+  fileutil=filesystems.Fileutil,
+  local=filesystems.Local,
+)[args.fs]()
+
+
+app = fastapi.FastAPI(debug=args.debug)
+
+
+@app.get('/api/exps')
+def get_exps():
+  print('GET /exps', flush=True)
+  folders = fs.list(basedir)
+  expids = [x.rsplit('/', 1)[-1] for x in folders]
+  return {'exps': expids}
+
+
+@app.get('/api/exp/{expid}')
+def get_exp(expid: str):
+  print(f'GET /exp/{expid}', flush=True)
+  folders = find_runs(basedir + '/' + expid)
+  folders = [x.removeprefix(str(basedir))[1:] for x in folders]
+  runids = [x.replace('/', ':') for x in folders]
+  return {'id': expid, 'runs': runids}
+
+
+@app.get('/api/run/{runid}')
+def get_run(runid: str):
+  print(f'GET /run/{runid}', flush=True)
+  folder = basedir + '/' + runid.replace(':', '/') + '/scope'
+  children = fs.list(folder)
+  children = [x.removeprefix(str(basedir))[1:] for x in children]
+  colids = [x.replace('/', ':') for x in children]
+  return {'id': runid, 'cols': colids}
+
+
+@app.get('/api/col/{colid}')
+def get_col(colid: str):
+  print(f'GET /col/{colid}', flush=True)
+  ext = colid.rsplit('.', 1)[-1]
+  path = basedir + '/' + colid.replace(':', '/')
+  runid = colid.rsplit(':', 2)[0]  # Remove metric name and scope folder.
+  if ext == 'float':
+    buffer = fs.read(path)
+    steps, values = tuple(zip(*struct.iter_unpack('>qd', buffer)))
+    return {'id': colid, 'run': runid, 'steps': steps, 'values': values}
+  elif ext in ('txt', 'mp4', 'webm'):
+    buffer = fs.read(path + '/index')
+    steps, idents = tuple(zip(*struct.iter_unpack('q8s', buffer)))
+    filenames = [f'{s:020}-{x.hex()}.{ext}' for s, x in zip(steps, idents)]
+    values = [f'{colid}:{x}' for x in filenames]
+    return {'id': colid, 'run': runid, 'steps': steps, 'values': values}
+  else:
+    raise NotImplementedError((colid, ext))
+
+
+@app.get('/api/file/{fileid}')
+def get_file(request: fastapi.Request, fileid: str):
+  print(f'GET /file/{fileid}', flush=True)
+  ext = fileid.rsplit('.', 1)[-1]
+  path = basedir + '/' + fileid.replace(':', '/')
+  if ext == 'txt':
+    text = fs.read(path).decode('utf-8')
+    return {'id': fileid, 'text': text}
+  elif ext in ('mp4', 'webm'):
+    filesize = fs.size(path)
+    openfn = functools.partial(fs.open, path)
+    content_type = f'video/{ext}'
+    return RangeResponse(request, openfn, filesize, content_type)
+  else:
+    raise NotImplementedError((fileid, ext))
+
+
+dist = pathlib.Path(__file__).parent / 'dist'
+app.mount('/', fastapi.staticfiles.StaticFiles(directory=dist, html=True))
+
+
+def find_runs(folder, maxdepth=args.maxdepth, workers=64):
+  if not workers:
+    runs = []
+    queue = [(folder, 0)]
+    while queue:
+      node, depth = queue.pop(0)
+      children = fs.list(node)
+      if any(x.endswith('/scope') for x in children):
+        runs.append(node)
+      elif depth < maxdepth:
+        queue += [(x, depth + 1) for x in children]
+    return runs
+  runs = []
+  with concurrent.futures.ThreadPoolExecutor(workers) as pool:
+    future = pool.submit(fs.list, folder)
+    future.parent = folder
+    future.depth = 0
+    queue = [future]
+    while queue:
+      current = queue.pop(0)
+      children = current.result()
+      if any(x.endswith('/scope') for x in children):
+        runs.append(current.parent)
+      elif current.depth < maxdepth:
+        for child in children:
+          future = pool.submit(fs.list, child)
+          future.parent = child
+          future.depth = current.depth + 1
+          queue.append(future)
+  return runs
+
+
+def RangeResponse(request, openfn, filesize, content_type):
+  headers = {
+      'content-type': content_type,
+      'accept-ranges': 'bytes',
+      'content-length': str(filesize),
+      'access-control-expose-headers': (
+          'content-type, accept-ranges, content-length, '
+          'content-range, content-encoding'
+      ),
+  }
+  range_header = request.headers.get('range')
+  if range_header:
+    try:
+      h = range_header.replace('bytes=', '').split('-')
+      start = int(h[0]) if h[0] != '' else 0
+      end = int(h[1]) if h[1] != '' else filesize - 1
+    except ValueError:
+      raise fastapi.HTTPException(
+        fastapi.status.HTTP_416_REQUESTED_RANGE_NOT_SATISFIABLE,
+        detail=f'Invalid request range (Range:{range_header!r})')
+    if start > end or start < 0 or end > filesize - 1:
+      raise fastapi.HTTPException(
+        fastapi.status.HTTP_416_REQUESTED_RANGE_NOT_SATISFIABLE,
+        detail=f'Invalid request range (Range:{range_header!r})')
+    size = end - start + 1
+    headers['content-length'] = str(size)
+    headers['content-range'] = f'bytes {start}-{end}/{filesize}'
+    status_code = fastapi.status.HTTP_206_PARTIAL_CONTENT
+  else:
+    start = 0
+    end = filesize - 1
+    status_code = fastapi.status.HTTP_200_OK
+  stop = end + 1
+  def iterfile(chunksize=int(2e5)):
+    with openfn(start, stop) as f:
+      total = stop - start
+      nbytes = 0
+      while nbytes < total:
+        chunk = f.read(min(chunksize, total - nbytes))
+        nbytes += len(chunk)
+        yield chunk
+  return fastapi.responses.StreamingResponse(
+    iterfile(), headers=headers, status_code=status_code)

scope-0.3.2/LICENSE

@@ -1,19 +0,0 @@
-Copyright (c) 2024 Danijar Hafner
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

scope-0.3.2/MANIFEST.in

@@ -1 +0,0 @@
-include requirements.txt

scope-0.3.2/PKG-INFO

@@ -1,10 +0,0 @@
-Metadata-Version: 2.1
-Name: scope
-Version: 0.3.2
-Summary: Metrics logging and analysis
-Home-page: http://github.com/danijar/scope
-Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
-Description-Content-Type: text/markdown
-License-File: LICENSE

scope-0.3.2/pyproject.toml

@@ -1,5 +0,0 @@
-[tool.pytest.ini_options]
-markers = ['slow']
-addopts = ['--strict-config', '-ra']
-pythonpath = ['.']
-testpaths = ['tests']

scope-0.3.2/requirements.txt

@@ -1,3 +0,0 @@
-av
-numpy
-pillow

scope-0.3.2/scope.egg-info/PKG-INFO

@@ -1,10 +0,0 @@
-Metadata-Version: 2.1
-Name: scope
-Version: 0.3.2
-Summary: Metrics logging and analysis
-Home-page: http://github.com/danijar/scope
-Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
-Description-Content-Type: text/markdown
-License-File: LICENSE

scope-0.3.2/scope.egg-info/SOURCES.txt

@@ -1,18 +0,0 @@
-LICENSE
-MANIFEST.in
-README.md
-pyproject.toml
-requirements.txt
-setup.py
-scope/__init__.py
-scope/formats.py
-scope/reader.py
-scope/writer.py
-scope.egg-info/PKG-INFO
-scope.egg-info/SOURCES.txt
-scope.egg-info/dependency_links.txt
-scope.egg-info/requires.txt
-scope.egg-info/top_level.txt
-tests/test_float.py
-tests/test_image.py
-tests/test_video.py

scope-0.3.2/scope.egg-info/dependency_links.txt

@@ -1 +0,0 @@
-

scope-0.3.2/scope.egg-info/requires.txt

@@ -1,3 +0,0 @@
-av
-numpy
-pillow

scope-0.3.2/scope.egg-info/top_level.txt

@@ -1 +0,0 @@
-scope

scope-0.3.2/setup.cfg

@@ -1,4 +0,0 @@
-[egg_info]
-tag_build = 
-tag_date = 0
-