scope 0.4.4__tar.gz → 0.4.5__tar.gz

{scope-0.4.4 → scope-0.4.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scope
-Version: 0.4.4
+Version: 0.4.5
 Summary: Metrics logging and analysis
 Home-page: http://github.com/danijar/scope
 Classifier: Intended Audience :: Science/Research
@@ -17,10 +17,10 @@ Scalable metrics logging and analysis.
 
 - 🚀 **Scalable:** Quickly log and view petabytes of metrics, thousands of keys, and large videos.
 - 🎞️ **Formats:** Log and view scalars, text, images, and videos. Easy to extend with custom formats.
-- 🧑🏻‍🔬 **Productivity:** Metrics viewer with focus on power users and full keyboard support.
+- 🧑🏻‍🔬 **Productivity:** Metrics viewer with focus on power users with full keyboard support.
 - ☁️ **Cloud support:** Directly write to and read from Cloud storage via pathlib interface.
 - 🍃 **Lightweight:** The writer and reader measure only ~400 lines of Python code.
-- 🧱 **Reliability:** Unit tested and used across diverse research projects.
+- 🧱 **Reliable:** Unit tested and used across diverse research projects.
 
 ## Usage
 
@@ -36,10 +36,14 @@ pip install scope
 import scope
 
 writer = scope.Writer(logdir)
-for step in range(3)
-  video = np.zeros((100, 640, 360, 3), np.uint8)
-  writer.add(step, {'foo': 42, 'bar': video, 'baz': 'Hello World'})
-writer.flush()
+
+for step in range(3):
+  writer.add(step, {
+      'foo': 42,
+      'bar': np.zeros((100, 640, 360, 3), np.uint8),
+      'baz': 'Hello World',
+  })
+  writer.flush()
 ```
 
 ### Viewing

{scope-0.4.4 → scope-0.4.5}/README.md

@@ -1,55 +1,59 @@
-# 🔬 Scope
-
-Scalable metrics logging and analysis.
-
-## Features
-
-- 🚀 **Scalable:** Quickly log and view petabytes of metrics, thousands of keys, and large videos.
-- 🎞️ **Formats:** Log and view scalars, text, images, and videos. Easy to extend with custom formats.
-- 🧑🏻‍🔬 **Productivity:** Metrics viewer with focus on power users and full keyboard support.
-- ☁️ **Cloud support:** Directly write to and read from Cloud storage via pathlib interface.
-- 🍃 **Lightweight:** The writer and reader measure only ~400 lines of Python code.
-- 🧱 **Reliability:** Unit tested and used across diverse research projects.
-
-## Usage
-
-### Installation
-
-```sh
-pip install scope
-```
-
-### Writing
-
-```python
-import scope
-
-writer = scope.Writer(logdir)
-for step in range(3)
-  video = np.zeros((100, 640, 360, 3), np.uint8)
-  writer.add(step, {'foo': 42, 'bar': video, 'baz': 'Hello World'})
-writer.flush()
-```
-
-### Viewing
-
-```sh
-python -m scope.viewer --basedir ... --port 8000
-```
-
-### Reading
-
-```python
-import scope
-
-reader = scope.Reader(logdir)
-print(reader.keys())               # ('foo', 'bar', 'baz')
-
-print(reader.length('foo'))        # 3
-steps, values = reader['foo']
-print(steps)                       # np.array([0, 1, 2], np.int64)
-print(values)                      # np.array([42, 42, 42], np.float64)
-
-steps, filenames = reader['bar']
-reader.load('bar', filenames[-1])  # np.zeros((100, 640, 360, 3), np.uint8)
-```
+# 🔬 Scope
+
+Scalable metrics logging and analysis.
+
+## Features
+
+- 🚀 **Scalable:** Quickly log and view petabytes of metrics, thousands of keys, and large videos.
+- 🎞️ **Formats:** Log and view scalars, text, images, and videos. Easy to extend with custom formats.
+- 🧑🏻‍🔬 **Productivity:** Metrics viewer with focus on power users with full keyboard support.
+- ☁️ **Cloud support:** Directly write to and read from Cloud storage via pathlib interface.
+- 🍃 **Lightweight:** The writer and reader measure only ~400 lines of Python code.
+- 🧱 **Reliable:** Unit tested and used across diverse research projects.
+
+## Usage
+
+### Installation
+
+```sh
+pip install scope
+```
+
+### Writing
+
+```python
+import scope
+
+writer = scope.Writer(logdir)
+
+for step in range(3):
+  writer.add(step, {
+      'foo': 42,
+      'bar': np.zeros((100, 640, 360, 3), np.uint8),
+      'baz': 'Hello World',
+  })
+  writer.flush()
+```
+
+### Viewing
+
+```sh
+python -m scope.viewer --basedir ... --port 8000
+```
+
+### Reading
+
+```python
+import scope
+
+reader = scope.Reader(logdir)
+print(reader.keys())               # ('foo', 'bar', 'baz')
+
+print(reader.length('foo'))        # 3
+steps, values = reader['foo']
+print(steps)                       # np.array([0, 1, 2], np.int64)
+print(values)                      # np.array([42, 42, 42], np.float64)
+
+steps, filenames = reader['bar']
+reader.load('bar', filenames[-1])  # np.zeros((100, 640, 360, 3), np.uint8)
+```

{scope-0.4.4 → scope-0.4.5}/scope/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '0.4.4'
+__version__ = '0.4.5'
 
 from .reader import Reader
 from .writer import Writer

{scope-0.4.4 → scope-0.4.5}/viewer/config.py

@@ -7,6 +7,8 @@ config = elements.Flags(
     port=int(os.environ.get('SCOPE_PORT', 8000)),
     basedir=os.environ.get('SCOPE_BASEDIR', ''),
     filesystem=os.environ.get('SCOPE_FILESYSTEM', 'elements'),
+    cachedir='/tmp/scope-cache',
+    cachesize=int(4e9),  # 4 GB
     maxdepth=2,
     workers=32,
     debug=False,

{scope-0.4.4 → scope-0.4.5}/viewer/filesystems.py

@@ -1,8 +1,11 @@
-import elements
+import collections
 import io
 import os
+import pathlib
 import subprocess
 
+import elements
+
 
 class Local:
 
@@ -87,3 +90,58 @@ class Fileutil:
         return subprocess.check_output(cmd.split(), shell=False)
       except subprocess.CalledProcessError as e:
         raise RuntimeError(f'Error in subprocess: {e}')
+
+
+class WithFileCache:
+
+  def __init__(self, fs, cachedir, maxsize):
+    # Cache size can be exceeded if multiple workers download in parallel.
+    assert not isinstance(fs, Local)
+    if isinstance(cachedir, str):
+      cachedir = pathlib.Path(cachedir)
+    cachedir.mkdir(exist_ok=True, parents=True)
+    self.fs = fs
+    self.localfs = Local()
+    self.cachedir = cachedir
+    self.maxsize = maxsize
+
+  def list(self, path):
+    return self.fs.list(path)
+
+  def size(self, path):
+    localpath = self._getfile(path)
+    return self.localfs.size(localpath)
+
+  def read(self, path):
+    localpath = self._getfile(path)
+    return self.localfs.read(localpath)
+
+  def open(self, path, seek=0, limit=None):
+    localpath = self._getfile(path)
+    return self.localfs.open(localpath, seek, limit)
+
+  def _getfile(self, path):
+    name = str(path).replace(':', '').replace('//', '/').replace('/', ':')
+    localpath = self.cachedir / name
+    if not localpath.exists():
+      buffer = self.fs.read(path)
+      self._freeup(len(buffer), path)
+      localpath.write_bytes(buffer)
+    return localpath
+
+  def _freeup(self, needed, path):
+    # Catch errors because parallel workers share the cache.
+    pairs = []
+    for path in self.cachedir.glob('*'):
+      try:
+        stat = path.stat()
+        pairs.append((path, stat))
+      except FileNotFoundError:
+        pass
+    pairs = sorted(pairs, key=lambda x: x[1].st_ctime)
+    while sum([s.st_size for p, s in pairs]) + needed > self.maxsize:
+      path, _ = pairs.pop(0)
+      try:
+        path.unlink()
+      except IndexError:
+        pass

{scope-0.4.4 → scope-0.4.5}/viewer/server.py

@@ -20,6 +20,10 @@ fs = dict(
   local=filesystems.Local,
 )[config.filesystem]()
 
+is_local = isinstance(fs, filesystems.Local)
+if config.cachedir and config.cachesize and not is_local:
+  fs = filesystems.WithFileCache(fs, config.cachedir, config.cachesize)
+
 
 @app.get('/api/exps')
 def get_exps():