scope 0.2.0__tar.gz

scope-0.2.0/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.1
+Name: scope
+Version: 0.2.0
+Summary: Metrics logging and analysis.
+Home-page: http://github.com/danijar/scope
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Description-Content-Type: text/markdown

scope-0.2.0/pyproject.toml

@@ -0,0 +1,5 @@
+[tool.pytest.ini_options]
+markers = ['slow']
+addopts = ['--strict-config', '-ra']
+pythonpath = ['.']
+testpaths = ['tests']

scope-0.2.0/scope/__init__.py

@@ -0,0 +1,4 @@
+__version__ = '0.2.0'
+
+from .writer import Writer
+from .reader import Reader

scope-0.2.0/scope/columns.py

@@ -0,0 +1,184 @@
+import io
+import struct
+import time
+
+import av
+import numpy as np
+from PIL import Image
+
+
+def table_length(filename, fmt):
+  return filename.stat().st_size // struct.calcsize(fmt)
+
+
+def table_write(filename, fmt, *cols):
+  rows = tuple(zip(*cols))
+  size = struct.calcsize(fmt)
+  buffer = bytearray(len(rows) * size)
+  for index, row in enumerate(rows):
+    struct.pack_into(fmt, buffer, index * size, *row)
+  with filename.open('ab') as f:
+    f.write(buffer)
+
+
+def table_read(filename, fmt, start=0, stop=None):
+  assert stop is None or start < stop, (start, stop)
+  size = struct.calcsize(fmt)
+  with filename.open('rb') as f:
+    start and f.seek(start * size)
+    buffer = f.read((stop - start) * size if stop else None)
+  rows = struct.iter_unpack(fmt, buffer)
+  cols = tuple(zip(*rows))
+  return cols
+
+
+class FloatColumn:
+
+  def __init__(self, logdir, key):
+    name = key.replace('/', '-') + '.float'
+    self.filename = logdir / name
+
+  def validate(self, value):
+    assert value.dtype in (float, int) and value.ndim == 0, (
+        value.dtype, value.shape)
+    return value
+
+  def write(self, values):
+    steps, values = zip(*values)
+    table_write(self.filename, '>qd', steps, values)
+
+  def length(self):
+    return table_length(self.filename, '>qd')
+
+  def read(self, start, stop):
+    steps, values = table_read(self.filename, '>qd')
+    filtered = [(s, v) for s, v in zip(steps, values) if start <= s < stop]
+    steps, values = zip(*filtered) if filtered else ([], [])
+    steps = np.array(steps, np.int64)
+    values = np.array(values, np.float64)
+    return steps, values
+
+
+class FileColumn:
+
+  def __init__(self, logdir, key, ext, encfn, decfn):
+    name = key.replace('/', '-') + '.' + ext
+    self.folder = logdir / name
+    self.folder.mkdir(exist_ok=True)
+    self.index = self.folder / 'index'
+    self.rng = np.random.default_rng(seed=None)
+    self.ext = ext
+    self.encfn = encfn
+    self.decfn = decfn
+
+  def validate(self, value):
+    raise NotImplementedError
+
+  def write(self, values):
+    prefix = int(time.time()).to_bytes(4, 'big')
+    steps, values = zip(*values)
+    idents = [prefix + self.rng.bytes(4) for _ in range(len(steps))]
+    for ident, step, value in zip(idents, steps, values):
+      buffer = self.encfn(value)
+      with self._filename(step, ident).open('wb') as f:
+        f.write(buffer)
+    table_write(self.index, 'q8s', steps, idents)
+
+  def length(self):
+    return table_length(self.index, 'q8s')
+
+  def read(self, start, stop):
+    steps, idents = table_read(self.index, 'q8s')
+    filtered = [(s, v) for s, v in zip(steps, idents) if start <= s < stop]
+    steps, idents = zip(*filtered) if filtered else ([], [])
+    values = []
+    for step, ident in zip(steps, idents):
+      with self._filename(step, ident).open('rb') as f:
+        buffer = f.read()
+      values.append(self.decfn(buffer))
+    steps = np.array(steps, np.int64)
+    values = tuple(values)
+    return steps, values
+
+  def _filename(self, step, ident):
+    return self.folder / f'{step:020}-{ident.hex()}.{self.ext}'
+
+
+class TextColumn(FileColumn):
+
+  def __init__(self, logdir, key, fmt='txt'):
+    super().__init__(logdir, key, fmt, self.encode, self.decode)
+    self.fmt = fmt
+
+  def validate(self, value):
+    assert isinstance(value, str), type(value)
+    return value
+
+  def encode(self, value):
+    return value.encode('utf-8')
+
+  def decode(self, buffer):
+    return buffer.decode('utf-8')
+
+
+class ImageColumn(FileColumn):
+
+  def __init__(self, logdir, key, fmt='png', quality=None):
+    super().__init__(logdir, key, fmt, self.encode, self.decode)
+    self.fmt = fmt
+    self.quality = quality
+
+  def validate(self, value):
+    assert (
+        value.dtype == np.uint8 and value.ndim == 3 and
+        value.shape[-1] == 3), (value.dtype, value.shape)
+    return value
+
+  def encode(self, value):
+    fmt = ('jpeg' if self.fmt == 'jpg' else self.fmt).upper()
+    fp = io.BytesIO()
+    Image.fromarray(value).save(fp, format=fmt, quality=self.quality)
+    return fp.getvalue()
+
+  def decode(self, buffer):
+    return np.asarray(Image.open(io.BytesIO(buffer)).convert('RGB'))
+
+
+class VideoColumn(FileColumn):
+
+  def __init__(self, logdir, key, fmt='mp4', fps=20, codec='h264'):
+    super().__init__(logdir, key, fmt, self.encode, self.decode)
+    self.fmt = fmt
+    self.fps = fps
+    self.codec = codec
+
+  def validate(self, value):
+    assert (
+        value.dtype == np.uint8 and value.ndim == 4 and
+        value.shape[-1] == 3), (value.dtype, value.shape)
+    return value
+
+  def encode(self, array):
+    T, H, W, C = array.shape
+    fp = io.BytesIO()
+    output = av.open(fp, mode='w', format=self.fmt)
+    stream = output.add_stream(self.codec, rate=float(self.fps))
+    stream.width = W
+    stream.height = H
+    stream.pix_fmt = 'yuv420p'
+    for t in range(T):
+      frame = av.VideoFrame.from_ndarray(array[t], format='rgb24')
+      frame.pts = t
+      output.mux(stream.encode(frame))
+    output.mux(stream.encode(None))
+    output.close()
+    return fp.getvalue()
+
+  def decode(self, buffer):
+    container = av.open(io.BytesIO(buffer))
+    array = []
+    for frame in container.decode(video=0):
+      array.append(frame.to_ndarray(format='rgb24'))
+    array = np.stack(array)
+    container.close()
+    return array

scope-0.2.0/scope/reader.py

@@ -0,0 +1,43 @@
+import re
+import pathlib
+
+import numpy as np
+
+from . import columns
+
+
+class Reader:
+
+  def __init__(self, logdir):
+    if isinstance(logdir, str):
+      logdir = pathlib.Path(logdir)
+    self.coltypes = {
+        'float': columns.FloatColumn,
+        'png': columns.ImageColumn,
+        'mp4': columns.VideoColumn,
+    }
+    self.columns = {}
+    for child in sorted(logdir.glob('*')):
+      name, ext = child.name.rsplit('.', 1)
+      key = name.replace('-', '/')
+      assert re.match(r'[a-z0-9_]+(/[a-z0-9_]+)?', key), key
+      self.columns[key] = self.coltypes[ext](logdir, key)
+
+  def keys(self):
+    return tuple(self.columns.keys())
+
+  def length(self, key):
+    return self.columns[key].length()
+
+  def __getitem__(self, index):
+    if isinstance(index, str):
+      key, start, stop = index, -np.inf, +np.inf
+    else:
+      key, pos = index
+      if isinstance(pos, int):
+        start, stop = pos, pos + 1
+      else:
+        assert pos.step is None
+        start = -np.inf if pos.start is None else pos.start
+        stop = +np.inf if pos.stop is None else pos.stop
+    return self.columns[key].read(start, stop)

scope-0.2.0/scope/writer.py

@@ -0,0 +1,71 @@
+import concurrent.futures
+import pathlib
+import re
+from functools import partial as bind
+
+import numpy as np
+
+from . import columns
+
+
+class Writer:
+
+  def __init__(self, logdir, fps=20, workers=32):
+    if isinstance(logdir, str):
+      logdir = pathlib.Path(logdir)
+    self.logdir = logdir
+    self.logdir.mkdir(parents=True, exist_ok=True)
+    self.fps = fps
+    self.workers = workers
+    self.coltypes = [
+        (lambda x: isinstance(x, str), columns.TextColumn),
+        (lambda x: x.ndim == 0, columns.FloatColumn),
+        (lambda x: x.ndim == 3, bind(columns.ImageColumn, fmt='png')),
+        (lambda x: x.ndim == 4, bind(columns.VideoColumn, fmt='mp4', fps=fps)),
+    ]
+    self.columns = {}
+    self.values = {}
+    if workers:
+      self.pool = concurrent.futures.ThreadPoolExecutor(workers, 'writer')
+      self.futures = []
+
+  def add(self, step, *args, **kwargs):
+    step = int(step)
+    mapping = dict(*args, **kwargs)
+    for key, value in mapping.items():
+      value = value if isinstance(value, str) else np.asarray(value)
+      if key not in self.columns:
+        assert re.match(r'[a-z0-9_]+(/[a-z0-9_]+)?', key), key
+        for applies, coltype in self.coltypes:
+          if applies(value):
+            break
+        else:
+          raise NotImplementedError((
+              key, value,
+              getattr(value, 'shape', None),
+              getattr(value, 'dtype', None)))
+        self.columns[key] = coltype(self.logdir, key)
+        self.values[key] = []
+      column = self.columns[key]
+      try:
+        value = column.validate(value)
+      except Exception:
+        print(f"Error validating key '{key}' with value '{value}'.")
+        raise
+      self.values[key].append((step, value))
+
+  def flush(self):
+    keys = [key for key, values in self.values.items() if values]
+    if self.workers:
+      list(self.futures)
+      columns = [self.columns[x] for x in keys]
+      values = [self.values[x] for x in keys]
+      self.futures = self.pool.map(lambda x, y: x.write(y), columns, values)
+    else:
+      for key in keys:
+        try:
+          self.columns[key].write(self.values[key])
+        except Exception:
+          print(f"Exception writing '{key}' column.")
+          raise
+    self.values = {key: [] for key in self.values.keys()}

scope-0.2.0/scope.egg-info/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.1
+Name: scope
+Version: 0.2.0
+Summary: Metrics logging and analysis.
+Home-page: http://github.com/danijar/scope
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Description-Content-Type: text/markdown

scope-0.2.0/scope.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+README.md
+pyproject.toml
+setup.py
+scope/__init__.py
+scope/columns.py
+scope/reader.py
+scope/writer.py
+scope.egg-info/PKG-INFO
+scope.egg-info/SOURCES.txt
+scope.egg-info/dependency_links.txt
+scope.egg-info/top_level.txt
+tests/test_float.py
+tests/test_image.py
+tests/test_video.py

scope-0.2.0/scope.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

scope-0.2.0/scope.egg-info/top_level.txt

@@ -0,0 +1 @@
+scope

scope-0.2.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

scope-0.2.0/setup.py

@@ -0,0 +1,34 @@
+import pathlib
+import re
+import setuptools
+
+
+def parse_requirements(filename):
+  requirements = pathlib.Path(filename)
+  requirements = requirements.read_text().split('\n')
+  requirements = [x for x in requirements if x.strip()]
+  return requirements
+
+
+def parse_version(filename):
+  text = (pathlib.Path(__file__).parent / filename).read_text()
+  version = re.search(r"__version__ = '(.*)'", text).group(1)
+  return version
+
+
+setuptools.setup(
+    name='scope',
+    version=parse_version('scope/__init__.py'),
+    description='Metrics logging and analysis.',
+    url='http://github.com/danijar/scope',
+    long_description=pathlib.Path('README.md').read_text(),
+    long_description_content_type='text/markdown',
+    packages=['scope'],
+    include_package_data=True,
+    install_requires=parse_requirements('requirements.txt'),
+    classifiers=[
+        'Intended Audience :: Science/Research',
+        'License :: OSI Approved :: MIT License',
+        'Programming Language :: Python :: 3',
+    ],
+)

scope-0.2.0/tests/test_float.py

@@ -0,0 +1,70 @@
+import pathlib
+
+import scope
+import numpy as np
+
+
+class TestFloat:
+
+  def test_roundtrip(self, tmpdir):
+    logdir = pathlib.Path(tmpdir)
+    writer = scope.Writer(logdir, workers=0)
+    writer.add(0, {'foo': 12})
+    writer.add(5, {'foo': 42, 'bar': np.float64(np.pi)})
+    writer.flush()
+    assert {x.name for x in logdir.glob('*')} == {'foo.float', 'bar.float'}
+    assert (logdir / 'foo.float').stat().st_size == (8 + 8) * 2
+    assert (logdir / 'bar.float').stat().st_size == (8 + 8) * 1
+    reader = scope.Reader(logdir)
+    assert reader.keys() == tuple(sorted(['foo', 'bar']))
+    assert reader.length('foo') == 2
+    assert reader.length('bar') == 1
+    assert equal(reader['foo'], ([0, 5], [12, 42]), (np.int64, np.float64))
+    assert equal(reader['bar'], ([5], [np.pi]), (np.int64, np.float64))
+
+  def test_slicing(self, tmpdir):
+    logdir = pathlib.Path(tmpdir)
+    writer = scope.Writer(logdir, workers=0)
+    writer.add(0, {'foo': 12})
+    writer.add(5, {'foo': 42})
+    writer.flush()
+    reader = scope.Reader(logdir)
+    assert equal(reader['foo', 0], ([0], [12]))
+    assert equal(reader['foo', :2], ([0], [12]))
+    assert equal(reader['foo', :5], ([0], [12]))
+    assert equal(reader['foo', :6], ([0, 5], [12, 42]))
+    assert equal(reader['foo', 1:6], ([5], [42]))
+    assert equal(reader['foo', :-1], ([], []))
+    assert equal(reader['foo', 7:], ([], []))
+
+  def test_workers(self, tmpdir):
+    logdir = pathlib.Path(tmpdir)
+    writer = scope.Writer(logdir, workers=8)
+    for step in range(10):
+      writer.add(step, {'foo': step, 'bar': step})
+    writer.flush()
+    writer.flush()  # Block until previous flush is done.
+    assert {x.name for x in logdir.glob('*')} == {'foo.float', 'bar.float'}
+    assert (logdir / 'foo.float').stat().st_size == (8 + 8) * 10
+    assert (logdir / 'bar.float').stat().st_size == (8 + 8) * 10
+    reader = scope.Reader(logdir)
+    assert equal(reader['foo'], (np.arange(10), np.arange(10)))
+    assert equal(reader['bar'], (np.arange(10), np.arange(10)))
+
+  def test_namescopes(self, tmpdir):
+    logdir = pathlib.Path(tmpdir)
+    writer = scope.Writer(logdir, workers=0)
+    writer.add(0, {'foo/bar': 12})
+    writer.flush()
+    assert {x.name for x in logdir.glob('*')} == {'foo-bar.float'}
+    reader = scope.Reader(logdir)
+    assert reader.keys() == ('foo/bar',)
+    assert reader.length('foo/bar') == 1
+    assert equal(reader['foo/bar'], ([0], [12]), (np.int64, np.float64))
+
+
+def equal(actuals, references, dtypes=None):
+  dtypes = dtypes or [x.dtype for x in actuals]
+  assert len(actuals) == len(references) == len(dtypes)
+  references = [np.asarray(x, d) for x, d in zip(actuals, dtypes)]
+  return all((x == y).all() for x, y in zip(actuals, references))

scope-0.2.0/tests/test_image.py

@@ -0,0 +1,84 @@
+import pathlib
+
+import scope
+import numpy as np
+
+
+class TestImage:
+
+  def test_roundtrip(self, tmpdir):
+    logdir = pathlib.Path(tmpdir)
+    writer = scope.Writer(logdir, workers=0)
+    img1 = np.ones((64, 128, 3), np.uint8) + 12
+    img2 = np.ones((64, 128, 3), np.uint8) + 255
+    writer.add(0, {'foo': img1})
+    writer.add(5, {'foo': img2})
+    writer.flush()
+    assert {x.name for x in logdir.glob('*')} == {'foo.png'}
+    assert (logdir / 'foo.png' / 'index').stat().st_size == (8 + 8) * 2
+    assert len(list((logdir / 'foo.png').glob('*'))) == 1 + 2
+    reader = scope.Reader(logdir)
+    assert reader.keys() == ('foo',)
+    assert reader.length('foo') == 2
+    steps, values = reader['foo']
+    assert (steps == np.array([0, 5])).all()
+    assert (values == np.array([img1, img2])).all()
+
+  def test_slicing(self, tmpdir):
+    logdir = pathlib.Path(tmpdir)
+    writer = scope.Writer(logdir, workers=0)
+    img1 = np.ones((64, 128, 3), np.uint8) + 12
+    img2 = np.ones((64, 128, 3), np.uint8) + 255
+    writer.add(0, {'foo': img1})
+    writer.add(5, {'foo': img2})
+    writer.flush()
+    assert {x.name for x in logdir.glob('*')} == {'foo.png'}
+    assert (logdir / 'foo.png' / 'index').stat().st_size == (8 + 8) * 2
+    reader = scope.Reader(logdir)
+    assert reader.keys() == ('foo',)
+    assert reader.length('foo') == 2
+    steps, values = reader['foo']
+    assert (steps == np.array([0, 5])).all()
+    assert (values == np.array([img1, img2])).all()
+    assert (reader['foo', 0][1] == img1[None]).all()
+    assert (reader['foo', :5][1] == img1[None]).all()
+    assert (reader['foo', :6][1] == np.array([img1, img2])).all()
+    assert (reader['foo', 1:6][1] == img2[None]).all()
+    assert reader['foo', :-1][1] == ()
+    assert reader['foo', 6:][1] == ()
+
+  def test_workers(self, tmpdir):
+    logdir = pathlib.Path(tmpdir)
+    writer = scope.Writer(logdir, workers=8)
+    for step in range(5):
+      for key in ('foo', 'bar', 'baz'):
+        writer.add(step, {key: np.full((64, 128, 3), step, np.uint8)})
+    writer.flush()
+    writer.flush()  # Block until previous flush is done.
+    assert {x.name for x in logdir.glob('*')} == {
+        'foo.png', 'bar.png', 'baz.png'}
+    for key in ('foo', 'bar', 'baz'):
+      assert (logdir / f'{key}.png' / 'index').stat().st_size == (8 + 8) * 5
+      assert len(list((logdir / f'{key}.png').glob('*'))) == 1 + 5
+    reader = scope.Reader(logdir)
+    assert reader.keys() == tuple(sorted(['foo', 'bar', 'baz']))
+    for key in ('foo', 'bar', 'baz'):
+      assert reader.length(key) == 5
+      steps, values = reader[key]
+      assert (steps == np.arange(5)).all()
+      assert all(x.dtype == np.uint8 for x in values)
+      reference = np.arange(5, dtype=np.uint8)[:, None, None, None]
+      assert (np.array(values) == reference).all()
+
+  def test_namescopes(self, tmpdir):
+    img = np.ones((64, 128, 3), np.uint8) + 12
+    logdir = pathlib.Path(tmpdir)
+    writer = scope.Writer(logdir, workers=0)
+    writer.add(0, {'foo/bar': img})
+    writer.flush()
+    assert {x.name for x in logdir.glob('*')} == {'foo-bar.png'}
+    assert len(list((logdir / 'foo-bar.png').glob('*'))) == 1 + 1
+    reader = scope.Reader(logdir)
+    assert reader.keys() == ('foo/bar',)
+    assert reader.length('foo/bar') == 1
+    assert (reader['foo/bar'][1] == img).all()

scope-0.2.0/tests/test_video.py

@@ -0,0 +1,25 @@
+import pathlib
+
+import scope
+import numpy as np
+
+
+class TestVideo:
+
+  def test_roundtrip(self, tmpdir):
+    logdir = pathlib.Path(tmpdir)
+    writer = scope.Writer(logdir, workers=0)
+    vid1 = np.ones((5, 64, 128, 3), np.uint8) + 12
+    vid2 = np.ones((5, 64, 128, 3), np.uint8) + 255
+    writer.add(0, {'foo': vid1})
+    writer.add(5, {'foo': vid2})
+    writer.flush()
+    assert {x.name for x in logdir.glob('*')} == {'foo.mp4'}
+    assert (logdir / 'foo.mp4' / 'index').stat().st_size == (8 + 8) * 2
+    assert len(list((logdir / 'foo.mp4').glob('*'))) == 1 + 2
+    reader = scope.Reader(logdir)
+    assert reader.keys() == ('foo',)
+    assert reader.length('foo') == 2
+    steps, values = reader['foo']
+    assert (steps == np.array([0, 5])).all()
+    assert np.allclose(values, [vid1, vid2], rtol=0.1)