scope 0.2.2__tar.gz → 0.3.0__tar.gz

{scope-0.2.2/scope.egg-info → scope-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scope
-Version: 0.2.2
+Version: 0.3.0
 Summary: Metrics logging and analysis
 Home-page: http://github.com/danijar/scope
 Classifier: Intended Audience :: Science/Research

scope-0.3.0/scope/__init__.py

@@ -0,0 +1,9 @@
+__version__ = '0.3.0'
+
+from .reader import Reader
+from .writer import Writer
+
+from .formats import table_read
+from .formats import table_append
+
+from . import formats

scope-0.3.0/scope/formats.py

@@ -0,0 +1,210 @@
+import io
+import struct
+import time
+
+import av
+import numpy as np
+import PIL.Image
+
+
+class Float:
+
+  @property
+  def extension(self):
+    return 'float'
+
+  def valid(self, x):
+    return x.ndim == 0 and np.isreal(x)
+
+  def create(self, path):
+    pass
+
+  def write(self, path, steps, values):
+    table_append(path, '>qd', steps, values)
+
+  def read(self, path):
+    steps, values = table_read(path, '>qd')
+    steps = np.int64(steps)
+    values = np.float64(values)
+    return steps, values
+
+  def length(self, path):
+    return table_length(path, '>qd')
+
+
+class Text:
+
+  @property
+  def extension(self):
+    return 'txt'
+
+  def valid(self, x):
+    return isinstance(x, str)
+
+  def create(self, path):
+    path.mkdir(exist_ok=True)
+
+  def write(self, path, steps, values):
+    files_write(path, steps, values, self.encode)
+
+  def read(self, path):
+    return files_read(path)
+
+  def encode(self, value):
+    return value.encode('utf-8')
+
+  def decode(self, buffer):
+    return buffer.deceode('utf-8')
+
+  def length(self, path):
+    return files_length(path)
+
+
+class Image:
+
+  def __init__(self, ext='png', quality=90):
+    self.ext = ext
+    self.quality = quality
+
+  @property
+  def extension(self):
+    return self.ext
+
+  def valid(self, x):
+    return (
+        x.dtype == np.uint8 and
+        x.ndim == 3 and
+        x.shape[-1] in (1, 3))
+
+  def create(self, path):
+    path.mkdir(exist_ok=True)
+
+  def write(self, path, steps, values):
+    files_write(path, steps, values, self.encode)
+
+  def read(self, path):
+    return files_read(path)
+
+  def encode(self, value):
+    if value.shape[-1] == 1:
+      value = value.repeat(3, -1)
+    fmt = ('jpeg' if self.ext == 'jpg' else self.ext).upper()
+    fp = io.BytesIO()
+    PIL.Image.fromarray(value).save(fp, fmt, quality=self.quality)
+    return fp.getvalue()
+
+  def decode(self, buffer):
+    return np.asarray(PIL.Image.open(io.BytesIO(buffer)).convert('RGB'))
+
+  def length(self, path):
+    return files_length(path)
+
+
+class Video:
+
+  def __init__(self, ext='mp4', codec='h264', fps=10):
+    self.ext = ext
+    self.codec = codec
+    self.fps = fps
+
+  @property
+  def extension(self):
+    return self.ext
+
+  def valid(self, x):
+    return (
+        x.dtype == np.uint8 and
+        x.ndim == 4 and
+        x.shape[-1] in (1, 3))
+
+  def create(self, path):
+    path.mkdir(exist_ok=True)
+
+  def write(self, path, steps, values):
+    files_write(path, steps, values, self.encode)
+
+  def read(self, path):
+    return files_read(path)
+
+  def encode(self, value):
+    if value.shape[-1] == 1:
+      value = value.repeat(3, -1)
+    T, H, W, _ = value.shape
+    fp = io.BytesIO()
+    output = av.open(fp, mode='w', format=self.ext)
+    stream = output.add_stream(self.codec, rate=float(self.fps))
+    stream.width = W
+    stream.height = H
+    stream.pix_fmt = 'yuv420p'
+    for t in range(T):
+      frame = av.VideoFrame.from_ndarray(value[t], format='rgb24')
+      frame.pts = t
+      output.mux(stream.encode(frame))
+    output.mux(stream.encode(None))
+    output.close()
+    return fp.getvalue()
+
+  def decode(self, buffer):
+    container = av.open(io.BytesIO(buffer))
+    value = []
+    for frame in container.decode(video=0):
+      value.append(frame.to_ndarray(format='rgb24'))
+    value = np.stack(value)
+    container.close()
+    return value
+
+  def length(self, path):
+    return files_length(path)
+
+
+def table_append(filename, fmt, *cols):
+  rows = tuple(zip(*cols))
+  size = struct.calcsize(fmt)
+  buffer = bytearray(len(rows) * size)
+  for index, row in enumerate(rows):
+    struct.pack_into(fmt, buffer, index * size, *row)
+  with filename.open('ab') as f:
+    f.write(buffer)
+
+
+def table_read(filename, fmt, start=0, stop=None):
+  assert stop is None or start < stop, (start, stop)
+  if start == 0 and stop is None:
+    buffer = filename.read_bytes()
+  else:
+    size = struct.calcsize(fmt)
+    with filename.open('rb') as f:
+      start and f.seek(start * size)
+      buffer = f.read((stop - start) * size if stop else None)
+  rows = struct.iter_unpack(fmt, buffer)
+  cols = tuple(zip(*rows))
+  return cols
+
+
+def table_length(filename, fmt):
+  return filename.stat().st_size // struct.calcsize(fmt)
+
+
+def files_write(path, steps, values, encode):
+  rng = np.random.default_rng(seed=None)
+  prefix = int(time.time()).to_bytes(4, 'big')
+  idents = [prefix + rng.bytes(4) for _ in range(len(steps))]
+  for ident, step, value in zip(idents, steps, values):
+    filename = f'{step:020}-{ident.hex()}{path.suffix}'
+    buffer = encode(value)
+    with (path / filename).open('wb') as f:
+      f.write(buffer)
+  table_append(path / 'index', 'q8s', steps, idents)
+
+
+def files_read(path):
+  steps, idents = table_read(path / 'index', 'q8s')
+  filenames = [
+      path / f'{step:020}-{ident.hex()}{path.suffix}'
+      for step, ident in zip(steps, idents)]
+  steps = np.int64(steps)
+  return steps, filenames
+
+
+def files_length(path):
+  return table_length(path / 'index', 'q8s')

scope-0.3.0/scope/reader.py

@@ -0,0 +1,44 @@
+import re
+import pathlib
+
+from . import formats
+
+
+FORMATS = [
+    formats.Text(),
+    formats.Float(),
+    formats.Image(),
+    formats.Video(),
+]
+
+
+class Reader:
+
+  def __init__(self, logdir, formats=FORMATS):
+    if isinstance(logdir, str):
+      logdir = pathlib.Path(logdir)
+    self.logdir = logdir
+    self.fmts = {x.extension: x for x in formats}
+    self.cols = {}
+    for child in sorted(logdir.glob('*')):
+      basename, ext = child.name.rsplit('.', 1)
+      key = basename.replace('-', '/')
+      assert re.match(r'[a-z0-9_]+(/[a-z0-9_]+)?', key), key
+      self.cols[key] = (child.name, self.fmts[ext])
+
+  def keys(self):
+    return tuple(self.cols.keys())
+
+  def __getitem__(self, key):
+    name, fmt = self.cols[key]
+    return fmt.read(self.logdir / name)
+
+  def length(self, key):
+    name, fmt = self.cols[key]
+    return fmt.length(self.logdir / name)
+
+  def load(self, key, filename):
+    _, fmt = self.cols[key]
+    buffer = (self.logdir / filename).read_bytes()
+    value = fmt.decode(buffer)
+    return value

scope-0.3.0/scope/writer.py

@@ -0,0 +1,96 @@
+import concurrent.futures
+import dataclasses
+import pathlib
+import re
+
+import numpy as np
+
+from . import formats
+
+
+FORMATS = [
+    formats.Text(),
+    formats.Float(),
+    formats.Image(),
+    formats.Video(),
+]
+
+
+@dataclasses.dataclass
+class Column:
+
+  fmt: str
+  name: str
+  created: bool
+  steps: list
+  values: list
+
+
+class Writer:
+
+  def __init__(self, logdir, fps=20, workers=8, formats=FORMATS):
+    if isinstance(logdir, str):
+      logdir = pathlib.Path(logdir)
+    self.logdir = logdir
+    self.logdir.mkdir(parents=True, exist_ok=True)
+    self.fps = fps
+    self.workers = workers
+    self.rng = np.random.default_rng(seed=None)
+    self.fmts = FORMATS
+    self.cols = {}
+    if workers:
+      self.pool = concurrent.futures.ThreadPoolExecutor(workers, 'scope')
+      self.futures = []
+
+  def add(self, step, *args, **kwargs):
+    assert isinstance(step, (int, np.integer)), type(step)
+    step = int(step)
+    mapping = dict(*args, **kwargs)
+    for key, value in mapping.items():
+      if not isinstance(value, str):
+        value = np.asarray(value)
+      if key not in self.cols:
+        assert re.match(r'[a-z0-9_]+(/[a-z0-9_]+)?', key), key
+        for fmt in self.fmts:
+          if fmt.valid(value):
+            break
+        else:
+          raise NotImplementedError(
+              f"No format supports key '{key}' with {self._info(value)}")
+        name = key.replace('/', '-') + '.' + fmt.extension
+        self.cols[key] = Column(fmt, name, False, [], [])
+      col = self.cols[key]
+      if not col.fmt.valid(value):
+        raise ValueError(
+            f"Key '{key}' contains invalid value {self._info(value)}")
+      col.steps.append(step)
+      col.values.append(value)
+
+  def flush(self):
+    if self.workers:
+      list(self.futures)
+      jobs = [(c, c.steps, c.values) for c in self.cols.values() if c.steps]
+      self.futures = self.pool.map(self._write, *zip(*jobs))
+    else:
+      for col in self.cols.values():
+        if col.steps:
+          self._write(col, col.steps, col.values)
+    for col in self.cols.values():
+      col.steps = []
+      col.values = []
+
+  def _write(self, col, steps, values):
+    try:
+      path = self.logdir / col.name
+      if not col.created:
+        col.fmt.create(path)
+        col.created = True
+      col.fmt.write(path, steps, values)
+    except Exception:
+      print(f"Exception writing '{col.name}' column")
+      raise
+
+  def _info(self, value):
+    if hasattr(value, 'dtype') and hasattr(value, 'shape'):
+      return f"dtype '{value.dtype}' and shape '{value.shape}'"
+    return f"type '{type(value)}'"

{scope-0.2.2 → scope-0.3.0/scope.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scope
-Version: 0.2.2
+Version: 0.3.0
 Summary: Metrics logging and analysis
 Home-page: http://github.com/danijar/scope
 Classifier: Intended Audience :: Science/Research

{scope-0.2.2 → scope-0.3.0}/scope.egg-info/SOURCES.txt

@@ -4,7 +4,7 @@ pyproject.toml
 requirements.txt
 setup.py
 scope/__init__.py
-scope/columns.py
+scope/formats.py
 scope/reader.py
 scope/writer.py
 scope.egg-info/PKG-INFO

{scope-0.2.2 → scope-0.3.0}/tests/test_float.py

@@ -22,21 +22,6 @@ class TestFloat:
     assert equal(reader['foo'], ([0, 5], [12, 42]), (np.int64, np.float64))
     assert equal(reader['bar'], ([5], [np.pi]), (np.int64, np.float64))
 
-  def test_slicing(self, tmpdir):
-    logdir = pathlib.Path(tmpdir)
-    writer = scope.Writer(logdir, workers=0)
-    writer.add(0, {'foo': 12})
-    writer.add(5, {'foo': 42})
-    writer.flush()
-    reader = scope.Reader(logdir)
-    assert equal(reader['foo', 0], ([0], [12]))
-    assert equal(reader['foo', :2], ([0], [12]))
-    assert equal(reader['foo', :5], ([0], [12]))
-    assert equal(reader['foo', :6], ([0, 5], [12, 42]))
-    assert equal(reader['foo', 1:6], ([5], [42]))
-    assert equal(reader['foo', :-1], ([], []))
-    assert equal(reader['foo', 7:], ([], []))
-
   def test_workers(self, tmpdir):
     logdir = pathlib.Path(tmpdir)
     writer = scope.Writer(logdir, workers=8)
@@ -62,6 +47,21 @@ class TestFloat:
     assert reader.length('foo/bar') == 1
     assert equal(reader['foo/bar'], ([0], [12]), (np.int64, np.float64))
 
+  # def test_slicing(self, tmpdir):
+  #   logdir = pathlib.Path(tmpdir)
+  #   writer = scope.Writer(logdir, workers=0)
+  #   writer.add(0, {'foo': 12})
+  #   writer.add(5, {'foo': 42})
+  #   writer.flush()
+  #   reader = scope.Reader(logdir)
+  #   assert equal(reader['foo', 0], ([0], [12]))
+  #   assert equal(reader['foo', :2], ([0], [12]))
+  #   assert equal(reader['foo', :5], ([0], [12]))
+  #   assert equal(reader['foo', :6], ([0, 5], [12, 42]))
+  #   assert equal(reader['foo', 1:6], ([5], [42]))
+  #   assert equal(reader['foo', :-1], ([], []))
+  #   assert equal(reader['foo', 7:], ([], []))
+
 
 def equal(actuals, references, dtypes=None):
   dtypes = dtypes or [x.dtype for x in actuals]

{scope-0.2.2 → scope-0.3.0}/tests/test_image.py

@@ -20,32 +20,10 @@ class TestImage:
     reader = scope.Reader(logdir)
     assert reader.keys() == ('foo',)
     assert reader.length('foo') == 2
-    steps, values = reader['foo']
+    steps, filenames = reader['foo']
+    values = [reader.load('foo', x) for x  in filenames]
     assert (steps == np.array([0, 5])).all()
-    assert (values == np.array([img1, img2])).all()
-
-  def test_slicing(self, tmpdir):
-    logdir = pathlib.Path(tmpdir)
-    writer = scope.Writer(logdir, workers=0)
-    img1 = np.ones((64, 128, 3), np.uint8) + 12
-    img2 = np.ones((64, 128, 3), np.uint8) + 255
-    writer.add(0, {'foo': img1})
-    writer.add(5, {'foo': img2})
-    writer.flush()
-    assert {x.name for x in logdir.glob('*')} == {'foo.png'}
-    assert (logdir / 'foo.png' / 'index').stat().st_size == (8 + 8) * 2
-    reader = scope.Reader(logdir)
-    assert reader.keys() == ('foo',)
-    assert reader.length('foo') == 2
-    steps, values = reader['foo']
-    assert (steps == np.array([0, 5])).all()
-    assert (values == np.array([img1, img2])).all()
-    assert (reader['foo', 0][1] == img1[None]).all()
-    assert (reader['foo', :5][1] == img1[None]).all()
-    assert (reader['foo', :6][1] == np.array([img1, img2])).all()
-    assert (reader['foo', 1:6][1] == img2[None]).all()
-    assert reader['foo', :-1][1] == ()
-    assert reader['foo', 6:][1] == ()
+    assert (np.array(values) == np.array([img1, img2])).all()
 
   def test_workers(self, tmpdir):
     logdir = pathlib.Path(tmpdir)
@@ -64,7 +42,8 @@ class TestImage:
     assert reader.keys() == tuple(sorted(['foo', 'bar', 'baz']))
     for key in ('foo', 'bar', 'baz'):
       assert reader.length(key) == 5
-      steps, values = reader[key]
+      steps, filenames = reader[key]
+      values = [reader.load(key, x) for x  in filenames]
       assert (steps == np.arange(5)).all()
       assert all(x.dtype == np.uint8 for x in values)
       reference = np.arange(5, dtype=np.uint8)[:, None, None, None]
@@ -81,4 +60,29 @@ class TestImage:
     reader = scope.Reader(logdir)
     assert reader.keys() == ('foo/bar',)
     assert reader.length('foo/bar') == 1
-    assert (reader['foo/bar'][1] == img).all()
+    _, filenames = reader['foo/bar']
+    assert len(filenames) == 1
+    assert (reader.load('foo/bar', filenames[0]) == img).all()
+
+  # def test_slicing(self, tmpdir):
+  #   logdir = pathlib.Path(tmpdir)
+  #   writer = scope.Writer(logdir, workers=0)
+  #   img1 = np.ones((64, 128, 3), np.uint8) + 12
+  #   img2 = np.ones((64, 128, 3), np.uint8) + 255
+  #   writer.add(0, {'foo': img1})
+  #   writer.add(5, {'foo': img2})
+  #   writer.flush()
+  #   assert {x.name for x in logdir.glob('*')} == {'foo.png'}
+  #   assert (logdir / 'foo.png' / 'index').stat().st_size == (8 + 8) * 2
+  #   reader = scope.Reader(logdir)
+  #   assert reader.keys() == ('foo',)
+  #   assert reader.length('foo') == 2
+  #   steps, values = reader['foo']
+  #   assert (steps == np.array([0, 5])).all()
+  #   assert (values == np.array([img1, img2])).all()
+  #   assert (reader['foo', 0][1] == img1[None]).all()
+  #   assert (reader['foo', :5][1] == img1[None]).all()
+  #   assert (reader['foo', :6][1] == np.array([img1, img2])).all()
+  #   assert (reader['foo', 1:6][1] == img2[None]).all()
+  #   assert reader['foo', :-1][1] == ()
+  #   assert reader['foo', 6:][1] == ()

{scope-0.2.2 → scope-0.3.0}/tests/test_video.py

@@ -20,6 +20,7 @@ class TestVideo:
     reader = scope.Reader(logdir)
     assert reader.keys() == ('foo',)
     assert reader.length('foo') == 2
-    steps, values = reader['foo']
+    steps, filenames = reader['foo']
+    values = [reader.load('foo', x) for x in filenames]
     assert (steps == np.array([0, 5])).all()
     assert np.allclose(values, [vid1, vid2], rtol=0.1)

scope-0.2.2/scope/__init__.py

@@ -1,4 +0,0 @@
-__version__ = '0.2.2'
-
-from .writer import Writer
-from .reader import Reader

scope-0.2.2/scope/columns.py

@@ -1,188 +0,0 @@
-import io
-import struct
-import time
-
-import av
-import numpy as np
-from PIL import Image
-
-
-def table_length(filename, fmt):
-  return filename.stat().st_size // struct.calcsize(fmt)
-
-
-def table_write(filename, fmt, *cols):
-  rows = tuple(zip(*cols))
-  size = struct.calcsize(fmt)
-  buffer = bytearray(len(rows) * size)
-  for index, row in enumerate(rows):
-    struct.pack_into(fmt, buffer, index * size, *row)
-  with filename.open('ab') as f:
-    f.write(buffer)
-
-
-def table_read(filename, fmt, start=0, stop=None):
-  assert stop is None or start < stop, (start, stop)
-  size = struct.calcsize(fmt)
-  with filename.open('rb') as f:
-    start and f.seek(start * size)
-    buffer = f.read((stop - start) * size if stop else None)
-  rows = struct.iter_unpack(fmt, buffer)
-  cols = tuple(zip(*rows))
-  return cols
-
-
-class FloatColumn:
-
-  def __init__(self, logdir, key):
-    name = key.replace('/', '-') + '.float'
-    self.filename = logdir / name
-
-  def validate(self, value):
-    assert value.dtype in (float, int) and value.ndim == 0, (
-        value.dtype, value.shape)
-    return value
-
-  def write(self, values):
-    steps, values = zip(*values)
-    table_write(self.filename, '>qd', steps, values)
-
-  def length(self):
-    return table_length(self.filename, '>qd')
-
-  def read(self, start, stop):
-    steps, values = table_read(self.filename, '>qd')
-    filtered = [(s, v) for s, v in zip(steps, values) if start <= s < stop]
-    steps, values = zip(*filtered) if filtered else ([], [])
-    steps = np.array(steps, np.int64)
-    values = np.array(values, np.float64)
-    return steps, values
-
-
-class FileColumn:
-
-  def __init__(self, logdir, key, ext, encfn, decfn):
-    name = key.replace('/', '-') + '.' + ext
-    self.folder = logdir / name
-    self.folder.mkdir(exist_ok=True)
-    self.index = self.folder / 'index'
-    self.rng = np.random.default_rng(seed=None)
-    self.ext = ext
-    self.encfn = encfn
-    self.decfn = decfn
-
-  def validate(self, value):
-    raise NotImplementedError
-
-  def write(self, values):
-    prefix = int(time.time()).to_bytes(4, 'big')
-    steps, values = zip(*values)
-    idents = [prefix + self.rng.bytes(4) for _ in range(len(steps))]
-    for ident, step, value in zip(idents, steps, values):
-      buffer = self.encfn(value)
-      with self._filename(step, ident).open('wb') as f:
-        f.write(buffer)
-    table_write(self.index, 'q8s', steps, idents)
-
-  def length(self):
-    return table_length(self.index, 'q8s')
-
-  def read(self, start, stop):
-    steps, idents = table_read(self.index, 'q8s')
-    filtered = [(s, v) for s, v in zip(steps, idents) if start <= s < stop]
-    steps, idents = zip(*filtered) if filtered else ([], [])
-    values = []
-    for step, ident in zip(steps, idents):
-      with self._filename(step, ident).open('rb') as f:
-        buffer = f.read()
-      values.append(self.decfn(buffer))
-    steps = np.array(steps, np.int64)
-    values = tuple(values)
-    return steps, values
-
-  def _filename(self, step, ident):
-    return self.folder / f'{step:020}-{ident.hex()}.{self.ext}'
-
-
-class TextColumn(FileColumn):
-
-  def __init__(self, logdir, key, fmt='txt'):
-    super().__init__(logdir, key, fmt, self.encode, self.decode)
-    self.fmt = fmt
-
-  def validate(self, value):
-    assert isinstance(value, str), type(value)
-    return value
-
-  def encode(self, value):
-    return value.encode('utf-8')
-
-  def decode(self, buffer):
-    return buffer.decode('utf-8')
-
-
-class ImageColumn(FileColumn):
-
-  def __init__(self, logdir, key, fmt='png', quality=None):
-    super().__init__(logdir, key, fmt, self.encode, self.decode)
-    self.fmt = fmt
-    self.quality = quality
-
-  def validate(self, value):
-    assert (
-        value.dtype == np.uint8 and value.ndim == 3 and
-        value.shape[-1] in (1, 3)), (value.dtype, value.shape)
-    return value
-
-  def encode(self, value):
-    if value.shape[-1] == 1:
-      value = value.repeat(3, -1)
-    fmt = ('jpeg' if self.fmt == 'jpg' else self.fmt).upper()
-    fp = io.BytesIO()
-    Image.fromarray(value).save(fp, format=fmt, quality=self.quality)
-    return fp.getvalue()
-
-  def decode(self, buffer):
-    return np.asarray(Image.open(io.BytesIO(buffer)).convert('RGB'))
-
-
-class VideoColumn(FileColumn):
-
-  def __init__(self, logdir, key, fmt='mp4', fps=20, codec='h264'):
-    super().__init__(logdir, key, fmt, self.encode, self.decode)
-    self.fmt = fmt
-    self.fps = fps
-    self.codec = codec
-
-  def validate(self, value):
-    assert (
-        value.dtype == np.uint8 and value.ndim == 4 and
-        value.shape[-1] in (1, 3)), (value.dtype, value.shape)
-    return value
-
-  def encode(self, value):
-    if value.shape[-1] == 1:
-      value = value.repeat(3, -1)
-    T, H, W, C = value.shape
-    fp = io.BytesIO()
-    output = av.open(fp, mode='w', format=self.fmt)
-    stream = output.add_stream(self.codec, rate=float(self.fps))
-    stream.width = W
-    stream.height = H
-    stream.pix_fmt = 'yuv420p'
-    for t in range(T):
-      frame = av.VideoFrame.from_ndarray(value[t], format='rgb24')
-      frame.pts = t
-      output.mux(stream.encode(frame))
-    output.mux(stream.encode(None))
-    output.close()
-    return fp.getvalue()
-
-  def decode(self, buffer):
-    container = av.open(io.BytesIO(buffer))
-    value = []
-    for frame in container.decode(video=0):
-      value.append(frame.to_ndarray(format='rgb24'))
-    value = np.stack(value)
-    container.close()
-    return value

scope-0.2.2/scope/reader.py

@@ -1,43 +0,0 @@
-import re
-import pathlib
-
-import numpy as np
-
-from . import columns
-
-
-class Reader:
-
-  def __init__(self, logdir):
-    if isinstance(logdir, str):
-      logdir = pathlib.Path(logdir)
-    self.coltypes = {
-        'float': columns.FloatColumn,
-        'png': columns.ImageColumn,
-        'mp4': columns.VideoColumn,
-    }
-    self.columns = {}
-    for child in sorted(logdir.glob('*')):
-      name, ext = child.name.rsplit('.', 1)
-      key = name.replace('-', '/')
-      assert re.match(r'[a-z0-9_]+(/[a-z0-9_]+)?', key), key
-      self.columns[key] = self.coltypes[ext](logdir, key)
-
-  def keys(self):
-    return tuple(self.columns.keys())
-
-  def length(self, key):
-    return self.columns[key].length()
-
-  def __getitem__(self, index):
-    if isinstance(index, str):
-      key, start, stop = index, -np.inf, +np.inf
-    else:
-      key, pos = index
-      if isinstance(pos, int):
-        start, stop = pos, pos + 1
-      else:
-        assert pos.step is None
-        start = -np.inf if pos.start is None else pos.start
-        stop = +np.inf if pos.stop is None else pos.stop
-    return self.columns[key].read(start, stop)

scope-0.2.2/scope/writer.py

@@ -1,71 +0,0 @@
-import concurrent.futures
-import pathlib
-import re
-from functools import partial as bind
-
-import numpy as np
-
-from . import columns
-
-
-class Writer:
-
-  def __init__(self, logdir, fps=20, workers=32):
-    if isinstance(logdir, str):
-      logdir = pathlib.Path(logdir)
-    self.logdir = logdir
-    self.logdir.mkdir(parents=True, exist_ok=True)
-    self.fps = fps
-    self.workers = workers
-    self.coltypes = [
-        (lambda x: isinstance(x, str), columns.TextColumn),
-        (lambda x: x.ndim == 0, columns.FloatColumn),
-        (lambda x: x.ndim == 3, bind(columns.ImageColumn, fmt='png')),
-        (lambda x: x.ndim == 4, bind(columns.VideoColumn, fmt='mp4', fps=fps)),
-    ]
-    self.columns = {}
-    self.values = {}
-    if workers:
-      self.pool = concurrent.futures.ThreadPoolExecutor(workers, 'writer')
-      self.futures = []
-
-  def add(self, step, *args, **kwargs):
-    step = int(step)
-    mapping = dict(*args, **kwargs)
-    for key, value in mapping.items():
-      value = value if isinstance(value, str) else np.asarray(value)
-      if key not in self.columns:
-        assert re.match(r'[a-z0-9_]+(/[a-z0-9_]+)?', key), key
-        for applies, coltype in self.coltypes:
-          if applies(value):
-            break
-        else:
-          raise NotImplementedError((
-              key, value,
-              getattr(value, 'shape', None),
-              getattr(value, 'dtype', None)))
-        self.columns[key] = coltype(self.logdir, key)
-        self.values[key] = []
-      column = self.columns[key]
-      try:
-        value = column.validate(value)
-      except Exception:
-        print(f"Error validating key '{key}' with value '{value}'.")
-        raise
-      self.values[key].append((step, value))
-
-  def flush(self):
-    keys = [key for key, values in self.values.items() if values]
-    if self.workers:
-      list(self.futures)
-      columns = [self.columns[x] for x in keys]
-      values = [self.values[x] for x in keys]
-      self.futures = self.pool.map(lambda x, y: x.write(y), columns, values)
-    else:
-      for key in keys:
-        try:
-          self.columns[key].write(self.values[key])
-        except Exception:
-          print(f"Exception writing '{key}' column.")
-          raise
-    self.values = {key: [] for key in self.values.keys()}