elements 2.2.0__tar.gz → 3.1.0__tar.gz

{elements-2.2.0/elements.egg-info → elements-3.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: elements
-Version: 2.2.0
+Version: 3.1.0
 Summary: Building blocks for productive research.
 Home-page: http://github.com/danijar/elements
 Classifier: Intended Audience :: Science/Research

{elements-2.2.0 → elements-3.1.0}/elements/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '2.2.0'
+__version__ = '3.1.0'
 
 from .agg import Agg
 from .checkpoint import Checkpoint

elements-3.1.0/elements/agg.py

@@ -0,0 +1,143 @@
+import math
+import operator
+from collections import defaultdict
+from functools import partial as bind
+
+import numpy as np
+
+
+class Agg:
+
+  def __init__(self, maxlen=1e6):
+    self.reducers = defaultdict(list)
+    self.names = {}
+    self.maxlen = int(maxlen)
+
+  def add(self, key_or_dict, value=None, agg='default', prefix=None):
+    if value is not None:
+      self._add_single(key_or_dict, value, agg, prefix)
+      return
+    for key, value in key_or_dict.items():
+      self._add_single(key, value, agg, prefix)
+
+  def result(self, reset=True, prefix=None):
+    metrics = {}
+    for key, reducers in self.reducers.items():
+      if len(reducers) == 1:
+        metrics[key] = reducers[0].current()
+      else:
+        for name, reducer in zip(self.names[key], reducers):
+          metrics[f'{key}/{name}'] = reducer.current()
+    if prefix:
+      metrics = {f'{prefix}/{k}': v for k, v in metrics.items()}
+    reset and self.reset()
+    return metrics
+
+  def reset(self):
+    self.reducers.clear()
+
+  def _add_single(self, key, value, agg, prefix):
+    key = f'{prefix}/{key}' if prefix else key
+    reducers = self.reducers[key]
+    if reducers:
+      for reducer in reducers:
+        reducer.update(value)
+      return
+    if agg == 'default':
+      agg = 'avg' if np.asarray(value).ndim <= 1 else 'last'
+    if isinstance(agg, str):
+      aggs = (agg,)
+      self.names[key] = None
+    else:
+      aggs = agg
+      self.names[key] = aggs
+    for agg in aggs:
+      if agg == 'avg':
+        reducer = Mean(value)
+      elif agg == 'sum':
+        reducer = Sum(value)
+      elif agg == 'min':
+        reducer = Min(value)
+      elif agg == 'max':
+        reducer = Max(value)
+      elif agg == 'stack':
+        reducer = Stack(value, self.maxlen)
+      elif agg == 'last':
+        reducer = Last(value)
+      else:
+        raise ValueError(agg)
+      reducers.append(reducer)
+
+
+class Reducer:
+
+  def __init__(self, scalar_fn, array_fn, initial):
+    self.scalar_fn = scalar_fn
+    self.array_fn = array_fn
+    self.is_scalar = isinstance(initial, (int, float))
+    if self.is_scalar:
+      self.interm = initial
+    else:
+      self.interm = np.array(initial, np.float64)
+    self.count = 1
+
+  def update(self, value):
+    if self.is_scalar:
+      if math.isnan(value):
+        return
+      self.interm = self.scalar_fn(self.interm, value)
+    else:
+      value = np.asarray(value)
+      if np.isnan(value).any():
+        return
+      self.interm = self.array_fn(self.interm, value)
+    self.count += 1
+
+  def current(self):
+    return np.array(self.interm)
+
+
+class Mean:
+
+  def __init__(self, initial):
+    self.reducer = Sum(initial)
+
+  def update(self, value):
+    self.reducer.update(value)
+
+  def current(self):
+    return self.reducer.current() / self.reducer.count
+
+
+class Stack:
+
+  def __init__(self, initial, maxlen=1e5):
+    self.stack = [initial]
+    self.maxlen = int(maxlen)
+
+  def update(self, value):
+    if len(self.stack) < self.maxlen:
+      self.stack.append(value)
+
+  def current(self):
+    return np.stack(self.stack)
+
+
+class Last:
+
+  def __init__(self, initial):
+    self.value = initial
+
+  def update(self, value):
+    self.value = value
+
+  def current(self):
+    return self.value
+
+
+Sum = bind(
+    Reducer, operator.add, lambda x, y: np.add(x, y, out=x, dtype=np.float64))
+Min = bind(
+    Reducer, min, lambda x, y: np.minimum(x, y, out=x, dtype=np.float64))
+Max = bind(
+    Reducer, max, lambda x, y: np.maximum(x, y, out=x, dtype=np.float64))

{elements-2.2.0 → elements-3.1.0}/elements/checkpoint.py

@@ -1,8 +1,10 @@
 import concurrent.futures
-import json
+import pickle
 import time
 
+from . import printing
 from . import path
+from . import timer
 
 
 class Checkpoint:
@@ -12,7 +14,7 @@ class Checkpoint:
     self._values = {}
     self._parallel = parallel
     if self._parallel:
-      self._worker = concurrent.futures.ThreadPoolExecutor(1)
+      self._worker = concurrent.futures.ThreadPoolExecutor(1, 'checkpoint')
       self._promise = None
 
   def __setattr__(self, name, value):
@@ -31,7 +33,7 @@ class Checkpoint:
     if name.startswith('_'):
       raise AttributeError(name)
     try:
-      return getattr(self._values, name)
+      return self._values[name]
     except AttributeError:
       raise ValueError(name)
 
@@ -48,33 +50,39 @@ class Checkpoint:
   def save(self, filename=None, keys=None):
     assert self._filename or filename
     filename = path.Path(filename or self._filename)
-    print(f'Writing checkpoint: {filename}')
+    printing.print_(f'Writing checkpoint: {filename}')
     if self._parallel:
       self._promise and self._promise.result()
       self._promise = self._worker.submit(self._save, filename, keys)
     else:
       self._save(filename, keys)
 
+  @timer.section('checkpoint_save')
   def _save(self, filename, keys):
     keys = tuple(self._values.keys() if keys is None else keys)
     assert all([not k.startswith('_') for k in keys]), keys
     data = {k: self._values[k].save() for k in keys}
     data['_timestamp'] = time.time()
     filename.parent.mkdirs()
-    # Write to a temporary file and then atomically rename, so that the
-    # requested filename either contains a full checkpoint or does not exist if
-    # writing was interrupted.
-    tmp = filename.parent / (filename.name + '.tmp')
-    tmp.write(json.dumps(data), mode='wb')
-    tmp.move(filename)
-    print(f'Wrote checkpoint: {filename}')
+    content = pickle.dumps(data)
+    if str(filename).startswith('gs://'):
+      filename.write(content, mode='wb')
+    else:
+      # Write to a temporary file and then atomically rename, so that the file
+      # either contains a complete write or not update at all if writing was
+      # interrupted.
+      tmp = filename.parent / (filename.name + '.tmp')
+      tmp.write(content, mode='wb')
+      tmp.move(filename)
+    print('Wrote checkpoint.')
 
+  @timer.section('checkpoint_load')
   def load(self, filename=None, keys=None):
     assert self._filename or filename
     self._promise and self._promise.result()  # Wait for last save.
     filename = path.Path(filename or self._filename)
-    print(f'Loading checkpoint: {filename}')
-    data = json.loads(filename.read('rb'))
+    printing.print_(f'Loading checkpoint: {filename}')
+    data = pickle.loads(filename.read('rb'))
     keys = tuple(data.keys() if keys is None else keys)
     for key in keys:
       if key.startswith('_'):
@@ -82,10 +90,10 @@ class Checkpoint:
       try:
         self._values[key].load(data[key])
       except Exception:
-        print(f'Error loading {key} from checkpoint.')
+        print(f"Error loading '{key}' from checkpoint.")
         raise
     age = time.time() - data['_timestamp']
-    print(f'Loaded checkpoint from {age:.0f} seconds ago.')
+    printing.print_(f'Loaded checkpoint from {age:.0f} seconds ago.')
 
   def load_or_save(self):
     if self.exists():

{elements-2.2.0 → elements-3.1.0}/elements/config.py

@@ -30,9 +30,10 @@ class Config(dict):
     if filename.suffix == '.json':
       filename.write(json.dumps(dict(self)))
     elif filename.suffix in ('.yml', '.yaml'):
-      import ruamel.yaml as yaml
+      from ruamel.yaml import YAML
+      yaml = YAML(typ='safe')
       with io.StringIO() as stream:
-        yaml.safe_dump(dict(self), stream)
+        yaml.dump(dict(self), stream)
         filename.write(stream.getvalue())
     else:
       raise NotImplementedError(filename.suffix)
@@ -41,10 +42,11 @@ class Config(dict):
   def load(cls, filename):
     filename = path.Path(filename)
     if filename.suffix == '.json':
-      return cls(json.loads(filename.read_text()))
+      return cls(json.loads(filename.read()))
     elif filename.suffix in ('.yml', '.yaml'):
-      import ruamel.yaml as yaml
-      return cls(yaml.safe_load(filename.read_text()))
+      from ruamel.yaml import YAML
+      yaml = YAML(typ='safe')
+      return cls(yaml.load(filename.read()))
     else:
       raise NotImplementedError(filename.suffix)
 

{elements-2.2.0 → elements-3.1.0}/elements/flags.py

@@ -12,9 +12,11 @@ class Flags:
   def parse(self, argv=None, help_exits=True):
     parsed, remaining = self.parse_known(argv)
     for flag in remaining:
-      if flag.startswith('--'):
-        raise ValueError(f"Flag '{flag}' did not match any config keys.")
-    assert not remaining, remaining
+      if flag.startswith('--') and flag[2:] not in self._config.flat:
+        raise KeyError(f"Flag '{flag}' did not match any config keys.")
+    if remaining:
+      raise ValueError(
+          f'Could not parse all arguments. Remaining: {remaining}')
     return parsed
 
   def parse_known(self, argv=None, help_exits=False):
@@ -52,25 +54,40 @@ class Flags:
       return
     if not key:
       vals = ', '.join(f"'{x}'" for x in vals)
-      raise ValueError(f"Values {vals} were not preceded by any flag.")
+      remaining.extend(vals)
+      return
+      # raise ValueError(f"Values {vals} were not preceded by any flag.")
     name = key[len('--'):]
     if '=' in name:
       remaining.extend([key] + vals)
       return
-    if self._config.IS_PATTERN.fullmatch(name):
+    if not vals:
+      remaining.extend([key])
+      return
+      # raise ValueError(f"Flag '{key}' was not followed by any values.")
+    if name.endswith('+') and name[:-1] in self._config:
+      key = name[:-1]
+      default = self._config[key]
+      if not isinstance(default, tuple):
+        raise TypeError(
+            f"Cannot append to key '{key}' which is of type "
+            f"'{type(default).__name__}' instead of tuple.")
+      if key not in parsed:
+        parsed[key] = default
+      parsed[key] += self._parse_flag_value(default, vals, key)
+    elif self._config.IS_PATTERN.fullmatch(name):
       pattern = re.compile(name)
-      keys = {k for k in self._config.flat if pattern.fullmatch(k)}
+      keys = [k for k in self._config.flat if pattern.fullmatch(k)]
+      if keys:
+        for key in keys:
+          parsed[key] = self._parse_flag_value(self._config[key], vals, key)
+      else:
+        remaining.extend([key] + vals)
     elif name in self._config:
-      keys = [name]
+      key = name
+      parsed[key] = self._parse_flag_value(self._config[key], vals, key)
     else:
-      keys = []
-    if not keys:
       remaining.extend([key] + vals)
-      return
-    if not vals:
-      raise ValueError(f"Flag '{key}' was not followed by any values.")
-    for key in keys:
-      parsed[key] = self._parse_flag_value(self._config[key], vals, key)
 
   def _parse_flag_value(self, default, value, key):
     value = value if isinstance(value, (tuple, list)) else (value,)
@@ -78,7 +95,9 @@ class Flags:
       if len(value) == 1 and ',' in value[0]:
         value = value[0].split(',')
       return tuple(self._parse_flag_value(default[0], [x], key) for x in value)
-    assert len(value) == 1, value
+    if len(value) != 1:
+      raise TypeError(
+          f"Expected a single value for key '{key}' but got: {value}")
     value = str(value[0])
     if default is None:
       return value
@@ -92,11 +111,16 @@ class Flags:
       try:
         value = float(value)  # Allow scientific notation for integers.
         assert float(int(value)) == value
-      except (TypeError, AssertionError):
-        message = f"Expected int but got float '{value}' for key '{key}'."
+      except (ValueError, TypeError, AssertionError):
+        message = f"Expected int but got '{value}' for key '{key}'."
         raise TypeError(message)
       return int(value)
     if isinstance(default, dict):
-      raise TypeError(
+      raise KeyError(
           f"Key '{key}' refers to a whole dict. Please speicfy a subkey.")
-    return type(default)(value)
+    try:
+      return type(default)(value)
+    except ValueError:
+      raise TypeError(
+          f"Cannot convert '{value}' to type '{type(default).__name__}' for "
+          f"key '{key}'.")

{elements-2.2.0 → elements-3.1.0}/elements/printing.py

@@ -2,15 +2,22 @@ import re
 
 try:
   import colored
-  REGEX_TOKEN = re.compile(r"([^a-zA-Z0-9-_./'\"\[\]])", re.MULTILINE)
-  REGEX_NUMBER = re.compile(r'[-+]?[0-9]+[0-9.]*(e[-+]?[0-9])?')
-  KEYWORDS = ()  # ('True', 'False', 'None', 'bool', 'int', 'str', 'float')
 except ImportError:
-  print('For colored outputs: pip install colored')
   colored = None
+  print('For colored outputs: pip install colored')
+
 
+REGEX_TOKEN = re.compile(
+    r"([^a-zA-Z0-9-_./'\"\[\]]|['\"][^\s]*['\"])", re.MULTILINE)
+REGEX_NUMBER = re.compile(
+    r'([-+]?[0-9]+[0-9.,]*(e[-+]?[0-9])?|nan|-?inf)')
+KEYWORDS = (
+    'True', 'False', 'None', 'bool', 'int', 'str', 'float',
+    'uint8', 'float16', 'float32', 'int32', 'int64')
 
-def print_(value, color=True, **kwargs):
+
+def print_(*values, color=True, **kwargs):
+  value = kwargs.get('sep', ' ').join(str(x) for x in values)
   assert not color or isinstance(color, (bool, str)), color
   if isinstance(color, str) and colored:
     value = colored.stylize(value, colored.fg(color))
@@ -20,34 +27,36 @@ def print_(value, color=True, **kwargs):
     tokens = REGEX_TOKEN.split(value) + [None]
     for i, token in enumerate(tokens[:-1]):
       new = prev.copy()
-      stripped = token.strip()
+      word = token.strip()
       new[2] = None
-      if not stripped:
+      if not word:
         new[0] = None
-      elif stripped in '/-+':
+      elif word in '/-+':
         new[0] = 'green'
         new[2] = True
-      elif stripped in '{}()<>,:':
+      elif word in '{}()<>,:':
         new[0] = 'white'
       elif token == '=':
         new[0] = 'white'
-      elif stripped[0].isalpha() and tokens[i + 1] == '=':
+      elif word[0].isalpha() and tokens[i + 1] == '=':
         new[0] = 'magenta'
-      elif stripped in KEYWORDS:
+      elif word in KEYWORDS:
         new[0] = 'blue'
-      elif stripped.startswith('---'):
+      elif word.startswith('---'):
         new[1] = True
-      elif REGEX_NUMBER.match(stripped):
+      elif REGEX_NUMBER.match(word):
         new[0] = 'blue'
-      elif stripped[0] == stripped[-1] == "'":
+      elif word[0] == word[-1] == "'":
         new[0] = 'red'
-      elif stripped[0] == stripped[-1] == '"':
+      elif word[0] == word[-1] == '"':
         new[0] = 'red'
-      elif stripped[0] == '[' and stripped[-1] == ']':
+      elif word[0] == '[' and word[-1] == ']':
         new[0] = 'cyan'
-      elif stripped[0] == '/':
+      elif any(word.startswith(x) for x in ('/', '~', './')):
         new[0] = 'yellow'
-      elif stripped[0] == stripped[0].upper():
+      elif len(word) >= 3 and word[0] == word[-1] and word[0] in ("'", '"'):
+        new[0] = 'green'
+      elif word[0] == word[0].upper():
         new[0] = None
       else:
         new[0] = None

{elements-2.2.0 → elements-3.1.0}/elements/tree.py

@@ -20,6 +20,11 @@ def map_(fn, *trees, isleaf=None):
     assert all(set(x.keys()) == set(first.keys()) for x in trees), (
         printing.format_(trees))
     return {k: map_(fn, *[t[k] for t in trees], **kw) for k in first}
+  if hasattr(first, 'keys') and hasattr(first, 'get'):
+    assert all(set(x.keys()) == set(first.keys()) for x in trees), (
+        printing.format_(trees))
+    return type(first)(
+        {k: map_(fn, *[t[k] for t in trees], **kw) for k in first})
   return fn(*trees)
 
 

{elements-2.2.0 → elements-3.1.0}/elements/usage.py

@@ -50,7 +50,7 @@ class NvsmiStats:
 
   @timer.section('nvsmi_stats')
   def __call__(self):
-    output = os.popen('nvidia-smi --query -d UTILIZATION').read()
+    output = os.popen('nvidia-smi --query -d UTILIZATION 2>&1').read()
     if not output:
       print('To log GPU stats, make sure nvidia-smi is working.')
       return {}

{elements-2.2.0 → elements-3.1.0}/elements/uuid.py

@@ -5,11 +5,18 @@ import numpy as np
 
 
 class UUID:
+  """UUID that is stored as 16 byte string and can be converted to and from
+  int, string, and array types."""
+
+  __slots__ = ('value', '_hash')
 
   DEBUG_ID = None
   BASE62 = string.digits + string.ascii_letters
   BASE62REV = {x: i for i, x in enumerate(BASE62)}
 
+  # def __new__(cls, val=None):
+  #   return val or np.random.randint(1, 2 ** 63)
+
   @classmethod
   def reset(cls, *, debug):
     cls.DEBUG_ID = 0 if debug else None
@@ -21,10 +28,13 @@ class UUID:
       else:
         type(self).DEBUG_ID += 1
         self.value = self.DEBUG_ID.to_bytes(16, 'big')
-    elif isinstance(value, type(self)):
+    elif isinstance(value, UUID):
       self.value = value.value
     elif isinstance(value, int):
       self.value = value.to_bytes(16, 'big')
+    elif isinstance(value, bytes):
+      assert len(value) == 16, value
+      self.value = value
     elif isinstance(value, str):
       if self.DEBUG_ID is None:
         integer = 0
@@ -37,7 +47,7 @@ class UUID:
       self.value = value.tobytes()
     else:
       raise ValueError(value)
-    assert type(self.value) == bytes, type(self.value)
+    assert type(self.value) == bytes, type(self.value)  # noqa
     assert len(self.value) == 16, len(self.value)
     self._hash = hash(self.value)
 

{elements-2.2.0 → elements-3.1.0/elements.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: elements
-Version: 2.2.0
+Version: 3.1.0
 Summary: Building blocks for productive research.
 Home-page: http://github.com/danijar/elements
 Classifier: Intended Audience :: Science/Research

{elements-2.2.0 → elements-3.1.0}/elements.egg-info/SOURCES.txt

@@ -25,4 +25,7 @@ elements.egg-info/PKG-INFO
 elements.egg-info/SOURCES.txt
 elements.egg-info/dependency_links.txt
 elements.egg-info/requires.txt
-elements.egg-info/top_level.txt
+elements.egg-info/top_level.txt
+tests/test_basics.py
+tests/test_flags.py
+tests/test_path.py

elements-3.1.0/elements.egg-info/requires.txt

@@ -0,0 +1 @@
+numpy

{elements-2.2.0 → elements-3.1.0}/requirements-optional.txt

@@ -1,7 +1,9 @@
 colored
-gil_load
+gputil
 matplotlib
 mlflow
+psutil
 pytest
+ruamel.yaml
 tensorflow-cpu
 wandb

elements-3.1.0/requirements.txt

@@ -0,0 +1 @@
+numpy

elements-3.1.0/tests/test_basics.py

@@ -0,0 +1,70 @@
+import elements
+import pytest
+
+
+class TestBasics:
+
+  def test_config(self):
+    config = elements.Config({'one.two': 12, 'foo': {'bar': True}})
+    assert config.one.two == 12
+    assert config.foo.bar is True
+    assert config['foo'].bar is True
+    assert 'foo' in config
+    assert 'foo.bar' in config
+    assert str(config)
+    _ = config.update({'one.two': 42})
+    with pytest.raises(KeyError):
+      _ = config.update({'new': 1})
+    with pytest.raises(TypeError):
+      _ = config.update({'one.two': 'string'})
+    with pytest.raises(AttributeError):
+      config.one.two = 1
+    with pytest.raises(TypeError):
+      elements.Config({'foo': lambda: None})
+
+  def test_flags(self):
+    flags = elements.Flags(foo=12, bar={'baz': True})
+    assert flags.parse(['--bar.baz', 'False']).foo == 12
+    assert flags.parse(['--bar.baz', 'False']).bar.baz is False
+    assert flags.parse(['--bar.*', 'False']).bar.baz is False
+    with pytest.raises(TypeError):
+      flags.parse(['--bar.baz', '12'])
+    with pytest.raises(KeyError):
+      flags.parse(['--.*unknown.*', '12'])
+    _, remaining = flags.parse_known(['one=two', '--foo', '42', '--three'])
+    assert remaining == ['one=two', '--three']
+    flags = elements.Flags({'foo': 12})
+    _, remaining = flags.parse_known(['--help'], help_exits=False)
+    assert remaining == ['--help']
+
+  def test_logger(self, capsys):
+    step = elements.Counter()
+    logger = elements.Logger(step, [elements.logger.TerminalOutput()])
+    logger.scalar('name', 15)
+    logger.write()
+    output = capsys.readouterr()
+    print(output)
+
+  def test_every(self):
+    should = elements.when.Every(5)
+    result = []
+    for i in range(16):
+      if should(i):
+        result.append(i)
+    assert result == [0, 5, 10, 15]
+
+  def test_once(self):
+    should = elements.when.Once()
+    result = []
+    for i in range(16):
+      if should():
+        result.append(i)
+    assert result == [0]
+
+  def test_until(self):
+    should = elements.when.Until(6)
+    result = []
+    for i in range(16):
+      if should(i):
+        result.append(i)
+    assert result == [0, 1, 2, 3, 4, 5]

elements-3.1.0/tests/test_flags.py

@@ -0,0 +1,126 @@
+import elements
+import pytest
+
+
+class TestFlags:
+
+  def test_int(self):
+    flags = elements.Flags({'foo': 42})
+    assert flags.parse(['--foo=1']).foo == 1
+    assert flags.parse(['--foo=1.0']).foo == 1
+    assert flags.parse(['--foo=1e2']).foo == 100
+    with pytest.raises(TypeError):
+      flags.parse(['--foo=0.5'])
+    with pytest.raises(TypeError):
+      flags.parse(['--foo=foo'])
+    with pytest.raises(TypeError):
+      assert flags.parse(['--foo=1,2,3'])
+
+  def test_float(self):
+    flags = elements.Flags({'foo': 1.0})
+    assert flags.parse(['--foo=0.5']).foo == 0.5
+    assert flags.parse(['--foo=1']).foo == 1.0
+    assert flags.parse(['--foo=1e2']).foo == 1e2
+    with pytest.raises(TypeError):
+      flags.parse(['--foo=True'])
+    with pytest.raises(TypeError):
+      flags.parse(['--foo=foo'])
+    with pytest.raises(TypeError):
+      assert flags.parse(['--foo=0.5,1.0'])
+
+  def test_bool(self):
+    flags = elements.Flags({'foo': True})
+    assert flags.parse(['--foo=True']).foo is True
+    assert flags.parse(['--foo=False']).foo is False
+    with pytest.raises(TypeError):
+      flags.parse(['--foo=true'])
+    with pytest.raises(TypeError):
+      flags.parse(['--foo=1'])
+    with pytest.raises(TypeError):
+      flags.parse(['--foo=foo'])
+    with pytest.raises(TypeError):
+      assert flags.parse(['--foo=True,False'])
+
+  def test_str(self):
+    flags = elements.Flags({'foo': 'hello'})
+    assert flags.parse(['--foo=hi']).foo == 'hi'
+    assert flags.parse(['--foo=1,2,3']).foo == '1,2,3'
+    assert flags.parse(['--foo=']).foo == ''
+    with pytest.raises(TypeError):
+      assert flags.parse(['--foo', '1', '2', '3'])
+
+  def test_sequence(self):
+    flags = elements.Flags({'foo': [1, 2]})
+    assert flags.parse(['--foo=1']).foo == (1,)
+    assert flags.parse(['--foo=1,2,3']).foo == (1, 2, 3)
+    assert flags.parse(['--foo', '1,2,3']).foo == (1, 2, 3)
+    assert flags.parse(['--foo', '1', '2', '3']).foo == (1, 2, 3)
+    with pytest.raises(TypeError):
+      assert flags.parse(['--foo', 'False'])
+    with pytest.raises(TypeError):
+      assert flags.parse(['--foo=1,2,0.5'])
+
+  def test_append(self):
+    flags = elements.Flags({'foo': [1, 2]})
+    assert flags.parse(['--foo+=3']).foo == (1, 2, 3)
+    assert flags.parse(['--foo+', '3']).foo == (1, 2, 3)
+    with pytest.raises(TypeError):
+      assert flags.parse(['--foo+', '0.5'])
+    assert flags.parse(['--foo=1', '--foo+=2', '--foo+=3']).foo == (1, 2, 3)
+    assert flags.parse(['--foo+=3', '--foo+=4']).foo == (1, 2, 3, 4)
+
+  def test_nested(self):
+    flags = elements.Flags({'foo.bar': 12})
+    assert flags.parse(['--foo.bar=42']).foo.bar == 42
+    with pytest.raises(KeyError):
+      assert flags.parse(['--foo=42'])
+    with pytest.raises(KeyError):
+      assert flags.parse(['--foo.baz=42'])
+    flags = elements.Flags({'foo': {'bar': 12}})
+    assert flags.parse(['--foo.bar=42']).foo.bar == 42
+
+  def test_regex(self):
+    flags = elements.Flags({'foo.bar': 12, 'baz': 'text'})
+    assert flags.parse([r'--.*\.bar=42']).foo.bar == 42
+    assert flags.parse([r'--.*z$=hello']).baz == 'hello'
+    parsed = flags.parse([r'--.*=42'])
+    assert parsed.foo.bar == 42
+    assert parsed.baz == '42'
+    with pytest.raises(TypeError):
+      assert flags.parse([r'--.*\.bar=0.5'])
+
+  def test_kwargs(self):
+    assert elements.Flags(foo=42).parse(['--foo=12']).foo == 12
+    assert elements.Flags(foo=42, bar='text').parse(['--bar=i']).bar == 'i'
+
+  def test_multiple(self):
+    defaults = elements.Config(foo=12, bar=0.5, baz='text')
+    flags = elements.Flags(defaults)
+    assert flags.parse([]) == defaults
+    assert flags.parse(['--bar', '2.5', '--foo=1']) == (
+        defaults.update(foo=1, bar=2.5))
+    with pytest.raises(ValueError):
+      flags.parse(['--bar', '--foo=1'])
+
+  def test_invalid(self):
+    flags = elements.Flags(foo=12)
+    with pytest.raises(KeyError):
+      flags.parse(['--bar=1'])
+    with pytest.raises(ValueError):
+      flags.parse(['foo=1'])
+    with pytest.raises(ValueError):
+      flags.parse(['1', '--foo=5'])
+
+  def test_from_yaml(self, tmpdir):
+    filename = elements.Path(tmpdir) / 'defaults.yaml'
+    filename.write("""
+    foo: 42
+    parent.child: 12
+    seq: [1, 2, 3]
+    scope:
+      inside: foo
+    """)
+    defaults = elements.Config.load(filename)
+    flags = elements.Flags(defaults)
+    assert flags.parse(['--parent.child=42']).parent.child == 42
+    assert flags.parse(['--seq=2,4,6']).seq == (2, 4, 6)

elements-3.1.0/tests/test_path.py

@@ -0,0 +1,45 @@
+import elements
+
+
+class TestPath:
+
+  def test_str_canonical(self):
+    examples = ['/', 'foo/bar', 'file.txt', '/bar.tar.gz']
+    for example in examples:
+      assert str(elements.Path(example)) == example
+
+  def test_parent_and_name(self):
+    examples = ['foo/bar', '/bar.tar.gz', 'file.txt', 'foo/bar/baz']
+    for example in examples:
+      path = elements.Path(example)
+      assert path == path.parent / path.name
+
+  def test_stem_and_suffix(self):
+    examples = ['foo/bar', '/bar.tar.gz', 'file.txt', 'foo/bar/baz']
+    for example in examples:
+      path = elements.Path(example)
+      assert path.name == path.stem + path.suffix
+
+  def test_leading_dot(self):
+    assert str(elements.Path('')) == '.'
+    assert str(elements.Path('.')) == '.'
+    assert str(elements.Path('./')) == '.'
+    assert str(elements.Path('./foo')) == 'foo'
+
+  def test_trailing_slash(self):
+    assert str(elements.Path('./')) == '.'
+    assert str(elements.Path('a/')) == 'a'
+    assert str(elements.Path('foo/bar/')) == 'foo/bar'
+
+  # @pytest.mark.filterwarnings('ignore::DeprecationWarning')
+  # def test_protocols(self):
+  #   assert str(elements.Path('gs://')) == ('gs://')
+  #   assert str(elements.Path('gs://foo/bar')) == 'gs://foo/bar'
+
+  def test_parent(self):
+    empty = elements.Path('.')
+    root = elements.Path('/')
+    assert (root / 'foo' / 'bar.txt').parent.parent == root
+    assert (empty / 'foo' / 'bar.txt').parent.parent == empty
+    assert root.parent == root
+    assert empty.parent == empty

elements-2.2.0/elements/agg.py

@@ -1,78 +0,0 @@
-from collections import defaultdict
-
-import numpy as np
-
-
-class Agg:
-
-  def __init__(self, maxlen=int(1e6)):
-    self.maxlen = maxlen
-    self.avgs = defaultdict(lambda: [0.0, 0.0])
-    self.sums = defaultdict(float)
-    self.mins = defaultdict(float)
-    self.maxs = defaultdict(float)
-    self.lasts = defaultdict(lambda: None)
-    self.stacks = defaultdict(list)
-
-  def add(
-      self, key_or_dict, value=None, agg='default', prefix=None, nan='keep'):
-    aggs = (agg,) if isinstance(agg, str) else agg
-    assert nan in ('keep', 'ignore')
-    if value is None:
-      for key, value in dict(key_or_dict).items():
-        key = f'{prefix}/{key}' if prefix else key
-        self._add_single(key, value, aggs, nan)
-    else:
-      assert not prefix, prefix
-      self._add_single(key_or_dict, value, aggs, nan)
-
-  def result(self, reset=True, prefix=None):
-    metrics = {}
-    metrics.update({k: v[0] / v[1] for k, v in self.avgs.items()})
-    metrics.update(self.sums)
-    metrics.update(self.mins.items())
-    metrics.update(self.maxs.items())
-    metrics.update(self.lasts.items())
-    metrics.update({k: np.stack(v) for k, v in self.stacks.items()})
-    if prefix:
-      metrics = {f'{prefix}/{k}': v for k, v in metrics.items()}
-    reset and self.reset()
-    return metrics
-
-  def reset(self):
-    self.avgs.clear()
-    self.sums.clear()
-    self.mins.clear()
-    self.maxs.clear()
-    self.lasts.clear()
-    self.stacks.clear()
-
-  def _add_single(self, key, value, aggs, nan):
-    value = np.asarray(value)
-    if nan == 'ignore' and np.isnan(value):
-      return
-    for agg in aggs:
-      name = key if len(aggs) == 1 else f'{key}/{agg}'
-      if agg == 'default':
-        agg = 'avg' if len(value.shape) <= 1 else 'last'
-      if agg == 'avg':
-        avg = self.avgs[name]
-        avg[0] += value
-        avg[1] += 1
-        # assert not np.shares_memory(self.avgs[name][0], value)
-      elif agg == 'sum':
-        self.sums[name] += value
-      elif agg == 'min':
-        self.mins[name] = min(self.mins[name], value)
-        # assert not np.shares_memory(self.mins[name], value)
-      elif agg == 'max':
-        self.maxs[name] = max(self.maxs[name], value)
-        # assert not np.shares_memory(self.mins[name], value)
-      elif agg == 'last':
-        self.lasts[name] = value
-      elif agg == 'stack':
-        stack = self.stacks[name]
-        if len(stack) < self.maxlen:
-          stack.append(value)
-      else:
-        raise KeyError(agg)

elements-2.2.0/elements.egg-info/requires.txt

@@ -1,4 +0,0 @@
-gputil
-numpy
-psutil
-ruamel.yaml

elements-2.2.0/requirements.txt

@@ -1,4 +0,0 @@
-gputil
-numpy
-psutil
-ruamel.yaml