cs-fileutils 20250429__py2.py3-none-any.whl

cs/fileutils.py

@@ -0,0 +1,1835 @@
+#!/usr/bin/env python3
+#
+# Assorted convenience functions for files and filenames/pathnames.
+# - Cameron Simpson <cs@cskk.id.au>
+
+''' My grab bag of convenience functions for files and filenames/pathnames.
+'''
+
+# pylint: disable=too-many-lines
+
+from __future__ import with_statement, print_function, absolute_import
+from contextlib import contextmanager
+import errno
+from functools import partial
+import gzip
+import os
+from os import SEEK_CUR, SEEK_END, SEEK_SET, O_RDONLY, read
+try:
+  from os import pread
+except ImportError:
+  pread = None
+from os.path import (
+    abspath,
+    basename,
+    dirname,
+    exists as existspath,
+    isabs as isabspath,
+    isdir,
+    join as joinpath,
+    splitext,
+)
+import shutil
+import stat
+import sys
+from tempfile import TemporaryFile, NamedTemporaryFile, mkstemp
+from threading import Lock, RLock
+import time
+
+from cs.buffer import CornuCopyBuffer
+from cs.context import stackattrs
+from cs.deco import fmtdoc, OBSOLETE, strable
+from cs.filestate import FileState
+from cs.fs import shortpath
+from cs.gimmicks import TimeoutError  # pylint: disable=redefined-builtin
+from cs.lex import as_lines, cutsuffix, common_prefix
+from cs.logutils import error, warning, debug
+from cs.pfx import Pfx, pfx, pfx_call
+from cs.progress import Progress, progressbar
+from cs.py3 import ustr, bytes, pread  # pylint: disable=redefined-builtin
+from cs.range import Range
+from cs.resources import RunState, uses_runstate
+from cs.result import CancellationError
+from cs.threads import locked, NRLock
+from cs.units import BINARY_BYTES_SCALE
+
+__version__ = '20250429'
+
+DISTINFO = {
+    'keywords': ["python2", "python3"],
+    'classifiers': [
+        "Programming Language :: Python",
+        "Programming Language :: Python :: 2",
+        "Programming Language :: Python :: 3",
+    ],
+    'install_requires': [
+        'cs.buffer',
+        'cs.context',
+        'cs.deco',
+        'cs.filestate',
+        'cs.fs>=shortpath',
+        'cs.gimmicks>=TimeoutError',
+        'cs.lex>=20200914',
+        'cs.logutils',
+        'cs.pfx>=pfx_call',
+        'cs.progress',
+        'cs.py3',
+        'cs.range',
+        'cs.resources',
+        'cs.result',
+        'cs.threads',
+        'cs.units',
+    ],
+}
+
+DEFAULT_POLL_INTERVAL = 1.0
+DEFAULT_READSIZE = 131072
+DEFAULT_TAIL_PAUSE = 0.25
+
+def seekable(fp):
+  ''' Try to test whether a filelike object is seekable.
+
+      First try the `IOBase.seekable` method, otherwise try getting a file
+      descriptor from `fp.fileno` and `os.stat()`ing that,
+      otherwise return `False`.
+  '''
+  try:
+    test = fp.seekable
+  except AttributeError:
+    try:
+      getfd = fp.fileno
+    except AttributeError:
+      return False
+    test = lambda: stat.S_ISREG(os.fstat(getfd()).st_mode)
+  return test()
+
+def rename_excl(oldpath, newpath):
+  ''' Safely rRename `oldpath` to `newpath`.
+      Raise `FileExistsError` if `newpath` already exists.
+  '''
+  with pfx_call(open, newpath, 'xb'):
+    pass
+  pfx_call(os.rename, oldpath, newpath)
+
+@OBSOLETE("rename_excl")
+def saferename(oldpath, newpath):
+  ''' Rename a path using `os.rename()`,
+      but raise an exception if the target path already exists.
+      Note: slightly racey.
+  '''
+  try:
+    os.lstat(newpath)
+    raise OSError(errno.EEXIST)
+  except OSError as e:
+    if e.errno != errno.ENOENT:
+      raise
+    os.rename(oldpath, newpath)
+
+def trysaferename(oldpath, newpath):
+  ''' A `saferename()` that returns `True` on success,
+      `False` on failure.
+  '''
+  try:
+    saferename(oldpath, newpath)
+  except OSError:
+    return False
+  ##except Exception:
+  ##  raise
+  return True
+
+def compare(f1, f2, mode="rb"):
+  ''' Compare the contents of two file-like objects `f1` and `f2` for equality.
+
+      If `f1` or `f2` is a string, open the named file using `mode`
+      (default: `"rb"`).
+  '''
+  if isinstance(f1, str):
+    with open(f1, mode) as f1fp:
+      return compare(f1fp, f2, mode)
+  if isinstance(f2, str):
+    with open(f2, mode) as f2fp:
+      return compare(f1, f2fp, mode)
+  return f1.read() == f2.read()
+
+# pylint: disable=too-many-locals,too-many-branches,too-many-statements
+@contextmanager
+def NamedTemporaryCopy(f, progress=False, progress_label=None, **nt_kw):
+  ''' A context manager yielding a temporary copy of `filename`
+      as returned by `NamedTemporaryFile(**nt_kw)`.
+
+      Parameters:
+      * `f`: the name of the file to copy, or an open binary file,
+        or a `CornuCopyBuffer`
+      * `progress`: an optional progress indicator, default `False`;
+        if a `bool`, show a progress bar for the copy phase if true;
+        if an `int`, show a progress bar for the copy phase
+        if the file size equals or exceeds the value;
+        otherwise it should be a `cs.progress.Progress` instance
+      * `progress_label`: option progress bar label,
+        only used if a progress bar is made
+      Other keyword parameters are passed to `tempfile.NamedTemporaryFile`.
+  '''
+  if isinstance(f, str):
+    # copy named file
+    filename = f
+    progress_label = (
+        "copy " + repr(filename) if progress_label is None else progress_label
+    )
+    # should we use shutil.copy() and display no progress?
+    if progress is False:
+      fast_mode = True
+    else:
+      with Pfx("stat(%r)", filename):
+        S = os.stat(filename)
+      fast_mode = stat.S_ISREG(S.st_mode)
+    if fast_mode:
+      with NamedTemporaryFile(**nt_kw) as T:
+        with Pfx("shutil.copy(%r,%r)", filename, T.name):
+          shutil.copy(filename, T.name)
+        yield T
+    else:
+      with Pfx("open(%r)", filename):
+        with open(filename, 'rb') as f2:
+          with NamedTemporaryCopy(f2, progress=progress,
+                                  progress_label=progress_label, **nt_kw) as T:
+            yield T
+    return
+  prefix = nt_kw.pop('prefix', None)
+  if prefix is None:
+    prefix = 'NamedTemporaryCopy'
+  # prepare the buffer and try to infer the length
+  if isinstance(f, CornuCopyBuffer):
+    length = None
+    bfr = f
+  else:
+    if isinstance(f, int):
+      fd = f
+      bfr = CornuCopyBuffer.from_fd(fd)
+    else:
+      bfr = CornuCopyBuffer.from_file(f)
+      try:
+        fd = f.fileno()
+      except AttributeError:
+        fd = None
+    if fd is None:
+      length = None
+    else:
+      S = os.fstat(fd)
+      length = S.st_size if stat.S_ISREG(S.st_mode) else None
+  # determine whether we need a progress bar
+  if isinstance(progress, bool):
+    need_bar = progress
+    progress = None
+  elif isinstance(progress, int):
+    need_bar = length is None or length >= progress
+    progress = None
+  else:
+    need_bar = False
+    assert isinstance(progress, Progress)
+  with NamedTemporaryFile(prefix=prefix, **nt_kw) as T:
+    it = (
+        bfr if need_bar else progressbar(
+            bfr,
+            label=progress_label,
+            total=length,
+            itemlenfunc=len,
+            units_scale=BINARY_BYTES_SCALE,
+        )
+    )
+    nbs = 0
+    for bs in it:
+      while bs:
+        nwritten = T.write(bs)
+        if progress is not None:
+          progress += nwritten
+        if nwritten != len(bs):
+          warning(
+              "NamedTemporaryCopy: %r.write(%d bytes) => %d",
+              T.name,
+              len(bs),
+              nwritten,
+          )
+          bs = bs[nwritten:]
+        else:
+          bs = b''
+        nbs += nwritten
+    bfr.close()
+    T.flush()
+    if length is not None and nbs != length:
+      warning(
+          "NamedTemporaryCopy: given length=%s, wrote %d bytes to %r",
+          length,
+          nbs,
+          T.name,
+      )
+    yield T
+
+# pylint: disable=too-many-arguments
+def rewrite(
+    filepath,
+    srcf,
+    mode='w',
+    backup_ext=None,
+    do_rename=False,
+    do_diff=None,
+    empty_ok=False,
+    overwrite_anyway=False
+):
+  ''' Rewrite the file `filepath` with data from the file object `srcf`.
+      Return `True` if the content was changed, `False` if unchanged.
+
+      Parameters:
+      * `filepath`: the name of the file to rewrite.
+      * `srcf`: the source file containing the new content.
+      * `mode`: the write-mode for the file, default `'w'` (for text);
+        use `'wb'` for binary data.
+      * `empty_ok`: if true (default `False`),
+        do not raise `ValueError` if the new data are empty.
+      * `overwrite_anyway`: if true (default `False`),
+        skip the content check and overwrite unconditionally.
+      * `backup_ext`: if a nonempty string,
+        take a backup of the original at `filepath + backup_ext`.
+      * `do_diff`: if not `None`, call `do_diff(filepath,tempfile)`.
+      * `do_rename`: if true (default `False`),
+        rename the temp file to `filepath`
+        after copying the permission bits.
+        Otherwise (default), copy the tempfile to `filepath`;
+        this preserves the file's inode and permissions etc.
+  '''
+  with Pfx("rewrite(%r)", filepath):
+    with NamedTemporaryFile(dir=dirname(filepath), mode=mode) as T:
+      T.write(srcf.read())
+      T.flush()
+      if not empty_ok:
+        st = os.stat(T.name)
+        if st.st_size == 0:
+          raise ValueError("no data in temp file")
+      if do_diff or not overwrite_anyway:
+        # need to compare data
+        if compare(T.name, filepath):
+          # data the same, do nothing
+          return False
+        if do_diff:
+          # call the supplied differ
+          do_diff(filepath, T.name)
+      if do_rename:
+        # rename new file into old path
+        # tries to preserve perms, but does nothing for other metadata
+        shutil.copymode(filepath, T.name)
+        if backup_ext:
+          os.link(filepath, filepath + backup_ext)
+        os.rename(T.name, filepath)
+      else:
+        # overwrite old file - preserves perms, ownership, hard links
+        if backup_ext:
+          shutil.copy2(filepath, filepath + backup_ext)
+        shutil.copyfile(T.name, filepath)
+  return True
+
+@contextmanager
+def rewrite_cmgr(filepath, mode='w', **kw):
+  ''' Rewrite a file, presented as a context manager.
+
+      Parameters:
+      * `mode`: file write mode, defaulting to "w" for text.
+
+      Other keyword parameters are passed to `rewrite()`.
+
+      Example:
+
+          with rewrite_cmgr(pathname, do_rename=True) as f:
+              ... write new content to f ...
+  '''
+  with NamedTemporaryFile(mode=mode) as T:
+    yield T
+    T.flush()
+    with open(T.name, 'rb') as f:
+      return rewrite(filepath, mode='wb', srcf=f, **kw)
+
+def abspath_from_file(path, from_file):
+  ''' Return the absolute path of `path` with respect to `from_file`,
+      as one might do for an include file.
+  '''
+  if not isabspath(path):
+    if not isabspath(from_file):
+      from_file = abspath(from_file)
+    path = joinpath(dirname(from_file), path)
+  return path
+
+def poll_file(path, old_state, reload_file, missing_ok=False):
+  ''' Watch a file for modification by polling its state as obtained
+      by `FileState()`.
+      Call `reload_file(path)` if the state changes.
+      Return `(new_state,reload_file(path))` if the file was modified
+      and was unchanged (stable state) before and after the reload_file().
+      Otherwise return `(None,None)`.
+
+      This may raise an `OSError` if the `path` cannot be `os.stat()`ed
+      and of course for any exceptions that occur calling `reload_file`.
+
+      If `missing_ok` is true then a failure to `os.stat()` which
+      raises `OSError` with `ENOENT` will just return `(None,None)`.
+  '''
+  try:
+    new_state = FileState(path)
+  except OSError as e:
+    if e.errno == errno.ENOENT:
+      if missing_ok:
+        return None, None
+    raise
+  if old_state is None or old_state != new_state:
+    # first stat or changed stat
+    R = reload_file(path)
+    try:
+      new_new_state = FileState(path)
+    except OSError as e:
+      if e.errno == errno.ENOENT:
+        if missing_ok:
+          return None, None
+      raise
+    # make sure file was unchanged
+    if new_new_state == new_state:
+      return new_state, R
+  return None, None
+
+def files_property(func):
+  ''' A property whose value reloads if any of a list of files changes.
+
+      Note: this is just the default mode for `make_files_property`.
+
+      `func` accepts the file path and returns the new value.
+      The underlying attribute name is `'_'+func.__name__`,
+      the default from `make_files_property()`.
+      The attribute *{attr_name}*`_lock` is a mutex controlling access to the property.
+      The attributes *{attr_name}*`_filestates` and *{attr_name}*`_paths` track the
+      associated file states.
+      The attribute *{attr_name}*`_lastpoll` tracks the last poll time.
+
+      The decorated function is passed the current list of files
+      and returns the new list of files and the associated value.
+
+      One example use would be a configuration file with recurive
+      include operations; the inner function would parse the first
+      file in the list, and the parse would accumulate this filename
+      and those of any included files so that they can be monitored,
+      triggering a fresh parse if one changes.
+
+      Example:
+
+          class C(object):
+            def __init__(self):
+              self._foo_path = '.foorc'
+            @files_property
+            def foo(self,paths):
+              new_paths, result = parse(paths[0])
+              return new_paths, result
+
+      The load function is called on the first access and on every
+      access thereafter where an associated file's `FileState` has
+      changed and the time since the last successful load exceeds
+      the poll_rate (1s). An attempt at avoiding races is made by
+      ignoring reloads that raise exceptions and ignoring reloads
+      where files that were stat()ed during the change check have
+      changed state after the load.
+  '''
+  return make_files_property()(func)
+
+# pylint: disable=too-many-statements
+@fmtdoc
+def make_files_property(
+    attr_name=None, unset_object=None, poll_rate=DEFAULT_POLL_INTERVAL
+):
+  ''' Construct a decorator that watches multiple associated files.
+
+      Parameters:
+      * `attr_name`: the underlying attribute, default: `'_'+func.__name__`
+      * `unset_object`: the sentinel value for "uninitialised", default: `None`
+      * `poll_rate`: how often in seconds to poll the file for changes,
+        default from `DEFAULT_POLL_INTERVAL`: `{DEFAULT_POLL_INTERVAL}`
+
+      The attribute *attr_name*`_lock` controls access to the property.
+      The attributes *attr_name*`_filestates` and *attr_name*`_paths` track the
+      associated files' state.
+      The attribute *attr_name*`_lastpoll` tracks the last poll time.
+
+      The decorated function is passed the current list of files
+      and returns the new list of files and the associated value.
+
+      One example use would be a configuration file with recursive
+      include operations; the inner function would parse the first
+      file in the list, and the parse would accumulate this filename
+      and those of any included files so that they can be monitored,
+      triggering a fresh parse if one changes.
+
+      Example:
+
+          class C(object):
+            def __init__(self):
+              self._foo_path = '.foorc'
+            @files_property
+            def foo(self,paths):
+              new_paths, result = parse(paths[0])
+              return new_paths, result
+
+      The load function is called on the first access and on every
+      access thereafter where an associated file's `FileState` has
+      changed and the time since the last successful load exceeds
+      the `poll_rate`.
+
+      An attempt at avoiding races is made by
+      ignoring reloads that raise exceptions and ignoring reloads
+      where files that were `os.stat()`ed during the change check have
+      changed state after the load.
+  '''
+
+  # pylint: disable=too-many-statements
+  def made_files_property(func):
+    if attr_name is None:
+      attr_value = '_' + func.__name__
+    else:
+      attr_value = attr_name
+    attr_lock = attr_value + '_lock'
+    attr_filestates = attr_value + '_filestates'
+    attr_paths = attr_value + '_paths'
+    attr_lastpoll = attr_value + '_lastpoll'
+
+    # pylint: disable=too-many-statements,too-many-branches
+    def getprop(self):
+      ''' Try to reload the property value from the file if the property value
+          is stale and the file has been modified since the last reload.
+      '''
+      with getattr(self, attr_lock):
+        now = time.time()
+        then = getattr(self, attr_lastpoll, None)
+        if then is None or then + poll_rate <= now:
+          setattr(self, attr_lastpoll, now)
+          old_paths = getattr(self, attr_paths)
+          old_filestates = getattr(self, attr_filestates, None)
+          preload_filestate_map = {}
+          if old_filestates is None:
+            changed = True
+          else:
+            changed = False
+            # Instead of breaking out of the loop below on the first change
+            # found we actually stat every file path because we want to
+            # maximise the coverage of the stability check after the load.
+            for path, old_filestate in zip(old_paths, old_filestates):
+              try:
+                new_filestate = FileState(path)
+              except OSError:
+                changed = True
+              else:
+                preload_filestate_map[path] = new_filestate
+                if old_filestate != new_filestate:
+                  changed = True
+          if changed:
+            try:
+              new_paths, new_value = func(self, old_paths)
+              new_filestates = [FileState(new_path) for new_path in new_paths]
+            except NameError:
+              raise
+            except AttributeError:
+              raise
+            except Exception as e:  # pylint: disable=broad-except
+              new_value = getattr(self, attr_value, unset_object)
+              if new_value is unset_object:
+                raise
+              debug(
+                  "exception reloading .%s, keeping cached value: %s",
+                  attr_value, e
+              )
+            else:
+              # examine new filestates in case they changed during load
+              # _if_ we knew about them from the earlier load
+              stable = True
+              for path, new_filestate in zip(new_paths, new_filestates):
+                if path in preload_filestate_map:
+                  if preload_filestate_map[path] != new_filestate:
+                    stable = False
+                    break
+              if stable:
+                setattr(self, attr_value, new_value)
+                setattr(self, attr_paths, new_paths)
+                setattr(self, attr_filestates, new_filestates)
+      return getattr(self, attr_value, unset_object)
+
+    return property(getprop)
+
+  return made_files_property
+
+# pylint: disable=too-many-branches
+@uses_runstate
+@pfx
+def makelockfile(
+    path,
+    *,
+    ext=None,
+    poll_interval=None,
+    timeout=None,
+    runstate: RunState,
+    keepopen=False,
+    max_interval=37,
+):
+  ''' Create a lockfile and return its path.
+      If `keepopen`, return a `(lockpath,lockfd)` 2-tuple.
+
+      The lockfile can be removed with `os.remove`.
+      This is the core functionality supporting the `lockfile()`
+      context manager.
+
+      Parameters:
+      * `path`: the base associated with the lock file,
+        often the filesystem object whose access is being managed.
+      * `ext`: the extension to the base used to construct the lockfile name.
+        Default: ".lock"
+      * `timeout`: maximum time to wait before failing.
+        Default: `None` (wait forever).
+        Note that zero is an accepted value
+        and requires the lock to succeed on the first attempt.
+      * `poll_interval`: polling frequency when timeout is not 0.
+      * `runstate`: optional `RunState` duck instance supporting cancellation.
+        Note that if a cancelled `RunState` is provided
+        no attempt will be made to make the lockfile.
+      * `keepopen`: optional flag, default `False`:
+        if true, do not close the lockfile and return `(lockpath,lockfd)`
+        being the lock file path and the open file descriptor
+  '''
+  if poll_interval is None:
+    poll_interval = DEFAULT_POLL_INTERVAL
+  if ext is None:
+    ext = '.lock'
+  if timeout is not None and timeout < 0:
+    raise ValueError("timeout should be None or >= 0, not %r" % (timeout,))
+  start = None
+  lockpath = path + ext
+  with Pfx("makelockfile: %r", lockpath):
+    while True:
+      if runstate.cancelled:
+        warning(
+            "%s cancelled; pid %d waited %ds", runstate, os.getpid(),
+            0 if start is None else time.time() - start
+        )
+        raise CancellationError("lock acquisition cancelled")
+      try:
+        lockfd = os.open(lockpath, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0)
+      except OSError as e:
+        if e.errno != errno.EEXIST:
+          raise
+        if timeout is not None and timeout <= 0:
+          # immediate failure
+          # pylint: disable=raise-missing-from
+          raise TimeoutError("pid %d timed out" % (os.getpid(),), timeout)
+        now = time.time()
+        # post: timeout is None or timeout > 0
+        if start is None:
+          # first try - set up counters
+          start = now
+          complaint_last = start
+          complaint_interval = 2 * max(DEFAULT_POLL_INTERVAL, poll_interval)
+        elif now - complaint_last >= complaint_interval:
+          warning("pid %d waited %ds", os.getpid(), now - start)
+          complaint_last = now
+          complaint_interval = min(complaint_interval * 2, max_interval)
+        # post: start is set
+        if timeout is None:
+          sleep_for = poll_interval
+        else:
+          sleep_for = min(poll_interval, start + timeout - now)
+        # test for timeout
+        if sleep_for <= 0:
+          # pylint: disable=raise-missing-from
+          raise TimeoutError("pid %d timed out" % (os.getpid(),), timeout)
+        time.sleep(sleep_for)
+        continue
+      else:
+        break
+    if keepopen:
+      return lockpath, lockfd
+    os.close(lockfd)
+    return lockpath
+
+@contextmanager
+def lockfile(path, _lockmap={}, _lockmap_lock=Lock(), **makelockfile_kw):
+  ''' A context manager which takes and holds a lock file.
+      An open file descriptor is kept for the lock file as well
+      to aid locating the process holding the lock file using eg `lsof`.
+      This is just a context manager shim for `makelockfile`
+      and all keyword arguments are plumbed through.
+  '''
+  with _lockmap_lock:
+    try:
+      nrlock = _lockmap[path]
+    except KeyError:
+      nrlock = _lockmap[path] = NRLock(path)
+  with nrlock:
+    lockpath, lockfd = makelockfile(path, keepopen=True, **makelockfile_kw)
+    try:
+      yield lockpath
+    finally:
+      try:
+        pfx_call(os.remove, lockpath)
+      except FileNotFoundError as e:
+        warning("lock file already removed: %s", e)
+      pfx_call(os.close, lockfd)
+
+def crop_name(name, ext=None, name_max=255):
+  ''' Crop a file basename so as not to exceed `name_max` in length.
+      Return the original `name` if it already short enough.
+      Otherwise crop `name` before the file extension
+      to make it short enough.
+
+      Parameters:
+      * `name`: the file basename to crop
+      * `ext`: optional file extension;
+        the default is to infer the extension with `os.path.splitext`.
+      * `name_max`: optional maximum length, default: `255`
+  '''
+  if ext is None:
+    base, ext = splitext(name)
+  else:
+    base = cutsuffix(name, ext)
+    if base is name:
+      base, ext = splitext(name)
+  max_base_len = name_max - len(ext)
+  if max_base_len < 0:
+    raise ValueError(
+        "cannot crop name before ext %r to <=%s: name=%r" %
+        (ext, name_max, name)
+    )
+  if len(base) <= max_base_len:
+    return name
+  return base[:max_base_len] + ext
+
+def max_suffix(dirpath, prefix):
+  ''' Compute the highest existing numeric suffix
+      for names starting with `prefix`.
+
+      This is generally used as a starting point for picking
+      a new numeric suffix.
+  '''
+  prefix = ustr(prefix)
+  maxn = None
+  pfxlen = len(prefix)
+  for e in os.listdir(dirpath):
+    e = ustr(e)
+    if len(e) <= pfxlen or not e.startswith(prefix):
+      continue
+    tail = e[pfxlen:]
+    if tail.isdigit():
+      n = int(tail)
+      if maxn is None:
+        maxn = n
+      elif maxn < n:
+        maxn = n
+  return maxn
+
+# pylint: disable=too-many-branches
+def mkdirn(path, sep=''):
+  ''' Create a new directory named `path+sep+n`,
+      where `n` exceeds any name already present.
+
+      Parameters:
+      * `path`: the basic directory path.
+      * `sep`: a separator between `path` and `n`.
+        Default: `''`
+  '''
+  with Pfx("mkdirn(path=%r, sep=%r)", path, sep):
+    if os.sep in sep:
+      raise ValueError("sep contains os.sep (%r)" % (os.sep,))
+    opath = path
+    if not path:
+      path = '.' + os.sep
+
+    if path.endswith(os.sep):
+      if sep:
+        raise ValueError(
+            "mkdirn(path=%r, sep=%r): using non-empty sep"
+            " with a trailing %r seems nonsensical" % (path, sep, os.sep)
+        )
+      dirpath = path[:-len(os.sep)]
+      prefix = ''
+    else:
+      dirpath = dirname(path)
+      if not dirpath:
+        dirpath = '.'
+      prefix = basename(path) + sep
+
+    if not isdir(dirpath):
+      error("parent not a directory: %r", dirpath)
+      return None
+
+    # do a quick scan of the directory to find
+    # if any names of the desired form already exist
+    # in order to start after them
+    maxn = max_suffix(dirpath, prefix)
+    if maxn is None:
+      newn = 0
+    else:
+      newn = maxn
+
+    while True:
+      newn += 1
+      newpath = path + sep + str(newn)
+      try:
+        os.mkdir(newpath)
+      except OSError as e:
+        if e.errno == errno.EEXIST:
+          # taken, try new value
+          continue
+        error("mkdir(%s): %s", newpath, e)
+        return None
+      if not opath:
+        newpath = basename(newpath)
+      return newpath
+
+def tmpdir():
+  ''' Return the pathname of the default temporary directory for scratch data,
+      the environment variable `$TMPDIR` or `'/tmp'`.
+  '''
+  return os.environ.get('TMPDIR', '/tmp')
+
+def tmpdirn(tmp=None):
+  ''' Make a new temporary directory with a numeric suffix.
+  '''
+  if tmp is None:
+    tmp = tmpdir()
+  return mkdirn(joinpath(tmp, basename(sys.argv[0])))
+
+def find(path, select=None, sort_names=True):
+  ''' Walk a directory tree `path`
+      yielding selected paths.
+
+      Note: not selecting a directory prunes all its descendants.
+  '''
+  if select is None:
+    select = lambda _: True
+  for dirpath, dirnames, filenames in os.walk(path):
+    if select(dirpath):
+      yield dirpath
+    else:
+      dirnames[:] = []
+      continue
+    if sort_names:
+      dirnames[:] = sorted(dirnames)
+      filenames[:] = sorted(filenames)
+    for filename in filenames:
+      filepath = joinpath(dirpath, filename)
+      if select(filepath):
+        yield filepath
+    dirnames[:] = [
+        dirname for dirname in dirnames if select(joinpath(dirpath, dirname))
+    ]
+
+def findup(path, test, first=False):
+  ''' Test the pathname `abspath(path)` and each of its ancestors
+      against the callable `test`,
+      yielding paths satisfying the test.
+
+      If `first` is true (default `False`)
+      this function always yields exactly one value,
+      either the first path satisfying the test or `None`.
+      This mode supports a use such as:
+
+          matched_path = next(findup(path, test, first=True))
+          # post condition: matched_path will be `None` on no match
+          # otherwise the first matching path
+  '''
+  path = abspath(path)
+  while True:
+    if test(path):
+      yield path
+      if first:
+        return
+    up = dirname(path)
+    if up == path:
+      break
+    path = up
+  if first:
+    yield None
+
+def common_path_prefix(*paths):
+  ''' Return the common path prefix of the `paths`.
+
+      Note that the common prefix of `'/a/b/c1'` and `'/a/b/c2'`
+      is `'/a/b/'`, _not_ `'/a/b/c'`.
+
+      Callers may find it useful to preadjust the supplied paths
+      with `normpath`, `abspath` or `realpath` from `os.path`;
+      see the `os.path` documentation for the various caveats
+      which go with those functions.
+
+      Examples:
+
+          >>> # the obvious
+          >>> common_path_prefix('', '')
+          ''
+          >>> common_path_prefix('/', '/')
+          '/'
+          >>> common_path_prefix('a', 'a')
+          'a'
+          >>> common_path_prefix('a', 'b')
+          ''
+          >>> # nonempty directory path prefixes end in os.sep
+          >>> common_path_prefix('/', '/a')
+          '/'
+          >>> # identical paths include the final basename
+          >>> common_path_prefix('p/a', 'p/a')
+          'p/a'
+          >>> # the comparison does not normalise paths
+          >>> common_path_prefix('p//a', 'p//a')
+          'p//a'
+          >>> common_path_prefix('p//a', 'p//b')
+          'p//'
+          >>> common_path_prefix('p//a', 'p/a')
+          'p/'
+          >>> common_path_prefix('p/a', 'p/b')
+          'p/'
+          >>> # the comparison strips complete unequal path components
+          >>> common_path_prefix('p/a1', 'p/a2')
+          'p/'
+          >>> common_path_prefix('p/a/b1', 'p/a/b2')
+          'p/a/'
+          >>> # contrast with cs.lex.common_prefix
+          >>> common_prefix('abc/def', 'abc/def1')
+          'abc/def'
+          >>> common_path_prefix('abc/def', 'abc/def1')
+          'abc/'
+          >>> common_prefix('abc/def', 'abc/def1', 'abc/def2')
+          'abc/def'
+          >>> common_path_prefix('abc/def', 'abc/def1', 'abc/def2')
+          'abc/'
+  '''
+  prefix = common_prefix(*paths)
+  if not prefix.endswith(os.sep):
+    path0 = paths[0]
+    if not all(map(lambda path: path == path0, paths)):
+      # strip basename from prefix
+      base = basename(prefix)
+      prefix = prefix[:-len(base)]
+  return prefix
+
+class Pathname(str):
+  ''' Subclass of str presenting convenience properties useful for
+      format strings related to file paths.
+  '''
+
+  _default_prefixes = (('$HOME/', '~/'),)
+
+  def __format__(self, fmt_spec):
+    ''' Calling format(<Pathname>, fmt_spec) treat `fmt_spec` as a new style
+        formatting string with a single positional parameter of `self`.
+    '''
+    if fmt_spec == '':
+      return str(self)
+    return fmt_spec.format(self)
+
+  @property
+  def dirname(self):
+    ''' The dirname of the Pathname.
+    '''
+    return Pathname(dirname(self))
+
+  @property
+  def basename(self):
+    ''' The basename of this Pathname.
+    '''
+    return Pathname(basename(self))
+
+  @property
+  def abs(self):
+    ''' The absolute form of this Pathname.
+    '''
+    return Pathname(abspath(self))
+
+  @property
+  def isabs(self):
+    ''' Whether this Pathname is an absolute Pathname.
+    '''
+    return isabspath(self)
+
+  @property
+  def short(self):
+    ''' The shortened form of this Pathname.
+    '''
+    return self.shorten()
+
+  def shorten(self, prefixes=None):
+    ''' Shorten a Pathname using ~ and ~user.
+    '''
+    return shortpath(self, prefixes=prefixes)
+
+def iter_fd(fd, **kw):
+  ''' Iterate over data from the file descriptor `fd`.
+  '''
+  for bs in CornuCopyBuffer.from_fd(fd, **kw):
+    yield bs
+
+def iter_file(f, **kw):
+  ''' Iterate over data from the file `f`.
+  '''
+  for bs in CornuCopyBuffer.from_file(f, **kw):
+    yield bs
+
+def byteses_as_fd(bss, **kw):
+  ''' Deliver the iterable of bytes `bss` as a readable file descriptor.
+      Return the file descriptor.
+      Any keyword arguments are passed to `CornuCopyBuffer.as_fd`.
+
+      Example:
+
+           # present a passphrase for use as in input file descrptor
+           # for a subprocess
+           rfd = byteses_as_fd([(passphrase + '\n').encode()])
+  '''
+  return CornuCopyBuffer(bss).as_fd(**kw)
+
+def datafrom_fd(fd, offset=None, readsize=None, aligned=True, maxlength=None):
+  ''' General purpose reader for file descriptors yielding data from `offset`.
+      **Note**: This does not move the file descriptor position
+      **if** the file is seekable.
+
+      Parameters:
+      * `fd`: the file descriptor from which to read.
+      * `offset`: the offset from which to read.
+        If omitted, use the current file descriptor position.
+      * `readsize`: the read size, default: `DEFAULT_READSIZE`
+      * `aligned`: if true (the default), the first read is sized
+        to align the new offset with a multiple of `readsize`.
+      * `maxlength`: if specified yield no more than this many bytes of data.
+  '''
+  try:
+    cur_offset = os.lseek(fd, 0, SEEK_CUR)
+    is_seekable = True
+  except OSError:
+    cur_offset = 0  # guess
+    is_seekable = False
+  if offset is None:
+    offset = cur_offset
+  if readsize is None:
+    readsize = DEFAULT_READSIZE
+  if aligned:
+    # do an initial read to align all subsequent reads
+    alignsize = offset % readsize
+    if alignsize > 0:
+      if maxlength is not None:
+        alignsize = min(maxlength, alignsize)
+      bs = pread(fd, alignsize, offset) if is_seekable else read(fd, alignsize)
+      if not bs:
+        return
+      yield bs
+      bslen = len(bs)
+      offset += bslen
+      if maxlength is not None:
+        maxlength -= bslen
+  while maxlength is None or maxlength > 0:
+    if maxlength is not None:
+      readsize = min(readsize, maxlength)
+    bs = pread(fd, readsize, offset) if is_seekable else read(fd, readsize)
+    if not bs:
+      return
+    yield bs
+    bslen = len(bs)
+    offset += bslen
+    if maxlength is not None:
+      maxlength -= bslen
+
+@strable(open_func=lambda filename: os.open(filename, flags=O_RDONLY))
+def datafrom(f, offset=None, readsize=None, maxlength=None):
+  ''' General purpose reader for files yielding data from `offset`.
+
+      *WARNING*: this function might move the file pointer.
+
+      Parameters:
+      * `f`: the file from which to read data;
+        if a string, the file is opened with mode="rb";
+        if an int, treated as an OS file descriptor;
+        otherwise presumed to be a file-like object.
+        If that object has a `.fileno()` method, treat that as an
+        OS file descriptor and use it.
+      * `offset`: starting offset for the data
+      * `maxlength`: optional maximum amount of data to yield
+      * `readsize`: read size, default DEFAULT_READSIZE.
+
+      For file-like objects, the read1 method is used in preference
+      to read if available. The file pointer is briefly moved during
+      fetches.
+  '''
+  if readsize is None:
+    readsize = DEFAULT_READSIZE
+  if isinstance(f, int):
+    # operating system file descriptor
+    for data in datafrom_fd(f, offset=offset, readsize=readsize,
+                            maxlength=maxlength):
+      yield data
+    return
+  # see if the file has a fileno; if so use datafrom_fd
+  try:
+    get_fileno = f.fileno
+  except AttributeError:
+    pass
+  else:
+    fd = get_fileno()
+    if stat.S_ISREG(os.fstat(fd).st_mode):
+      for data in datafrom_fd(fd, offset=offset, readsize=readsize,
+                              maxlength=maxlength):
+        yield data
+      return
+  # presume a file-like object
+  try:
+    read1 = f.read1
+  except AttributeError:
+    read1 = f.read
+  tell = f.tell
+  seek = f.seek
+  while maxlength is None or maxlength > 0:
+    offset0 = tell()
+    seek(offset, SEEK_SET)
+    n = readsize
+    if maxlength is not None:
+      n = min(n, maxlength)
+    bs = read1(n)
+    seek(offset0)
+    if not bs:
+      break
+    yield bs
+    offset += len(bs)
+    if maxlength is not None:
+      maxlength -= len(bs)
+      assert maxlength >= 0
+
+class ReadMixin(object):
+  ''' Useful read methods to accomodate modes not necessarily available in a class.
+
+      Note that this mixin presumes that the attribute `self._lock`
+      is a threading.RLock like context manager.
+
+      Classes using this mixin should consider overriding the default
+      .datafrom method with something more efficient or direct.
+  '''
+
+  def datafrom(self, offset, readsize=None):
+    ''' Yield data from the specified `offset` onward in some
+        approximation of the "natural" chunk size.
+
+        *NOTE*: UNLIKE the global datafrom() function, this method
+        MUST NOT move the logical file position. Implementors may need
+        to save and restore the file pointer within a lock around
+        the I/O if they do not use a direct access method like
+        os.pread.
+
+        The aspiration here is to read data with only a single call
+        to the underlying storage, and to return the chunks in
+        natural sizes instead of some default read size.
+
+        Classes using this mixin must implement this method.
+    '''
+    raise NotImplementedError(
+        "return an iterator which does not change the file offset"
+    )
+
+  def bufferfrom(self, offset):
+    ''' Return a CornuCopyBuffer from the specified `offset`.
+    '''
+    return CornuCopyBuffer(self.datafrom(offset), offset=offset)
+
+  # pylint: disable=too-many-branches
+  def read(self, size=-1, offset=None, longread=False):
+    ''' Read up to `size` bytes, honouring the "single system call"
+        spirit unless `longread` is true.
+
+        Parameters:
+        * `size`: the number of bytes requested. A size of -1 requests
+          all bytes to the end of the file.
+        * `offset`: the starting point of the read; if None, use the
+          current file position; if not None, seek to this position
+          before reading, even if `size` == 0.
+        * `longread`: switch from "single system call" to "as many
+          as required to obtain `size` bytes"; short data will still
+          be returned if the file is too short.
+    '''
+    bfr = getattr(self, '_reading_bfr', None)
+    if offset is None:
+      if bfr is None:
+        offset = self.tell()
+      else:
+        offset = bfr.offset
+    if size == -1:
+      size = max(len(self) - offset, 0)
+    if size == 0:
+      return b''
+    if longread:
+      bss = []
+    while size > 0:
+      with self._lock:
+        # We need to retest on each iteration because other reads
+        # may be interleaved, interfering with the buffer.
+        if bfr is None or bfr.offset != offset:
+          ##if bfr is not None:
+          ##  info(
+          ##      "ReadMixin.read: new bfr from offset=%d (old bfr was %s)",
+          ##      offset, bfr)
+          self._reading_bfr = bfr = self.bufferfrom(offset)
+        bfr.extend(1, short_ok=True)
+        if not bfr.buf:
+          break
+        consume = min(size, len(bfr.buf))
+        assert consume > 0
+        chunk = bfr.take(consume)
+        offset += consume
+        self.seek(offset)
+      assert len(chunk) == consume
+      if longread:
+        bss.append(chunk)
+      else:
+        return chunk
+      size -= consume
+    if not bss:
+      return b''
+    if len(bss) == 1:
+      return bss[0]
+    return b''.join(bss)
+
+  def read_n(self, n):
+    ''' Read `n` bytes of data and return them.
+
+        Unlike traditional file.read(), RawIOBase.read() may return short
+        data, thus this workalike, which may only return short data if it
+        hits EOF.
+    '''
+    if n < 1:
+      raise ValueError("n two low, expected >=1, got %r" % (n,))
+    data = bytearray(n)
+    nread = self.readinto(data)
+    if nread != len(data):
+      raise RuntimeError(
+          "  WRONG NUMBER OF BYTES(%d): data=%s" % (nread, data)
+      )
+    return memoryview(data)[:nread] if nread != n else data
+
+  @locked
+  def readinto(self, barray):
+    ''' Read data into a bytearray.
+    '''
+    needed = len(barray)
+    boff = 0
+    for bs in self.datafrom(self.tell()):
+      if not bs:
+        break
+      if len(bs) > needed:
+        bs = memoryview(bs)[:needed]
+      bs_len = len(bs)
+      boff2 = boff + bs_len
+      barray[boff:boff2] = bs
+      boff = boff2
+      needed -= bs_len
+    return boff
+
+class BackedFile(ReadMixin):
+  ''' A RawIOBase duck type
+      which uses a backing file for initial data
+      and writes new data to a front scratch file.
+  '''
+
+  def __init__(self, back_file, dirpath=None):
+    ''' Initialise the BackedFile using `back_file` for the backing data.
+    '''
+    self._offset = 0
+    self._dirpath = dirpath
+    self._lock = RLock()
+    self.back_file = back_file
+    self.front_file = TemporaryFile(dir=dirpath, buffering=0)
+    self.front_range = Range()
+    self.read_only = False
+
+  def __len__(self):
+    back_file = self.back_file
+    try:
+      back_len = len(back_file)
+    except TypeError:
+      back_pos = back_file.tell()
+      back_len = back_file.seek(0, 2)
+      back_file.seek(back_pos, 0)
+    return max(self.front_range.end, back_len)
+
+  @locked
+  def switch_back_file(self, new_back_file):
+    ''' Switch out one back file for another. Return the old back file.
+    '''
+    old_back_file = self.back_file
+    self.back_file = new_back_file
+    return old_back_file
+
+  def __enter__(self):
+    ''' BackedFile instances offer a context manager that take the lock,
+        allowing synchronous use of the file
+        without implementing a suite of special methods like pread/pwrite.
+    '''
+    self._lock.acquire()
+
+  def __exit__(self, *e):
+    self._lock.release()
+
+  def close(self):
+    ''' Close the BackedFile.
+        Flush contents. Close the front_file if necessary.
+    '''
+    self.front_file.close()
+    self.front_file = None
+
+  def tell(self):
+    ''' Report the current file pointer offset.
+    '''
+    return self._offset
+
+  @locked
+  def seek(self, pos, whence=SEEK_SET):
+    ''' Adjust the current file pointer offset.
+    '''
+    if whence == SEEK_SET:
+      self._offset = pos
+    elif whence == SEEK_CUR:
+      self._offset += pos
+    elif whence == SEEK_END:
+      endpos = self.back_file.seek(0, SEEK_END)
+      if self.front_range is not None:
+        endpos = max(endpos, self.front_range.end)
+      self._offset = endpos
+    else:
+      raise ValueError("unsupported whence value %r" % (whence,))
+
+  def datafrom(self, offset):
+    ''' Generator yielding natural chunks from the file commencing at offset.
+    '''
+    global_datafrom = globals()['datafrom']
+    front_file = self.front_file
+    try:
+      front_datafrom = front_file.datafrom
+    except AttributeError:
+      front_datafrom = partial(global_datafrom, front_file)
+    back_file = self.back_file
+    try:
+      back_datafrom = back_file.datafrom
+    except AttributeError:
+      back_datafrom = partial(global_datafrom, back_file)
+    for in_front, span in self.front_range.slices(offset, len(self)):
+      consume = len(span)
+      assert consume > 0
+      if in_front:
+        chunks = front_datafrom(span.start)
+      else:
+        chunks = back_datafrom(span.start)
+      for bs in chunks:
+        assert len(bs) > 0
+        if len(bs) > consume:
+          bs = memoryview(bs)[:consume]
+        yield bs
+        bs_len = len(bs)
+        consume -= bs_len
+        if consume <= 0:
+          break
+        offset += bs_len
+
+  @locked
+  def write(self, b):
+    ''' Write data to the front_file.
+    '''
+    if self.read_only:
+      raise RuntimeError("write to read-only BackedFile")
+    front_file = self.front_file
+    start = self._offset
+    front_file.seek(start)
+    written = front_file.write(b)
+    if written is None:
+      warning(
+          "front_file.write() returned None, assuming %d bytes written, data=%r",
+          len(b), b
+      )
+      written = len(b)
+    self.front_range.add_span(start, start + written)
+    return written
+
+# pylint: disable=too-few-public-methods,protected-access
+class BackedFile_TestMethods(object):
+  ''' Mixin for testing subclasses of BackedFile.
+      Tests self.backed_fp.
+  '''
+
+  # pylint: disable=no-member
+  def _eq(self, a, b, opdesc):
+    ''' Convenience wrapper for assertEqual.
+    '''
+    ##if a == b:
+    ##  print("OK: %s: %r == %r" % (opdesc, a, b), file=sys.stderr)
+    self.assertEqual(a, b, "%s: got %r, expected %r" % (opdesc, a, b))
+
+  # pylint: disable=no-member
+  def test_BackedFile(self):
+    ''' Test function for a BackedFile to use in unit test suites.
+    '''
+    from random import randint  # pylint: disable=import-outside-toplevel
+    backing_text = self.backing_text
+    bfp = self.backed_fp
+    # test reading whole file
+    bfp.seek(0)
+    bfp_text = bfp.read_n(len(bfp))
+    self._eq(backing_text, bfp_text, "backing_text vs bfp_text")
+    # test reading first 512 bytes only
+    bfp.seek(0)
+    bfp_leading_text = bfp.read_n(512)
+    self._eq(
+        backing_text[:512], bfp_leading_text,
+        "leading 512 bytes of backing_text vs bfp_leading_text"
+    )
+    # test writing some data and reading it back
+    random_chunk = bytes(randint(0, 255) for x in range(256))
+    bfp.seek(512)
+    bfp.write(random_chunk)
+    # check that the front file has a single span of the right dimensions
+    ffp = bfp.front_file
+    fr = bfp.front_range
+    self.assertIsNotNone(ffp)
+    self.assertIsNotNone(fr)
+    self.assertEqual(len(fr._spans), 1, "fr._spans = %r" % (fr._spans,))
+    self.assertEqual(fr._spans[0].start, 512)
+    self.assertEqual(fr._spans[0].end, 768)
+    # read the random data back from the front file
+    ffp.seek(512)
+    front_chunk = ffp.read(256)
+    self.assertEqual(random_chunk, front_chunk)
+    # read the random data back from the BackedFile
+    bfp.seek(512)
+    bfp_chunk = bfp.read_n(256)
+    self.assertEqual(bfp_chunk, random_chunk)
+    # read a chunk that overlaps the old data and the new data
+    bfp.seek(256)
+    overlap_chunk = bfp.read_n(512)
+    self.assertEqual(
+        len(overlap_chunk), 512, "overlap_chunk not 512 bytes: %d:%s" %
+        (len(overlap_chunk), bytes(overlap_chunk))
+    )
+    self.assertEqual(overlap_chunk, backing_text[256:512] + random_chunk)
+
+class Tee(object):
+  ''' An object with .write, .flush and .close methods
+      which copies data to multiple output files.
+  '''
+
+  def __init__(self, *fps):
+    ''' Initialise the Tee; any arguments are taken to be output file objects.
+    '''
+    self._fps = list(fps)
+
+  def add(self, output):
+    ''' Add a new output.
+    '''
+    self._fps.append(output)
+
+  def write(self, data):
+    ''' Write the data to all the outputs.
+        Note: does not detect or accodmodate short writes.
+    '''
+    for fp in self._fps:
+      fp.write(data)
+
+  def flush(self):
+    ''' Flush all the outputs.
+    '''
+    for fp in self._fps:
+      fp.flush()
+
+  def close(self):
+    ''' Close all the outputs and close the Tee.
+    '''
+    for fp in self._fps:
+      fp.close()
+    self._fps = None
+
+@contextmanager
+def tee(fp, fp2):
+  ''' Context manager duplicating `.write` and `.flush` from `fp` to `fp2`.
+  '''
+  old_write = fp.write
+  old_flush = fp.flush
+
+  def _write(*a, **kw):
+    fp2.write(*a, **kw)
+    return old_write(*a, **kw)
+
+  def _flush(*a, **kw):
+    fp2.flush(*a, **kw)
+    return old_flush(*a, **kw)
+
+  with stackattrs(fp, write=_write, flush=_flush):
+    yield
+
+class NullFile(object):
+  ''' Writable file that discards its input.
+
+      Note that this is _not_ an open of `os.devnull`;
+      it just discards writes and is not the underlying file descriptor.
+  '''
+
+  def __init__(self):
+    ''' Initialise the file offset to 0.
+    '''
+    self.offset = 0
+
+  def write(self, data):
+    ''' Discard data, advance file offset by length of data.
+    '''
+    dlen = len(data)
+    self.offset += dlen
+    return dlen
+
+  def flush(self):
+    ''' Flush buffered data to the subsystem.
+    '''
+
+def file_data(fp, nbytes=None, rsize=None):
+  ''' Read `nbytes` of data from `fp` and yield the chunks as read.
+
+      Parameters:
+      * `nbytes`: number of bytes to read; if None read until EOF.
+      * `rsize`: read size, default DEFAULT_READSIZE.
+  '''
+  # try to use the "short read" flavour of read if available
+  if rsize is None:
+    rsize = DEFAULT_READSIZE
+  try:
+    read1 = fp.read1
+  except AttributeError:
+    read1 = fp.read
+  ##prefix = "file_data(fp, nbytes=%d)" % (nbytes,)
+  copied = 0
+  while nbytes is None or nbytes > 0:
+    to_read = rsize if nbytes is None else min(nbytes, rsize)
+    data = read1(to_read)
+    if not data:
+      if nbytes is not None:
+        if copied > 0:
+          # no warning of nothing copied - that is immediate end of file - valid
+          warning(
+              "early EOF: only %d bytes read, %d still to go", copied, nbytes
+          )
+      break
+    yield data
+    copied += len(data)
+    if nbytes is not None:
+      nbytes -= len(data)
+
+def copy_data(fpin, fpout, nbytes, rsize=None):
+  ''' Copy `nbytes` of data from `fpin` to `fpout`,
+      return the number of bytes copied.
+
+      Parameters:
+      * `nbytes`: number of bytes to copy.
+        If `None`, copy until EOF.
+      * `rsize`: read size, default `DEFAULT_READSIZE`.
+  '''
+  copied = 0
+  for chunk in file_data(fpin, nbytes, rsize):
+    fpout.write(chunk)
+    copied += len(chunk)
+  return copied
+
+def read_data(fp, nbytes, rsize=None):
+  ''' Read `nbytes` of data from `fp`, return the data.
+
+      Parameters:
+      * `nbytes`: number of bytes to copy.
+        If `None`, copy until EOF.
+      * `rsize`: read size, default `DEFAULT_READSIZE`.
+  '''
+  bss = list(file_data(fp, nbytes, rsize))
+  if not bss:
+    return b''
+  if len(bss) == 1:
+    return bss[0]
+  return b''.join(bss)
+
+def read_from(fp, rsize=None, tail_mode=False, tail_delay=None):
+  ''' Generator to present text or data from an open file until EOF.
+
+      Parameters:
+      * `rsize`: read size, default: DEFAULT_READSIZE
+      * `tail_mode`: if true, yield an empty chunk at EOF, allowing resumption
+        if the file grows.
+  '''
+  if rsize is None:
+    rsize = DEFAULT_READSIZE
+  if tail_delay is None:
+    tail_delay = DEFAULT_TAIL_PAUSE
+  elif not tail_mode:
+    raise ValueError(
+        "tail_mode=%r but tail_delay=%r" % (tail_mode, tail_delay)
+    )
+  while True:
+    chunk = fp.read(rsize)
+    if not chunk:
+      if tail_mode:
+        # indicate EOF and pause
+        yield chunk
+        time.sleep(tail_delay)
+      else:
+        break
+    else:
+      yield chunk
+
+def lines_of(fp, partials=None):
+  ''' Generator yielding lines from a file until EOF.
+      Intended for file-like objects that lack a line iteration API.
+  '''
+  if partials is None:
+    partials = []
+  return as_lines(read_from(fp), partials)
+
+# pylint: disable=redefined-builtin
+@contextmanager
+def atomic_filename(
+    filename,
+    exists_ok=False,
+    placeholder=False,
+    dir=None,
+    prefix=None,
+    suffix=None,
+    rename_func=None,
+    **tempfile_kw
+):
+  ''' A context manager to create `filename` atomicly on completion.
+      This yields a `NamedTemporaryFile` to use to create the file contents.
+      On completion the temporary file is renamed to the target name `filename`.
+
+      If the caller decides to _not_ create the target they may remove the
+      temporary file. This is not considered an error.
+
+      Parameters:
+      * `filename`: the file name to create
+      * `exists_ok`: default `False`;
+        if true it not an error if `filename` already exists
+      * `placeholder`: create a placeholder file at `filename`
+        while the real contents are written to the temporary file
+      * `dir`: passed to `NamedTemporaryFile`, specifies the directory
+        to hold the temporary file; the default is `dirname(filename)`
+        to ensure the rename is atomic
+      * `prefix`: passed to `NamedTemporaryFile`, specifies a prefix
+        for the temporary file; the default is a dot (`'.'`) plus the prefix
+        from `splitext(basename(filename))`
+      * `suffix`: passed to `NamedTemporaryFile`, specifies a suffix
+        for the temporary file; the default is the extension obtained
+        from `splitext(basename(filename))`
+      * `rename_func`: a callable accepting `(tempname,filename)`
+        used to rename the temporary file to the final name; the
+        default is `os.rename` if `exists_ok` or `placeholder`,
+        otherwise `rename_excl`.
+        This parametr exists to accept something such as `FSTags.move`.
+      Other keyword arguments are passed to the `NamedTemporaryFile` constructor.
+
+      Example:
+
+          >>> import os
+          >>> from os.path import exists as existspath
+          >>> fn = 'test_atomic_filename'
+          >>> with atomic_filename(fn, mode='w') as f:
+          ...     assert not existspath(fn)
+          ...     print('foo', file=f)
+          ...     assert not existspath(fn)
+          ...
+          >>> assert existspath(fn)
+          >>> assert open(fn).read() == 'foo\\n'
+          >>> os.remove(fn)
+  '''
+  if dir is None:
+    dir = dirname(filename)
+  fprefix, fsuffix = splitext(basename(filename))
+  if prefix is None:
+    prefix = '.' + fprefix + '-'
+  if suffix is None:
+    suffix = fsuffix
+  if rename_func is None:
+    if exists_ok or placeholder:
+      rename_func = os.rename
+    else:
+      rename_func = rename_excl
+  if not exists_ok and existspath(filename):
+    raise FileExistsError(errno.EEXIST, os.strerror(errno.EEXIST), filename)
+  with NamedTemporaryFile(
+      dir=dir,
+      prefix=prefix,
+      suffix=suffix,
+      **tempfile_kw,
+  ) as T:
+    if placeholder:
+      # create a placeholder file
+      with open(filename, 'ab' if exists_ok else 'xb'):
+        pass
+    yield T
+    # if the caller removed the temp file
+    # do not create/replace the target
+    if existspath(T.name):
+      mtime = pfx_call(os.stat, T.name).st_mtime
+      try:
+        pfx_call(shutil.copystat, filename, T.name)
+      except FileNotFoundError:
+        pass
+      except OSError as e:
+        warning(
+            "defaut modes not copied from from placeholder %r: %s", filename, e
+        )
+      else:
+        # we make the attribute like the original, now bump the mtime
+        try:
+          atime = pfx_call(os.stat, filename).st_atime
+        except FileNotFoundError:
+          atime = mtime
+        pfx_call(os.utime, T.name, (atime, mtime))
+      # just in case something made the file
+      if not placeholder and not exists_ok and existspath(filename):
+        raise FileExistsError(
+            errno.EEXIST, os.strerror(errno.EEXIST), filename
+        )
+      pfx_call(rename_func, T.name, filename)
+    # recreate the temp file so that it can be cleaned up by NamedTemporaryFile
+    with pfx_call(open, T.name, 'xb'):
+      pass
+
+def atomic_copy2(srcpath, dstpath, *, follow_symlinks=True, **af_kw):
+  ''' Call `shutil.copy2` to copy `srcpath` to `dstpath` via a
+      temporary file using `atomic_filename`.
+      This differs from `shutil.copy2` in 2 ways:
+      - it is an error if `dstpath` already exists unless you supply
+        `exists_ok=True`
+      - the new copy appears atomicly when the copy is complete
+        instead of be visible partially complete during the copy
+      The `follow_symlinks=True` parameter is passed to `shutil.copy2`.
+      Other keyword parameters are passed to `atomic_filename`.
+  '''
+  with atomic_filename(dstpath, **af_kw) as af:
+    return shutil.copy2(srcpath, af.name, follow_symlinks=follow_symlinks)
+
+class RWFileBlockCache(object):
+  ''' A scratch file for storing data.
+  '''
+
+  def __init__(self, pathname=None, dirpath=None, suffix=None, lock=None):
+    ''' Initialise the file.
+
+        Parameters:
+        * `pathname`: path of file. If None, create a new file with
+          tempfile.mkstemp using dir=`dirpath` and unlink that file once
+          opened.
+        * `dirpath`: location for the file if made by mkstemp as above.
+        * `lock`: an object to use as a mutex, allowing sharing with
+          some outer system. A Lock will be allocated if omitted.
+    '''
+    opathname = pathname
+    if pathname is None:
+      tmpfd, pathname = mkstemp(dir=dirpath, suffix=suffix)
+    self.fd = os.open(pathname, os.O_RDWR | os.O_APPEND)
+    if opathname is None:
+      os.remove(pathname)
+      os.close(tmpfd)
+      self.pathname = None
+    else:
+      self.pathname = pathname
+    if lock is None:
+      lock = Lock()
+    self._lock = lock
+
+  def __str__(self):
+    return "%s(pathname=%s)" % (type(self).__name__, self.pathname)
+
+  def close(self):
+    ''' Close the file descriptors.
+    '''
+    with Pfx("%s.close", self):
+      fd = self.fd
+      if fd is None:
+        warning("fd already closed")
+      else:
+        os.close(fd)
+        self.fd = None
+
+  @property
+  def closed(self):
+    ''' Test whether the file descriptor has been closed.
+    '''
+    return self.fd is None
+
+  def put(self, data):
+    ''' Store `data`, return offset.
+    '''
+    fd = self.fd
+    with self._lock:
+      offset = os.lseek(fd, 0, 1)
+      if len(data) == 0:
+        length = 0
+      else:
+        length = os.write(fd, data)
+    assert length == len(data)
+    return offset
+
+  def get(self, offset, length):
+    ''' Get data from `offset` of length `length`.
+    '''
+    assert length > 0
+    fd = self.fd
+    data = os.pread(fd, length, offset)
+    assert len(data) == length
+    return data
+
+@contextmanager
+def gzifopen(path, mode='r', *a, **kw):
+  ''' Context manager to open a file which may be a plain file or a gzipped file.
+
+      If `path` ends with `'.gz'` then the filesystem paths attempted
+      are `path` and `path` without the extension, otherwise the
+      filesystem paths attempted are `path+'.gz'` and `path`.  In
+      this way a path ending in `'.gz'` indicates a preference for
+      a gzipped file otherwise an uncompressed file.
+
+      However, if exactly one of the paths exists already then only
+      that path will be used.
+
+      Note that the single character modes `'r'`, `'a'`, `'w'` and `'x'`
+      are text mode for both uncompressed and gzipped opens,
+      like the builtin `open` and *unlike* `gzip.open`.
+      This is to ensure equivalent behaviour.
+  '''
+  compresslevel = kw.pop('compresslevel', 9)
+  path0 = path
+  path, ext = splitext(path)
+  if ext == '.gz':
+    # gzip preferred
+    gzpath = path0
+    path1, path2 = gzpath, path
+  else:
+    # unzipped has precedence
+    gzpath = path0 + '.gz'
+    path1, path2 = path0, gzpath
+  # if exactly one of the files exists, try only that file
+  if existspath(path1) and not existspath(path2):
+    paths = (path1,)
+  elif existspath(path2) and not existspath(path1):
+    paths = (path2,)
+  else:
+    paths = (path1, path2)
+  for openpath in paths:
+    try:
+      with (gzip.open(openpath,
+                      (mode + 't' if mode in ('r', 'a', 'w', 'x') else mode), *
+                      a, compresslevel=compresslevel, **kw) if
+            openpath.endswith('.gz') else open(openpath, mode, *a, **kw)) as f:
+        yield f
+    except FileNotFoundError:
+      # last path to try
+      if openpath == paths[-1]:
+        raise
+      # not present, try the other file
+      continue
+    # open succeeded, we're done
+    return
+  raise RuntimeError("NOTREACHED")
+
+if __name__ == '__main__':
+  import cs.fileutils_tests
+  cs.fileutils_tests.selftest(sys.argv)