tslite-og 2.1.0__py3-none-any.whl

tslite.py

@@ -0,0 +1,1953 @@
+#!/usr/local/bin/python
+r"""
+  _       _ _ _
+ | |_ ___| (_) |_ ___
+ | __/ __| | | __/ _ \
+ | |_\__ \ | | ||  __/
+  \__|___/_|_|\__\___| v2.1.0
+  a lightweight timeseries library
+
+tslite: lightweight, portable time-series processing since 2009.
+
+Author: Gunnar Leffler.
+Copyright 2009-2026 Gunnar Leffler.
+SPDX-License-Identifier: Apache-2.0
+
+Distributed on PyPI as tslite-og.
+"""
+
+__version__ = "2.1.0"
+
+import datetime
+import json
+import math
+import os
+import re
+import struct
+import dateutil.parser as dateparser
+from dateutil.relativedelta import relativedelta
+from functools import wraps
+from bisect import bisect_left, bisect_right
+from statistics import pvariance
+from zoneinfo import ZoneInfo
+from decimal import Decimal, ROUND_DOWN, localcontext
+
+# NumPy is optional; the core library does not require it.
+try:
+  import numpy as np
+except ImportError:
+  _NUMPY_AVAILABLE = False
+else:
+  _NUMPY_AVAILABLE = True
+
+try:
+  import sqlite3
+except ImportError:
+  _SQLITE3_AVAILABLE = False
+else:
+  _SQLITE3_AVAILABLE = True
+
+
+def requires_numpy(f):
+  """Require NumPy while keeping the core library usable without it."""
+  @wraps(f)
+  def wrapper(*args, **kwargs):
+    if _NUMPY_AVAILABLE:
+      return f(*args, **kwargs)
+    else:
+      raise Warning("Numpy not available.  Cannot call %s" % f.__name__)
+
+  return wrapper
+
+
+def requires_SQLITE3(f):
+  @wraps(f)
+  def wrapper(*args, **kwargs):
+    if _SQLITE3_AVAILABLE:
+      return f(*args, **kwargs)
+    else:
+      raise Warning("SQLITE3 not available. Cannot call %s" % f.__name__)
+
+  return wrapper
+
+
+# A numeric reference point, not a timezone interpretation of observations.
+_EPOCH = datetime.datetime(1970, 1, 1)
+
+
+def _storage_time(timestamp, tz):
+  """Interpret a naive clock reading in an explicit zone, returning naive UTC.
+
+  Reject nonexistent and ambiguous local readings rather than guessing at DST.
+  """
+  if tz is None:
+    return timestamp
+  if timestamp.tzinfo is not None:
+    raise TypeError("Storage expects naive timestamps")
+  zone = ZoneInfo(tz)
+  candidates = set()
+  for fold in (0, 1):
+    instant = timestamp.replace(tzinfo=zone, fold=fold).astimezone(datetime.timezone.utc)
+    if instant.astimezone(zone).replace(tzinfo=None) == timestamp:
+      candidates.add(instant.replace(tzinfo=None))
+  if len(candidates) != 1:
+    raise ValueError(f"Ambiguous or nonexistent local timestamp: {timestamp} in {tz}")
+  return candidates.pop()
+
+
+def _loaded_time(timestamp, tz):
+  """Return a naive clock reading in the requested zone (or unchanged)."""
+  if tz is None:
+    return timestamp
+  return timestamp.replace(tzinfo=datetime.timezone.utc).astimezone(
+      ZoneInfo(tz)).replace(tzinfo=None, fold=0)
+
+
+def _epoch_microseconds(timestamp):
+  """Encode a naive timestamp exactly, without consulting the machine timezone."""
+  elapsed = timestamp - _EPOCH
+  return (elapsed.days * 86400 + elapsed.seconds) * 1000000 + elapsed.microseconds
+
+
+def _from_epoch_microseconds(timestamp):
+  """Decode integer microseconds to the original naive clock reading."""
+  return _EPOCH + datetime.timedelta(microseconds=timestamp)
+
+
+def _epoch_seconds(timestamp):
+  """Encode a naive timestamp as fractional seconds for legacy binary/math use."""
+  return (timestamp - _EPOCH).total_seconds()
+
+
+def _finite_values(records):
+  """Extract usable observations without treating missing values as zero."""
+  return [value for _, value in records if value is not None and math.isfinite(value)]
+
+
+def _finite_mean(records):
+  values = _finite_values(records)
+  return math.fsum(value / len(values) for value in values) if values else math.nan
+
+
+class timeseries:
+
+  def __init__(self, data=None):
+    """Construct sorted timestamp/value observations, preserving missing values.
+
+    Accepts iterable records with datetime or parseable string timestamps.
+    Numeric values are converted to float; explicit None values retain their
+    timestamps. Records shorter than two fields are ignored. Duplicate timestamps
+    use the last supplied value, including None. Input records are not aliased.
+    """
+    self.status = "OK"
+    #Data is a nested list with the following structure [datetime, float value,``]
+    self.data = []
+    self.decimals = 3
+    if data is not None:
+      for row in data:
+        if len(row) > 1:
+          self.safeinsert(row[0], row[1])
+
+  @classmethod
+  def createRegular(cls, start, end, interval, value=0):
+    """Create a constant-valued series on a regular time grid.
+
+    Args:
+        start (datetime.datetime): First timestamp.
+        end (datetime.datetime): Last allowed timestamp. Included if on the grid.
+        interval (str | datetime.timedelta): Positive duration, such as '1h',
+            '1.5h', or '1d6h'. As in TD(), 'Y' means 365 days.
+        value (float): Constant numeric value, converted to float. Defaults to zero.
+
+    Returns:
+        result (timeseries): A new series. Equal bounds produce one observation;
+            an off-grid end never produces an observation beyond the bound.
+
+    Raises:
+        TypeError: If bounds are not datetimes, interval has an unsupported
+            type, or naive and aware bounds are mixed.
+        ValueError: If end precedes start or interval is malformed/nonpositive.
+    """
+    if not isinstance(start, datetime.datetime) or not isinstance(end, datetime.datetime):
+      raise TypeError("start and end must be datetimes")
+    if end < start:
+      raise ValueError("end must not precede start")
+
+    result = cls()
+    interval = result.TD(interval, positive=True)
+
+    value = float(value)
+    timestamp = start
+    while timestamp <= end:
+      result.insert(timestamp, value)
+      # Avoid stepping beyond end, including overflow near datetime.max.
+      if end - timestamp < interval:
+        break
+      timestamp += interval
+    return result
+
+  #========================================================================
+  # IO and data manipulation methods
+  #========================================================================
+
+  def __str__(self):
+    '''Equivalent to toString() in other languages
+     returns a tab delineated timeseries'''
+    output = ""
+    template = "%s\t%.3f\n"
+    for line in self.data:
+      try:
+        output += template % (line[0].strftime("%d-%b-%Y %H%M"), line[1])
+      except Exception:
+        output += "%s\t\t\n" % line[0].strftime("%d-%b-%Y %H%M")
+    return output
+
+  def __getitem__(self, idx):
+    """Return a record or list slice using normal Python indexing.
+
+    Negative indices are supported; out-of-range integer indices raise
+    IndexError. Records are the underlying mutable [timestamp, value] lists.
+    """
+    return self.data[idx]
+
+  def __iter__(self):
+    """Return an iterator over observations in timestamp order.
+
+    Each item is the stored mutable [timestamp, value] record, not a copy.
+    """
+    return iter(self.data)
+
+  def __len__(self):
+    return (len(self.data))
+
+  def __eq__(self, other):
+    """Compare timestamps and values exactly, in observation order.
+
+    Use same_data() to ignore timestamps, or is_close() for explicit
+    tolerances. NaN values do not compare equal, even within the same series.
+    """
+    if not isinstance(other, timeseries):
+      return NotImplemented
+    if len(self.data) != len(other.data):
+      return False
+    return all(a[0] == b[0] and a[1] == b[1]
+               for a, b in zip(self.data, other.data))
+
+  def same_data(self, other, precision=6):
+    """Compare value sequences, ignoring timestamps.
+
+    Args:
+        other (timeseries): Series with the same number of observations.
+        precision (int): Decimal places used to format values before comparison,
+            preserving the legacy equality rule (not an absolute tolerance).
+
+    Returns:
+        result (bool): Whether the value sequences match in observation order.
+    """
+    if not isinstance(precision, int) or isinstance(precision, bool) or precision < 0:
+      raise ValueError("precision must be a nonnegative integer")
+    if not isinstance(other, timeseries):
+      return False
+    if len(self) != len(other):
+      return False
+    fmt = f'.{precision}f'
+    return all(format(a[1], fmt) == format(b[1], fmt)
+               for a, b in zip(self.data, other.data))
+
+  def is_close(self, other, *, time_tolerance=datetime.timedelta(0),
+               value_tolerance=1e-6, relative_tolerance=0.0):
+    """Compare aligned observations with explicit tolerances.
+
+    Observations are paired by position; lengths must match. This does not
+    interpolate, shift, or search for matching timestamps. Inputs are unchanged.
+
+    Args:
+        other (timeseries): Series to compare.
+        time_tolerance (str | float | datetime.timedelta): Inclusive timestamp
+            tolerance. Accepts a
+            timedelta, numeric seconds, or a duration string such as '1s' or
+            '2m30s'. Must be finite and nonnegative.
+        value_tolerance (float): Nonnegative finite absolute value tolerance.
+        relative_tolerance (float): Nonnegative finite relative value tolerance.
+            Values match when their difference is no greater than the larger
+            of value_tolerance and relative_tolerance times the larger magnitude.
+
+    Returns:
+        result (bool): Whether all paired timestamps and values are close. Two empty
+            series match. NaN values do not match; equal infinities do.
+
+    Mixing naive and aware timestamps raises TypeError rather than guessing
+    a timezone.
+
+    Raises:
+        ValueError: If a tolerance is invalid.
+    """
+    if isinstance(time_tolerance, str):
+      text = time_tolerance.strip()
+      if not re.fullmatch(r'(?:\d+(?:\.\d+)?[wdhms])+', text):
+        raise ValueError("invalid time tolerance")
+      units = {'w': 604800, 'd': 86400, 'h': 3600, 'm': 60, 's': 1}
+      seconds = sum(float(number) * units[unit] for number, unit in
+                    re.findall(r'(\d+(?:\.\d+)?)([wdhms])', text))
+    elif isinstance(time_tolerance, datetime.timedelta):
+      seconds = time_tolerance.total_seconds()
+    else:
+      try:
+        seconds = float(time_tolerance)
+      except (TypeError, ValueError):
+        raise ValueError("invalid time tolerance") from None
+    try:
+      absolute = float(value_tolerance)
+      relative = float(relative_tolerance)
+    except (TypeError, ValueError):
+      raise ValueError("invalid value tolerance") from None
+    if any(not math.isfinite(t) or t < 0 for t in (seconds, absolute, relative)):
+      raise ValueError("tolerances must be finite and nonnegative")
+    if not isinstance(other, timeseries) or len(self) != len(other):
+      return False
+    for a, b in zip(self.data, other.data):
+      if abs((a[0] - b[0]).total_seconds()) > seconds:
+        return False
+      if not math.isclose(a[1], b[1], rel_tol=relative, abs_tol=absolute):
+        return False
+    return True
+
+  def toDict(self):
+    '''Turns self.data into a dictionary for efficiency purposes'''
+    output = {}
+    for i in range(len(self.data)):
+      output[self.data[i][0]] = i
+    return output
+
+  def timestamps(self):
+    output = []
+    for i in range(len(self.data)):
+      output.append(self.data[i][0])
+    return output
+
+  def values(self):
+    output = []
+    for i in range(len(self.data)):
+      output.append(self.data[i][1])
+    return output
+
+  def toPlot(self):
+    '''Format timeseries for plotting by returning:
+       x: Timestamps
+       y: Values
+       Matplotlib Example: plt.plot(*timeseries.toPlot())
+    '''
+    return self.timestamps(), list(self.values())
+
+  def saveTSV(self, path):
+    '''Outputs the timeseries to a tab separated file'''
+    f = open(path, "w")
+    f.write(str(self))
+    f.close()
+
+  def loadTSV(self, path):
+    '''Reads a timeseries from a tsv file. Hash (#) can be used as comments
+       Format <Datetime>\t<value>
+       This method mutates the object, and also returns a pointer to self.
+    '''
+    lines = (line.rstrip("\n") for line in open(path, "r"))
+    return self.fromTSV(lines)
+
+  def fromTSV(self, lines):
+    '''reads a timeseries from a TSV string 
+       This method mutates the object, and also returns a pointer to self.
+    '''
+    count = 0
+    for s in lines:
+      count += 1
+      s = re.sub(r'#.*', '', s)  # Strip comments
+      if re.match(r'\S', s):  # Ignore blank lines
+        tokens = s.split("\t")
+        try:
+          if len(tokens) > 1:
+            self.safeinsert(tokens[0], float(tokens[1]))
+        except Exception:
+          self.status = "Error Parsing line %u" % (count)
+    return self
+
+  def saveBinary(self, path, tz=None):
+    '''Outputs the timeseries to a binary file'''
+    output = self.toBinary(tz=tz)
+    with open(path, "wb") as f:
+      f.write(output)
+
+  def toBinary(self, tz=None):
+    """Return a bytearray of native double epoch-second/value records.
+
+    Fractional seconds are preserved to the precision of a 64-bit float.
+    Contemporary dates round-trip at microsecond resolution; precision decreases
+    farther from the epoch. Whole-second records keep the existing layout and
+    bytes when previously written in GMT. Naive clock readings are preserved
+    across machines; input observations are unchanged. Aware inputs raise TypeError.
+    With tz, interpret naive inputs in that IANA timezone and store UTC instants.
+    Ambiguous or nonexistent local readings raise ValueError. None values are
+    encoded as NaN; the legacy double format cannot distinguish the two.
+    """
+    output = bytearray()
+    for timestamp, value in self.data:
+      seconds = _epoch_seconds(_storage_time(timestamp, tz))
+      output.extend(struct.pack("dd", seconds, math.nan if value is None else value))
+    return output
+
+  def loadBinary(self, path, tz=None):
+    '''Reads the timeseries from a binary file and inserts values into self'''
+    buf = bytearray(os.path.getsize(path))
+    with open(path, "rb") as f:
+      f.readinto(buf)
+      self.fromBinary(buf, tz=tz)
+    f.close()
+    return self
+
+  def fromBinary(self, buf, tz=None):
+    """Insert native double epoch-second/value records and return self.
+
+    Existing whole-second files and fractional-second records use the same
+    layout. Seconds are decoded by naive arithmetic, not machine-local time.
+    Float timestamps cannot preserve every microsecond throughout datetime's
+    full date range. With tz, decode UTC instants to naive readings in that
+    IANA timezone. Use the original writer's zone for old local-encoded files.
+    Repeated local clock labels use the usual last-insert-wins behavior.
+    Missing values encoded as NaN load as NaN, not None. Complete records are
+    retained if an incomplete trailing record exists; ignored trailing bytes
+    produce a warning in self.status. Otherwise successful reading sets status
+    to OK. Invalid complete records still raise their decoding error.
+    """
+    size = struct.calcsize("dd")
+    trailing = len(buf) % size
+    complete = len(buf) - trailing
+    for i in range(0, complete, size):
+      seconds, value = struct.unpack("dd", buf[i:i + size])
+      self.insert(_loaded_time(_EPOCH + datetime.timedelta(seconds=seconds), tz), value)
+    self.status = (f"Warning: binary input has an incomplete trailing record; "
+                   f"ignored {trailing} byte(s), read {complete // size} complete record(s)"
+                   if trailing else "OK")
+    return self
+
+  @requires_SQLITE3
+  def SQLITE3connect(self, dbPath):
+    '''
+    Get a SQLITE 3 database connection
+    dbPath - path to the database file  
+    '''
+    #initialize with Default Configuration
+    self.status = "OK"
+    #Database Cursors
+    dbconn = None
+    try:
+      dbconn = sqlite3.connect(dbPath)
+      if not dbconn:
+        self.status = f"\nCould not connect to {dbPath}\n"
+    except Exception as e:
+      self.status = f"\nCould not connect to {dbPath}\n {str (e)}"
+    return dbconn
+
+  @requires_SQLITE3
+  def SQLITE3disconnect(self, dbconn):
+    '''Disconnect from a SQLITE3 database connection '''
+    dbconn.close()
+
+  @requires_SQLITE3
+  def loadSQLITE3(self, conn, tsid, start_time=None, end_time=None, tz=None):
+    """Load a new series from a SQLite table storing integer epoch microseconds.
+
+    Either optional datetime bound may be supplied; both endpoints are inclusive.
+    Rows are read in timestamp order. Naive timestamp values and microsecond
+    bounds are preserved independently of the machine timezone; metadata belongs
+    to the application. With tz, decode UTC instants into naive readings in
+    that IANA timezone; bounds are interpreted in the same zone. Repeated local
+    labels use last-insert-wins. Errors are recorded on this object's status.
+    """
+    ts = timeseries()
+    cur = conn.cursor()
+    try:
+      # Quote a single literal identifier, not SQL supplied by the caller.
+      table = '"' + tsid.replace('"', '""') + '"'
+      sql = "SELECT * FROM " + table
+      conditions = []
+      parameters = []
+      for bound, comparison in [(start_time, '>='), (end_time, '<=')]:
+        if bound is not None:
+          conditions.append("timestamp " + comparison + " ?")
+          parameters.append(_epoch_microseconds(_storage_time(bound, tz)))
+      if conditions:
+        sql += " WHERE " + " AND ".join(conditions)
+      sql += " ORDER BY timestamp"
+      cur.execute(sql, parameters)
+      for row in cur:
+        ts.insert(_loaded_time(_from_epoch_microseconds(row[0]), tz), row[1])
+    except Exception as error:
+      self.status = f"Could not read {tsid}: {error}"
+    finally:
+      cur.close()
+    return ts
+
+
+  @requires_SQLITE3
+  def saveSQLITE3(self, conn, tsid, replace_table=False, tz=None):
+    """Store observations atomically in a SQLite table; return None.
+
+    Naive timestamps use integer microseconds from a naive epoch, preserving
+    datetime precision without applying a timezone. Aware timestamps are not
+    implicitly converted; failures are reported in status. With tz, interpret
+    naive readings in that IANA timezone and store UTC instants. Ambiguous or
+    nonexistent local readings fail rather than guessing. New timestamps are inserted; existing
+    values are updated only when different (using SQL's null-safe IS NOT).
+    Unchanged rows are not updated or deleted. With replace_table=True, the
+    table is dropped and recreated, as before.
+    Table names are literal identifiers and retain legacy uppercasing.
+
+    A savepoint isolates this write. Without an existing transaction, success
+    commits the save; inside one, the caller still owns commit or rollback.
+    Failures undo this save only and are recorded in self.status. Interrupts
+    also undo the save, then propagate. Input observations are unchanged.
+    """
+    cur = None
+    try:
+      table = '"' + tsid.upper().replace('"', '""') + '"'
+      cur = conn.cursor()
+      cur.execute("SAVEPOINT tslite_save")
+      try:
+        if replace_table:
+          cur.execute("DROP TABLE IF EXISTS " + table)
+        cur.execute("CREATE TABLE IF NOT EXISTS " + table +
+                    " (timestamp INTEGER PRIMARY KEY, val REAL)")
+        rows = ((_epoch_microseconds(_storage_time(timestamp, tz)), value)
+                for timestamp, value in self.data)
+        sql = ("INSERT INTO " + table + " (timestamp, val) VALUES (?, ?) "
+               "ON CONFLICT(timestamp) DO UPDATE SET val = excluded.val "
+               "WHERE val IS NOT excluded.val")
+        cur.executemany(sql, rows)
+        cur.execute("RELEASE SAVEPOINT tslite_save")
+      except BaseException:
+        # Include interrupts in cleanup, but let them propagate to the caller.
+        cur.execute("ROLLBACK TO SAVEPOINT tslite_save")
+        cur.execute("RELEASE SAVEPOINT tslite_save")
+        raise
+      self.status = "OK"
+    except Exception as error:
+      self.status = f"Could not store {tsid}: {error}"
+    finally:
+      if cur is not None:
+        cur.close()
+
+  def getStatus(self):
+    '''exceptions get dropped into self.status
+       This method gets status message of object and resets self.status to "OK" '''
+    s = self.status
+    self.status = "OK"
+    return s
+
+  def findValue(self, timestamp):
+    '''  returns a value at a given timestamp
+    returns None type if not found'''
+    idx = self.findIndex(timestamp)
+    if idx != -1:
+      return self.data[idx][1]
+    else:
+      return None
+
+  def findIndex(self, key):
+    '''  returns the index of a given timestamp
+    returns -1 if not found'''
+    imin = 0
+    imax = len(self.data) - 1
+    while (imax >= imin):
+      imid = imin + ((imax - imin) // 2)
+      if (self.data[imid][0] == key):
+        return imid
+      elif (self.data[imid][0] < key):
+        imin = imid + 1  #change min index to search upper subarray
+      else:
+        imax = imid - 1
+        #change max index to search lower subarray
+    return -1  # Key not found
+
+  def findClosestIndex(self, key):
+    """Return the nearest timestamp's index, or -1 for an empty series.
+
+    Exact matches return their index. Outside the series range, return the
+    nearest endpoint. Equal-distance ties choose the earlier observation.
+    Distances retain full timedelta precision; observations are unchanged.
+    """
+    if not self.data:
+      return -1
+    pos = bisect_left(self.data, key, key=lambda row: row[0])
+    if pos == 0:
+      return 0
+    if pos == len(self.data):
+      return pos - 1
+    if key - self.data[pos - 1][0] <= self.data[pos][0] - key:
+      return pos - 1
+    return pos
+
+  def safeinsert(self, datestamp, value):
+    '''takes raw input and attempts to make it work'''
+    if isinstance(datestamp, str):
+      datestamp = dateparser.parse(datestamp, fuzzy=True)
+    self.insert(datestamp, None if value is None else float(value))
+
+  def insert(self, datestamp, value):
+    """Insert an observation in timestamp order, replacing an exact match.
+
+    Newer timestamps use a fast append path. Other timestamps use binary search
+    for their insertion position; middle insertion still shifts later records.
+    """
+    if not self.data or datestamp > self.data[-1][0]:
+      self.data.append([datestamp, value])
+      return
+    i = bisect_left(self.data, datestamp, key=lambda row: row[0])
+    if i < len(self.data) and datestamp == self.data[i][0]:
+      self.data[i] = [datestamp, value]
+    else:
+      self.data.insert(i, [datestamp, value])
+
+  def truncate(self, precision):
+    """Return a new timeseries with values truncated to precision decimal places.
+
+    Digits are discarded toward zero, not rounded, including for negative
+    values. Precision must be a nonnegative integer. Timestamps and nonfinite
+    values are preserved; the input series is unchanged.
+    """
+    if not isinstance(precision, int) or isinstance(precision, bool) or precision < 0:
+      raise ValueError("precision must be a nonnegative integer")
+    output = timeseries()
+    for timestamp, value in self.data:
+      if math.isfinite(value):
+        # Use decimal text to avoid float multiplication turning 1.15 into
+        # 114.999... at two places. ROUND_DOWN means toward zero in Decimal.
+        decimal = Decimal(str(value))
+        with localcontext() as context:
+          context.prec = max(28, len(decimal.as_tuple().digits),
+                             decimal.adjusted() + precision + 1)
+          unit = Decimal((0, (1,), -precision))
+          value = float(decimal.quantize(unit, rounding=ROUND_DOWN))
+      output.insert(timestamp, value)
+    return output
+
+  def round(self, precision):
+    '''Rounds values in timeseries to a given number of decimal places
+    '''
+    output = timeseries()
+    for slice in self.data:
+      output.insert(slice[0], round(slice[1], precision))
+    return output
+
+  def merge(self, other):
+    """Return the sorted union of observations, preferring usable values.
+
+    At an exact timestamp match, finite data wins over None/NaN/infinity on
+    either side. If both values are usable or both unusable, other wins. Zero
+    is usable. Unmatched observations are retained, including missing values.
+    Inputs are unchanged and output records independent. This conflict policy
+    applies only to merge(); insert() remains unconditional replacement.
+    """
+    output = timeseries()
+    left = right = 0
+    while left < len(self.data) and right < len(other.data):
+      a = self.data[left]
+      b = other.data[right]
+      if a[0] < b[0]:
+        selected = a
+        left += 1
+      elif b[0] < a[0]:
+        selected = b
+        right += 1
+      else:
+        a_usable = a[1] is not None and math.isfinite(a[1])
+        b_usable = b[1] is not None and math.isfinite(b[1])
+        selected = a if a_usable and not b_usable else b
+        left += 1
+        right += 1
+      output.data.append(selected[:])
+    output.data.extend(self.data[i][:] for i in range(left, len(self.data)))
+    output.data.extend(other.data[i][:] for i in range(right, len(other.data)))
+    return output
+
+  def diff(self, other, precision=6):
+    """Return incoming observations that are new or differ at decimal precision.
+
+    Values are compared using fixed-point formatting (rounding, not truncation)
+    at precision decimal places; stored values are unchanged. Only other's
+    records are returned: new timestamps and changed matches. Left-only records
+    are omitted. None matches None, NaN matches NaN, and infinities match only
+    with the same sign. Missing versus numeric values differ. Output records
+    are independent; inputs are unchanged.
+    """
+    if isinstance(precision, bool) or not isinstance(precision, int) or precision < 0:
+      raise ValueError("precision must be a nonnegative integer")
+    output = timeseries()
+    for slice in self.data:
+      i = other.findIndex(slice[0])
+      if i == -1:
+        continue
+      oslice = other.data[i]
+      left, right = slice[1], oslice[1]
+      if left is None or right is None:
+        different = (left is None) != (right is None)
+      else:
+        different = format(left, f'.{precision}f') != format(right, f'.{precision}f')
+      if different:
+        output.insert(oslice[0], oslice[1])
+    for slice in other.data:
+      i = self.findIndex(slice[0])
+      if i == -1:
+        output.insert(slice[0], slice[1])
+    return output
+
+  def toHTML(self, css="", thead=""):
+    '''like __str__ only it outputs a HTML table'''
+    output = "<table " + css + ">"
+    output += thead
+    for line in self.data:
+      try:
+        output += "<tr><td>%s</td><td>&nbsp;&nbsp;%.2f</td></tr>" % (
+            line[0].strftime("%d-%b-%Y %H%M"), line[1])
+      except Exception:
+        output += "<tr><td>%s</td><td> </td></tr>" % line[0].strftime(
+            "%d-%b-%Y %H%M")
+    return output + "</table>"
+
+  def toJS(self, var, timefmt=None, decimals=None, strict=False):
+    """Return a JavaScript variable declaration wrapping toJSON().
+
+    The caller supplies a valid JavaScript identifier as var. Formatting and
+    nonfinite-value behavior are the same as toJSON().
+    """
+    return "var %s = %s;\n" % (var, self.toJSON(
+        timefmt=timefmt, decimals=decimals, strict=strict))
+
+  def toJSON(self, timefmt=None, decimals=None, strict=False):
+    """Return JSON timestamp/value pairs, preserving precision by default.
+
+    Timestamps use ISO format, including subseconds when present. Supply a
+    strftime timefmt to override this. Values retain full float precision unless
+    decimals is a nonnegative integer, in which case round() controls display
+    rounding. Trailing zeroes are not guaranteed for JSON numbers. self.decimals
+    does not affect serialization. No timezone conversion is performed.
+
+    Missing (None), NaN, and infinite values export as null, preserving their
+    timestamps for plotting gaps. Pass strict=True to reject NaN/infinity with
+    ValueError instead; None remains valid null. fromJSON() preserves null-valued
+    records as None. NaN/infinity distinctions are lost when exported as null.
+    Invalid formatting arguments or malformed records still raise errors.
+    Input observations are unchanged. Returns a string.
+    """
+    if decimals is not None and (isinstance(decimals, bool) or
+                                not isinstance(decimals, int) or decimals < 0):
+      raise ValueError("decimals must be a nonnegative integer or None")
+    output = []
+    for timestamp, value in self.data:
+      label = timestamp.isoformat() if timefmt is None else timestamp.strftime(timefmt)
+      if value is not None and not math.isfinite(value):
+        if strict:
+          raise ValueError(f"Nonfinite value at {label}: {value}")
+        value = None
+      if decimals is not None and value is not None:
+        value = round(value, decimals)
+      output.append([label, value])
+    return json.dumps(output, allow_nan=False)
+
+  def fromJSON(self,s):
+    ts = timeseries()
+    try:
+      j = json.loads(s)
+      ts = timeseries(j)
+    except Exception as e:
+      self.status = str(e)
+    return ts
+
+  def minDate(self, timefmt="%m/%d/%Y %k:%M:%S"):
+    try:
+      return self.data[0][0].strftime(timefmt)
+    except Exception:
+      return ""
+
+  #========================================================================
+  # computational methods
+  #========================================================================
+
+  def interpolateValue(self, x0, y0, x1, y1, x):
+    '''interpolate values'''
+    m = (y1 - y0) / (x1 - x0)
+    output = y0 + (x - x0) * m
+    return output
+
+  def interpolate(self, interval, anchor=None):
+    """Return linear interpolation at regular time steps.
+
+    Interval must be positive. Output stops at the last observation, including
+    it only if it coincides with an output step; no extrapolation is performed.
+    Exact timestamp matches retain their source values without recalculation.
+    Without anchor, steps start at the first observation; singleton or short-span
+    input returns a copy of that observation. An optional datetime anchor sets
+    alignment instead: output timestamps are anchor + n * interval for integer
+    n, restricted to the observed range. The anchor may precede or follow the
+    data. If no aligned step falls within the range, output is empty.
+    Empty input returns empty. Records are independent of the input.
+    Processing errors are recorded in self.status and return completed steps.
+    """
+    interval = self.TD(interval, positive=True)
+    output = timeseries()
+    if not self.data:
+      return output
+    try:
+      first = self.data[0][0]
+      if anchor is not None:
+        # Ceiling division finds the first aligned step at or after the data,
+        # including when the anchor lies later than the observations.
+        offset_steps = -((anchor - first) // interval)
+        first = anchor + offset_steps * interval
+      steps = (self.data[-1][0] - first) // interval
+      pos = 0
+      for step in range(steps + 1):
+        timestamp = first + step * interval
+        while pos + 1 < len(self.data) and self.data[pos + 1][0] <= timestamp:
+          pos += 1
+        left_time, left_value = self.data[pos]
+        if timestamp == left_time:
+          value = left_value
+        else:
+          right_time, right_value = self.data[pos + 1]
+          fraction = (timestamp - left_time) / (right_time - left_time)
+          value = left_value + fraction * (right_value - left_value)
+        output.data.append([timestamp, value])
+    except Exception as e:
+      self.status = str(e)
+    return output
+
+  def average(self, interval):
+    """Return interval means labeled at the beginning of each half-open bucket.
+
+    Buckets start at the first observation. Finite observations have equal weight;
+    None/NaN/infinity are excluded and reported with a source-status warning.
+    All-invalid observed buckets yield NaN; empty buckets are omitted and final
+    partial buckets included. Success without skips resets status to OK. Interval
+    must be positive. Inputs are unchanged; output records are independent.
+    Processing errors set self.status and return completed buckets.
+    """
+    interval = self.TD(interval, positive=True)
+    output = timeseries()
+    skipped = 0
+    if not self.data:
+      self.status = 'OK'
+      return output
+    try:
+      start = self.data[0][0]
+      buckets = {}
+      for timestamp, value in self.data:
+        bucket = start + ((timestamp - start) // interval) * interval
+        values = buckets.setdefault(bucket, [])
+        if value is None or not math.isfinite(value):
+          skipped += 1
+        else:
+          values.append(value)
+      for timestamp, values in buckets.items():
+        mean = math.fsum(value / len(values) for value in values) if values else math.nan
+        output.data.append([timestamp, mean])
+      self.status = (f"Warning: average skipped {skipped} missing/nonfinite "
+                     "observation(s); averages may be incomplete" if skipped else "OK")
+    except Exception as e:
+      self.status = str(e)
+    return output
+
+  def globalAverage(self):
+    """Return the simple arithmetic mean as a [timestamp, value] record.
+
+    Finite observations have equal weight, regardless of timestamp spacing.
+    None, NaN, and infinity are excluded; no usable values yield NaN. The timestamp
+    is the final observation's, even if invalid. Empty input returns None. The
+    result is a new record; input data is unchanged.
+    """
+    if not self.data:
+      return None
+    return [self.data[-1][0], _finite_mean(self.data)]
+
+  def globalMax(self):
+    """Return the global maximum as a new [timestamp, value] record.
+
+    None/NaN/infinity are skipped with a warning in self.status. Retains the
+    actual maximum's timestamp; ties select the earliest observation. Returns
+    None if no usable observations exist. Otherwise status is OK when no values
+    were skipped. The input is unchanged and the returned record is independent.
+    """
+    return self._global_extreme(max, 'globalMax')
+
+  def globalMin(self):
+    """Return the global minimum as a new [timestamp, value] record.
+
+    None/NaN/infinity are skipped with a warning in self.status. Retains the
+    actual minimum's timestamp; ties select the earliest observation. Returns
+    None if no usable observations exist. Otherwise status is OK when no values
+    were skipped. The input is unchanged and the returned record is independent.
+    """
+    return self._global_extreme(min, 'globalMin')
+
+  def _global_extreme(self, select, method):
+    valid = [row for row in self.data
+             if row[1] is not None and math.isfinite(row[1])]
+    skipped = len(self.data) - len(valid)
+    self.status = (f"Warning: {method} skipped {skipped} missing/nonfinite "
+                   "observation(s); extremum may be incomplete" if skipped else "OK")
+    return select(valid, key=lambda row: row[1])[:] if valid else None
+
+  def _linear_fit(self):
+    """Return origin, mean elapsed seconds, mean value, slope, and correlation."""
+    valid = [(t, v) for t, v in self.data if v is not None and math.isfinite(v)]
+    if not valid:
+      origin = self.data[0][0] if self.data else _EPOCH
+      return origin, 0.0, 0.0, 0.0, 0.0
+    origin = valid[0][0]
+    x = [(t - origin).total_seconds() for t, _ in valid]
+    y = [v for _, v in valid]
+    n = len(valid)
+    mean_x = math.fsum(value / n for value in x)
+    mean_y = math.fsum(value / n for value in y)
+    dx = [value - mean_x for value in x]
+    dy = [value - mean_y for value in y]
+    # Normalize centered values to keep squared sums in a safe numeric range.
+    scale_x = max(abs(value) for value in dx)
+    scale_y = max(abs(value) for value in dy)
+    if scale_x == 0 or scale_y == 0:
+      return origin, mean_x, mean_y, 0.0, 0.0
+    dx = [value / scale_x for value in dx]
+    dy = [value / scale_y for value in dy]
+    xx = math.fsum(value * value for value in dx)
+    yy = math.fsum(value * value for value in dy)
+    xy = math.fsum(a * b for a, b in zip(dx, dy))
+    slope = (xy / xx) * scale_y / scale_x
+    correlation = (xy / math.sqrt(xx)) / math.sqrt(yy)
+    return origin, mean_x, mean_y, slope, max(-1.0, min(1.0, correlation))
+
+  def linreg(self):
+    """Return least-squares (slope, epoch intercept, correlation).
+
+    The line is y = slope * x + intercept, with x in naive seconds since 1970.
+    Internally, relative timestamps and centered sums avoid epoch cancellation.
+    None, NaN, and infinite values are excluded. No usable observations return
+    (0, 0, 0); one usable observation or constant values yield a horizontal line.
+    Undefined correlation is reported as 0 by convention. Inputs are unchanged.
+    """
+    origin, mean_x, mean_y, slope, correlation = self._linear_fit()
+    if slope == 0:
+      return 0.0, mean_y, correlation
+    intercept = mean_y - slope * (_epoch_seconds(origin) + mean_x)
+    return slope, intercept, correlation
+
+  def trendline(self):
+    """Return fitted values at every original timestamp, with independent records.
+
+    Fit excludes None/NaN/infinite values, but predictions include their timestamps.
+    Evaluate the centered relative-time fit directly, avoiding epoch cancellation.
+    Empty input returns empty; no usable values yield zeros. A single usable value
+    yields a horizontal line. Inputs are unchanged; correlation follows linreg().
+    """
+    output = timeseries()
+    origin, mean_x, mean_y, slope, _ = self._linear_fit()
+    for timestamp, _ in self.data:
+      value = mean_y if slope == 0 else mean_y + slope * (
+          (timestamp - origin).total_seconds() - mean_x)
+      output.data.append([timestamp, value])
+    return output
+
+  def variance(self):
+    """Return the population variance as a [timestamp, value] record.
+
+    Finite observations have equal weight; None/NaN/infinity are excluded. The
+    denominator is the usable count, not count minus one. No usable values yield
+    NaN; one usable value yields zero. The timestamp is the final observation's.
+    Empty input returns None; the input series is unchanged.
+    """
+    if not self.data:
+      return None
+    values = _finite_values(self.data)
+    return [self.data[-1][0], pvariance(values) if values else math.nan]
+
+  def stddev(self):
+    """Return the population standard deviation as a [timestamp, value] record.
+
+    The value is the square root of variance(), labeled with the final
+    observation's timestamp. None/NaN/infinity are excluded; no usable values
+    yield NaN. Empty input returns None; the input is unchanged.
+    """
+    s = self.variance()
+    if s != None:
+      s[1] = math.sqrt(s[1])
+    return s
+
+  def movingstddev(self, interval):
+    """Return a new timeseries of population standard deviations by window.
+
+    Inclusive windows advance by interval from the first timestamp while their start is
+    before the final timestamp. Nonempty windows are labeled with their last
+    observation; empty windows are skipped. If adjacent windows have the same
+    label, the later window replaces the earlier result, as with insert().
+    None/NaN/infinity are excluded from each calculation; all-invalid windows
+    yield NaN and retain their labels. A singleton series has no windows.
+    """
+    output = timeseries()
+    interval = self.TD(interval, positive=True)
+    if self.data:
+      pointer = self.data[0][0]
+      while pointer < self.data[-1][0]:
+        stddev = self.subSlice(pointer, pointer + interval).stddev()
+        if stddev is not None:
+          output.insert(stddev[0], stddev[1])
+        pointer += interval
+    return output
+
+  def subSlice(self, starttime, endtime):
+    """Return independent observations between inclusive datetime bounds.
+
+    Empty, reversed, or nonoverlapping ranges return an empty timeseries. No
+    observations are filled or interpolated, and the input is unchanged.
+    """
+    output = timeseries()
+    first = bisect_left(self.data, starttime, key=lambda row: row[0])
+    stop = bisect_right(self.data, endtime, key=lambda row: row[0])
+    output.data = [self.data[i][:] for i in range(first, stop)]
+    return output
+
+  def getFiscalYear(self, start_date):
+    """Return available observations in the year beginning at start_date.
+
+    Accepts a date string (for example, "Oct 1 2025"), date, or datetime.
+    Dates start at midnight; datetimes retain their time of day. The start is
+    included and its anniversary one calendar year later is excluded. A
+    February 29 start ends at February 28 of the following year. No complete
+    year of data is required; missing observations are not filled. Returns a
+    new timeseries with independent records, leaving the input unchanged.
+    """
+    if isinstance(start_date, str):
+      start_date = dateparser.parse(start_date,
+                                    default=datetime.datetime(2000, 1, 1))
+    elif isinstance(start_date, datetime.date) and not isinstance(
+        start_date, datetime.datetime):
+      start_date = datetime.datetime.combine(start_date, datetime.time())
+    if not isinstance(start_date, datetime.datetime):
+      raise TypeError("start_date must be a date string, date, or datetime")
+    end_date = start_date + relativedelta(years=1)
+    return self.subSlice(start_date, end_date - datetime.timedelta(microseconds=1))
+
+  def getWY(self, WY):
+    """Return available observations for the October–September year ending in WY.
+
+    Equivalent to getFiscalYear() starting October 1 of WY - 1. Includes all
+    of September 30 and returns independent records without filling gaps.
+    """
+    return self.getFiscalYear(datetime.datetime(year=WY - 1, month=10, day=1))
+
+  def averageWY(self):
+    """Return calendar-position means labeled in the latest available water year.
+
+    Output uses only timestamps present in the latest October–September year.
+    Each position matches month, day, hour, minute, second, and microsecond across
+    all available years, including the latest; observations have equal weight.
+    February 29 matches leap days only. Absent positions are not invented or
+    shifted. None/NaN/infinity are excluded; all-invalid positions yield NaN.
+    Skipped matching observations warn in self.status, otherwise status is OK.
+    Empty input returns empty; output records are independent, inputs unchanged.
+    """
+    output = timeseries()
+    if not self.data:
+      self.status = 'OK'
+      return output
+
+    def position(timestamp):
+      return (timestamp.month, timestamp.day, timestamp.hour, timestamp.minute,
+              timestamp.second, timestamp.microsecond)
+
+    latest = self.data[-1][0]
+    latest_year = latest.year + (latest.month >= 10)
+    targets = [timestamp for timestamp, _ in self.data
+               if timestamp.year + (timestamp.month >= 10) == latest_year]
+    groups = {position(timestamp): [] for timestamp in targets}
+    skipped = 0
+    for timestamp, value in self.data:
+      key = position(timestamp)
+      if key not in groups:
+        continue
+      if value is None or not math.isfinite(value):
+        skipped += 1
+      else:
+        groups[key].append(value)
+    for timestamp in targets:
+      values = groups[position(timestamp)]
+      mean = math.fsum(value / len(values) for value in values) if values else math.nan
+      output.data.append([timestamp, mean])
+    self.status = (f"Warning: averageWY skipped {skipped} missing/nonfinite "
+                   "observation(s); averages may be incomplete" if skipped else "OK")
+    return output
+
+  def runningTotal(self):
+    """Return a new timeseries of running totals (partial sums).
+
+    Each value sums usable observations up to and including that timestamp.
+    None/NaN/infinity are skipped, carrying the known subtotal (initially zero).
+    Skips warn in self.status that totals may be incomplete; otherwise status
+    is OK. Original timestamps are preserved; the input series is unchanged.
+    """
+    output = timeseries()
+    total = 0
+    skipped = 0
+    for timestamp, value in self.data:
+      if value is None or not math.isfinite(value):
+        skipped += 1
+      else:
+        total += value
+      output.data.append([timestamp, total])
+    self._total_status('runningTotal', skipped)
+    return output
+
+  def _total_status(self, method, skipped):
+    """Report deliberately incomplete totals, rather than a processing failure."""
+    self.status = (f"Warning: {method} skipped {skipped} missing/nonfinite "
+                   "observation(s); totals may be incomplete" if skipped else "OK")
+
+  def accumulate(self, interval, override_startTime=None):
+    """Return interval totals, labeled at the beginning of half-open buckets.
+
+    Buckets start at override_startTime or the first observation. Earlier data
+    is excluded. Empty buckets are omitted. None/NaN/infinity are skipped with
+    a warning in self.status; all-invalid observed buckets yield NaN. Otherwise
+    status is OK. Interval must be positive; inputs are unchanged. Processing
+    errors set status and return completed buckets.
+    """
+    interval = self.TD(interval, positive=True)
+    output = timeseries()
+    skipped = 0
+    if not self.data:
+      self._total_status('accumulate', 0)
+      return output
+    try:
+      start = override_startTime if override_startTime is not None else self.data[0][0]
+      buckets = {}
+      for timestamp, value in self.data:
+        if timestamp < start:
+          continue
+        bucket = start + ((timestamp - start) // interval) * interval
+        total, usable = buckets.get(bucket, (0, 0))
+        if value is None or not math.isfinite(value):
+          skipped += 1
+        else:
+          total += value
+          usable += 1
+        buckets[bucket] = total, usable
+      for timestamp, (total, usable) in buckets.items():
+        output.data.append([timestamp, total if usable else math.nan])
+      self._total_status('accumulate', skipped)
+    except Exception as e:
+      self.status = str(e)
+    return output
+
+  def accumulateWY(self, interval, incrTS, offset=datetime.timedelta(days=0)):
+    """Return a new series extending cumulative water-year totals.
+
+    Increment timestamps label period beginnings. Buckets are [start, start +
+    interval), and cumulative results retain the bucket's beginning label.
+    The last existing cumulative record already includes its entire period;
+    new buckets begin one interval later. With no existing records, start at
+    the first increment with a zero balance. Empty buckets are omitted.
+
+    Reset before adding the first bucket in a new water year, even across gaps.
+    A bucket belongs wholly to the year containing its start; it is not split
+    at the boundary. Offset shifts the October 1 midnight reset by a timedelta
+    (for example, six hours). It does not shift bucket labels or alignment.
+    Existing observations and both inputs are unchanged. Processing errors are
+    recorded in self.status, preserving the status-based API. None/NaN/infinity
+    increments are skipped, carrying known subtotals through all-invalid buckets.
+    An unusable seed is replaced by zero for extension only. Skipped increments
+    or seed warn in self.status that totals may be incomplete; otherwise status
+    is OK. Existing records are preserved, even if invalid.
+    """
+    interval = self.TD(interval, positive=True)
+    output = timeseries()
+    output.data = [row[:] for row in self.data]
+    skipped = 0
+    if not incrTS.data:
+      self._total_status('accumulateWY', 0)
+      return output
+
+    def water_year(timestamp):
+      shifted = timestamp - offset
+      return shifted.year + (shifted.month >= 10)
+
+    try:
+      if self.data:
+        last_time, total = self.data[-1]
+        if total is None or not math.isfinite(total):
+          total = 0
+          skipped += 1
+        start = last_time + interval
+        previous_year = water_year(last_time)
+      else:
+        start = incrTS.data[0][0]
+        total = 0
+        previous_year = water_year(start)
+
+      # Group only observed buckets, so large gaps require no empty-bin walk.
+      buckets = {}
+      for timestamp, value in incrTS.data:
+        if timestamp < start:
+          continue
+        bucket = start + ((timestamp - start) // interval) * interval
+        buckets.setdefault(bucket, 0)
+        if value is None or not math.isfinite(value):
+          skipped += 1
+        else:
+          buckets[bucket] += value
+      for timestamp, value in buckets.items():
+        year = water_year(timestamp)
+        if year != previous_year:
+          total = 0
+        total += value
+        output.insert(timestamp, total)
+        previous_year = year
+      self._total_status('accumulateWY', skipped)
+    except Exception as e:
+      self.status = str(e)
+    return output
+
+  def firstdifference(self):
+    """Return a new timeseries of first differences, equivalent to simpledelta().
+
+    Each value is the current observation minus the previous one, labeled
+    with the later timestamp. Empty/singleton input returns an empty series;
+    the input series is unchanged.
+    """
+    return self.simpledelta()
+
+  def simpledelta(self):
+    """Return a new timeseries of successive value differences (first differences).
+
+    Each value is the current observation minus the previous one, labeled
+    with the later timestamp. These are differences, not rates: elapsed time
+    does not divide the values. Empty/singleton input returns an empty series;
+    the input series is unchanged.
+    """
+    output = timeseries()
+    if len(self.data) < 2:
+      return output
+    try:
+      for i in range(1, len(self.data)):
+        d = self.data[i][1] - self.data[i - 1][1]
+        output.insert(self.data[i][0], d)
+    except Exception as e:
+      self.status = str(e)
+    return output
+
+  def maxmin(self, interval, cmp):
+    """Return beginning-labeled extrema for half-open interval buckets.
+
+    Buckets are anchored at the first observation. cmp(candidate, current)
+    selects the replacement: use > for maxima or < for minima. Strict
+    comparisons retain the earlier observation on ties. None/NaN/infinity are
+    skipped with a source-status warning; all-invalid observed buckets yield
+    NaN. Empty buckets are omitted and final partial buckets included. Interval
+    must be positive. Inputs are unchanged and output records independent.
+    Success without skips sets status to OK; processing errors set status and
+    return an empty series.
+    """
+    interval = self.TD(interval, positive=True)
+    output = timeseries()
+    skipped = 0
+    if not self.data:
+      self.status = 'OK'
+      return output
+    try:
+      start = self.data[0][0]
+      buckets = {}
+      for timestamp, value in self.data:
+        bucket = start + ((timestamp - start) // interval) * interval
+        current = buckets.setdefault(bucket, None)
+        if value is None or not math.isfinite(value):
+          skipped += 1
+        elif current is None or cmp(value, current):
+          buckets[bucket] = value
+      output.data = [[timestamp, value if value is not None else math.nan]
+                     for timestamp, value in buckets.items()]
+      self.status = (f"Warning: maxmin skipped {skipped} missing/nonfinite "
+                     "observation(s); extrema may be incomplete" if skipped else "OK")
+    except Exception as e:
+      self.status = str(e)
+      return timeseries()
+    return output
+
+  @requires_numpy
+  def savitzky_golay(self, window_size, order, deriv=0, rate=1):
+    '''Savitzky-Golay Smoothing filter using numpy
+       window_size: number of items in window integer
+       order: polynomial order
+       deriv: defaults to 0
+       rate : defaults to 1
+    '''
+    try:
+      window_size = abs(int(window_size))
+      order = abs(int(order))
+    except ValueError:
+      raise ValueError("SGFilter:window size and order must be of type int")
+
+    if window_size % 2 != 1 or window_size < 1:
+      raise TypeError("SGFilter:window size must be positive number")
+    if window_size < order + 2:
+      raise TypeError(
+          "SGFilter:window size is too small for the polynomials order")
+
+    if not self.data:
+      return timeseries()
+
+    half_window = (window_size - 1) // 2
+    # Fit polynomial coefficients at offsets around the center observation.
+    offsets = range(-half_window, half_window + 1)
+    design = np.array([[offset**power for power in range(order + 1)]
+                       for offset in offsets], dtype=float)
+    weights = np.linalg.pinv(design)[deriv] * rate**deriv * math.factorial(deriv)
+
+    # Repeat endpoint values so every observation has a full window.
+    values = np.array(self.values(), dtype=float)
+    padded = np.pad(values, half_window, mode='edge')
+    filtered = np.convolve(weights[::-1], padded, mode='valid')
+    return timeseries(zip(self.timestamps(), filtered))
+
+  @requires_numpy
+  def remove_stddev_outliers(self, threshold=1.5):
+    '''Remove Outliers using Standard Deviation'''
+    good = []
+    _data = []
+    a = []
+
+    # get the slice of values from timeseries
+    # y = np.array(self.data)
+    # a = y[:,1]
+    for rec in self.data:
+      a.append(rec[1])
+
+    avg = np.average(a)
+
+    # pad the first element with average
+    a = [avg] + a
+    out = [None] * len(a)
+    good.append(a[0])
+
+    std = np.std(a)
+    stdm = std * threshold
+
+    for n in range(1, len(a)):
+      dev = abs(a[n] - good[-1])
+
+      if dev < stdm:
+        good.append(a[n])
+        out[n] = a[n]
+      else:
+        out[n] = None
+
+    # remove the first (avg) element
+    out = out[1:]
+
+    # replace the values, only non nulls
+    # these will get interpolated later
+    for x in range(0, len(self.data)):
+      if out[x] != None:
+        _data.append([self.data[x][0], out[x]])
+
+    return timeseries(_data)
+
+  def rollingaverage(self, interval, *, require_full_window=False):
+    """Return a new timeseries of forward-looking moving averages.
+
+    Observations have equal weight. Windows include both endpoints and average
+    available observations, even when the window extends beyond the series. Set require_full_window=True
+    to omit incomplete trailing windows. Each result is labeled with the
+    requested window end, which may be beyond the final observation. None, NaN,
+    and infinity are excluded; all-invalid windows yield NaN. Interval must be
+    positive. Input records are unchanged.
+    """
+    interval = self.TD(interval, positive=True)
+    _data = []
+    if self.data == []:
+      return timeseries()
+    try:
+      i = 0
+      count = len(self.data)
+      while i < count:
+        startTime = self.data[i][0]
+        endTime = startTime + interval
+        if require_full_window and endTime > self.data[-1][0]:
+          break
+        n = 0
+        while self.data[i + n][0] <= endTime:
+          n += 1
+          if i + n >= count:
+            break
+        _data.append([endTime, _finite_mean(self.data[i:i + n])])
+        i += 1
+    except Exception as e:
+      self.status = str(e)
+    return timeseries(_data)
+
+  def movingaverage(self, interval, *, require_full_window=False):
+    """Return a new timeseries of backward-looking moving averages.
+
+    Observations have equal weight. Windows include both endpoints and average
+    available observations, even when the window starts before the series. Set require_full_window=True
+    to omit incomplete leading windows. Each result retains the observation
+    timestamp (the window end). Interval must be positive. Input records are
+    unchanged. None/NaN/infinity are excluded; all-invalid windows yield NaN.
+    Gaps do not require a complete sequence of observations.
+    """
+    interval = self.TD(interval, positive=True)
+    _data = []
+    if self.data == []:
+      return timeseries()
+    try:
+      i = len(self.data) - 1
+      while i >= 0:
+        endTime = self.data[i][0]
+        startTime = endTime - interval
+        if require_full_window and startTime < self.data[0][0]:
+          break
+
+        n = 0
+        while self.data[i - n][0] >= startTime:
+          n += 1
+          if i - n < 0:
+            break
+        _data.append([endTime, _finite_mean(self.data[i - n + 1:i + 1])])
+        i -= 1
+    except Exception as e:
+      self.status = str(e)
+    return timeseries(_data)
+
+  def centerMovingAverage(self, interval):
+    """Return a new timeseries of centered moving averages.
+
+    Windows include both endpoints and extend half the interval in each
+    direction. Observations have equal weight; edge windows average whatever
+    is available. Each result keeps its observation's timestamp. Interval must
+    be positive, as a duration string or timedelta. None/NaN/infinity are excluded;
+    all-invalid windows yield NaN. Input records are unchanged.
+    """
+    interval = self.TD(interval, positive=True)
+    _data = []
+    if not self.data:
+      return timeseries()
+    try:
+      left = right = 0
+      total = 0
+      usable = 0
+      half = interval / 2
+      for timestamp, value in self.data:
+        startTime = timestamp - half
+        endTime = timestamp + half
+        while right < len(self.data) and self.data[right][0] <= endTime:
+          value = self.data[right][1]
+          if value is not None and math.isfinite(value):
+            total += value
+            usable += 1
+          right += 1
+        while left < right and self.data[left][0] < startTime:
+          value = self.data[left][1]
+          if value is not None and math.isfinite(value):
+            total -= value
+            usable -= 1
+          left += 1
+        if usable == 0:
+          total = 0
+        _data.append([timestamp, total / usable if usable else math.nan])
+    except Exception as e:
+      self.status = str(e)
+    return timeseries(_data)
+
+  def percent(self, denom):
+    """Return 100 * self / denom at exact matching timestamps.
+
+    Uses the same pointwise policy as div(): missing/nonfinite values, zero
+    denominators, and overflow yield NaN without dropping other matched results.
+    Unmatched timestamps are omitted. Inputs are unchanged and output records
+    independent. Unexpected errors are reported in self.status, as in operation().
+    """
+    return self.operation(lambda numerator, denominator: 100 * (numerator / denominator), denom)
+
+  def snap(self, interval, buffer, starttime=None):
+    """Return nearest observed values on a regular grid, without interpolation.
+
+    Grid points run from starttime (or the first observation) through the last
+    observation's timestamp. Each independently selects the closest observation
+    within the inclusive +/- buffer. Ties choose the earlier observation; missing
+    grid points are omitted. An observation may supply multiple grid points.
+    Inputs are unchanged. Interval must be positive and buffer nonnegative.
+
+    Buffers greater than half the interval are honored, with a warning in
+    self.status because search windows overlap. No Python warning is emitted.
+    Success otherwise resets status to OK; processing errors set status and
+    return an empty series, preserving the existing error-reporting convention.
+    """
+    output = timeseries()
+    interval = self.TD(interval, positive=True)
+    buffer = self.TD(buffer)
+    if buffer < datetime.timedelta(0):
+      raise ValueError("buffer must be nonnegative")
+    self.status = ("Warning: snap buffer exceeds half the interval; "
+                   "overlapping windows may reuse observations"
+                   if buffer > interval / 2 else "OK")
+    if not self.data:
+      return output
+    try:
+      endtime = self.data[-1][0]
+      t = starttime if starttime is not None else self.data[0][0]
+      while t <= endtime:
+        pos = bisect_left(self.data, t, key=lambda row: row[0])
+        # Only the immediate neighbors can be closest. Earlier wins a tie:
+        # snap selects an observation, rather than inventing an average value.
+        candidates = self.data[max(0, pos - 1):min(len(self.data), pos + 1)]
+        closest = min(candidates, key=lambda row: abs(row[0] - t))
+        if abs(closest[0] - t) <= buffer:
+          output.data.append([t, closest[1]])
+        t += interval
+    except Exception as e:
+      self.status = str(e)
+      return timeseries()
+    return output
+
+  def filldown(self, interval, starttime=None, offset=None, _endtime=None):
+    """Return a new timeseries carrying previous values forward on an interval.
+
+    Steps before the first observation are omitted: no value is invented.
+    An explicit starttime sets step alignment, even if it precedes the data.
+    _endtime overrides the final timestamp; offset
+    extends to the daily boundary at that offset, stopping there when already
+    aligned. Interval must be positive.
+    Empty input returns an empty series even with explicit bounds: there is
+    no value to carry. The input series is unchanged.
+    """
+    interval = self.TD(interval, positive=True)
+    ts = timeseries()
+    if not self.data:
+      return ts
+    try:
+      i = 0
+      endtime = self.data[-1][0]
+      if _endtime != None:
+        endtime = _endtime
+      if offset is not None:
+        boundary = endtime.replace(hour=0, minute=0, second=0, microsecond=0) + offset
+        if boundary < endtime:
+          day = datetime.timedelta(days=1)
+          boundary += -((boundary - endtime) // day) * day
+        endtime = boundary
+      t = starttime if starttime is not None else self.data[0][0]
+      if t < self.data[0][0]:
+        # Skip unsupported leading steps without changing the requested alignment.
+        t += -((t - self.data[0][0]) // interval) * interval
+      while t <= endtime:
+        while i < len(self.data) and self.data[i][0] <= t:
+          i += 1
+        if i:
+          ts.data.append([t, self.data[i - 1][1]])
+        t += interval
+    except Exception as e:
+      self.status = str(e)
+      return timeseries()
+    return ts
+
+  def fillMissing(self, interval, value, starttime=None, endtime=None):
+    """Return observations within inclusive bounds, filling missing regular steps.
+
+    Bounds default to the first and last source timestamps. Regular steps start
+    at starttime; existing observations, including off-step records, are retained
+    only within the bounds and are never overwritten. Missing steps receive the
+    caller's value. Interval must be positive. Empty input requires both bounds
+    to generate observations; reversed bounds return empty. Inputs are unchanged
+    and output records are independent. Errors set self.status and return empty.
+    """
+    interval = self.TD(interval, positive=True)
+    ts = timeseries()
+    if not self.data and (starttime is None or endtime is None):
+      return ts
+    if starttime is None:
+      starttime = self.data[0][0]
+    if endtime is None:
+      endtime = self.data[-1][0]
+    try:
+      first = bisect_left(self.data, starttime, key=lambda row: row[0])
+      stop = bisect_right(self.data, endtime, key=lambda row: row[0])
+      i = first
+      t = starttime
+      # Merge existing observations and regular steps in timestamp order.
+      while t <= endtime:
+        while i < stop and self.data[i][0] < t:
+          ts.data.append(self.data[i][:])
+          i += 1
+        if i < stop and self.data[i][0] == t:
+          ts.data.append(self.data[i][:])
+          i += 1
+        else:
+          ts.data.append([t, value])
+        # Avoid stepping beyond datetime.max after the final supported step.
+        if endtime - t < interval:
+          break
+        t += interval
+      while i < stop:
+        ts.data.append(self.data[i][:])
+        i += 1
+    except Exception as e:
+      self.status = str(e)
+      return timeseries()
+    return ts
+
+  def timeshift(self, tdelta):
+    ''' Shifts each timestamp a given time interval
+        tdelta: timedelta to shift
+        returns a timeseries object '''
+    _data = []
+    if self.data == []:
+      return timeseries()
+    try:
+      for line in self.data:
+        _data.append([line[0] + tdelta, line[1]])
+    except Exception as e:
+      self.status = str(e)
+      return timeseries()
+    return timeseries(_data)
+
+  def subtract(self, operand):
+    '''Subtracts an operand timeseries or constant from self'''
+    return self.operation(lambda x, y: x - y, operand)
+
+  def add(self, operand):
+    '''Subtracts an operand timeseries or constant from self'''
+    return self.operation(lambda x, y: x + y, operand)
+
+  def mul(self, operand):
+    '''multiplies an operand timeseries or constant to self'''
+    return self.operation(lambda x, y: x * y, operand)
+
+  def div(self, operand):
+    '''divides an self by an operand timeseries or constant'''
+    return self.operation(lambda x, y: x / y, operand)
+
+  def operation(self, op, operand):
+    """Return pointwise results for a numeric scalar or exactly matching series.
+
+    op is a callable taking two values. Series operands contribute only exact
+    timestamp matches; a matched None value is not an absent timestamp. Missing
+    or nonfinite operands, division by zero, overflow, and nonfinite results
+    produce NaN at that timestamp without discarding other results. Inputs are
+    unchanged and records are independent. Unexpected errors still set status
+    and return an empty series, preserving the existing error convention.
+    """
+    output = timeseries()
+    if not self.data:
+      return output
+
+    def calculate(left, right):
+      if (left is None or right is None or
+          not math.isfinite(left) or not math.isfinite(right)):
+        return math.nan
+      try:
+        value = op(left, right)
+        return value if value is not None and math.isfinite(value) else math.nan
+      except (ZeroDivisionError, OverflowError):
+        return math.nan
+
+    try:
+      if type(operand) is float or type(operand) is int:
+        for timestamp, value in self.data:
+          output.data.append([timestamp, calculate(value, operand)])
+      else:
+        for timestamp, value in self.data:
+          index = operand.findIndex(timestamp)
+          if index != -1:
+            output.data.append([timestamp, calculate(value, operand.data[index][1])])
+    except Exception as e:
+      self.status = str(e)
+      return timeseries()
+    return output
+
+
+  def cull(self, op, operand):
+    ''' culls data from self
+        op: lambda function to perform
+            example: lambda x,y: x > y 
+            would return values in this timeseries that are greater than y
+        operand: could be a timeseries or a float
+        returns a timeseries object
+    '''
+    _data = []
+    if self.data == []:
+      return timeseries()
+    try:
+      if type(operand) is float or type(operand) is int:
+        for line in self.data:
+          if op(line[1], operand):
+            _data.append(line)
+      else:
+        for line in self.data:
+          val = operand.findValue(line[0])
+          if val != None:
+            if op(line[1], val):
+              _data.append(line)
+    except Exception as e:
+      self.status = str(e)
+      return timeseries()
+    return timeseries(_data)
+
+  def cullAbove(self, operand):
+    '''Culls values from self above operand
+       operand: accepts timeseries or a scalar
+       returns a timeseries object
+    '''
+    return self.cull (lambda x, y: x < y,operand)
+
+  def cullBelow(self, operand):
+    '''Culls values from self below to input
+       operand: accepts timeseries or a scalar
+       returns a timeseries object
+    '''
+    return self.cull (lambda x, y: x > y,operand)
+
+  def cullEqual(self, operand):
+    '''Culls values from self that are equal to input
+       operand: accepts timeseries or a scalar
+       returns a timeseries object
+    '''
+    return self.cull (lambda x, y: x != y, operand)
+
+  def cut(self, other):
+    ''' cuts a timeseries from self where datetimes intersect with
+        other. 
+        returns a timeseries object
+    '''
+    output = timeseries()
+    if self.data == []:
+      return timeseries()
+    try:
+      for line in other.data:
+        val = self.findValue(line[0])
+        if val != None:
+          output.insert(line[0], line[1])
+    except Exception as e:
+      self.status = str(e)
+      return timeseries()
+    return output
+
+  def TD(self, input, *, positive=False):
+    """Return a timedelta parsed from a duration string, or pass one through.
+
+    Units are w, Y (365 days), d, h, m, and s. Fractional and signed
+    components are supported, e.g. '1d 6h', '.5s', or '-1h30m' (minus
+    one hour plus thirty minutes). Zero/negative durations are valid unless
+    positive=True, which requires a duration strictly greater than zero.
+    Malformed/out-of-range strings raise ValueError; other types raise
+    TypeError. No partial duration is returned after a parse failure.
+    """
+    if isinstance(input, datetime.timedelta):
+      if positive and input <= datetime.timedelta(0):
+        raise ValueError("interval must be positive")
+      return input
+    if not isinstance(input, str):
+      raise TypeError("duration must be a string or timedelta")
+    text = ''.join(input.split())
+    component = r'([+-]?(?:\d+(?:\.\d*)?|\.\d+))([wYdhms])'
+    if not re.fullmatch(f'(?:{component})+', text):
+      raise ValueError(f"invalid duration: {input!r}")
+    seconds_per_unit = {'w': 604800, 'Y': 31536000, 'd': 86400,
+                        'h': 3600, 'm': 60, 's': 1}
+    output = datetime.timedelta(0)
+    try:
+      for number, unit in re.findall(component, text):
+        output += datetime.timedelta(seconds=float(number) * seconds_per_unit[unit])
+    except (OverflowError, ValueError) as error:
+      raise ValueError(f"duration out of range: {input!r}") from error
+    if positive and output <= datetime.timedelta(0):
+      raise ValueError("interval must be positive")
+    return output
+
+  def parseTimedelta(self, input, *, positive=False):
+    return self.TD(input, positive=positive)
+
+
+class rdb:
+  #construtor rewrites a path to a RDB file
+  def __init__(self, path):
+    #initialize with Default Configuration
+    self.status = "OK"
+    #format = INDEP   SHIFT   DEP     STOR
+    self.data = []
+    if path:
+      self.loadRDB(path)
+
+  def __str__(self):
+    output = "INDEP	SHIFT	DEP	STOR\n16N	16N	16N	1S\n"
+    for line in self.data:
+      output += "{}\n".format("\t".join(line))
+    return output
+
+  def loadRDB(self, path):
+    self.path = path
+    try:
+      self.data = self.parseRDB(open(path, "r"))
+    except Exception as e:
+      self.status = "\n{}".format(str(e))
+
+  def parseRDB(self, stream):
+    output = []
+    try:
+      for line in stream:
+        if (not "#" in line) and (len(line) > 1):
+          output.append(line.strip().split("\t"))
+    except Exception as e:
+      self.status = "\n%s" % str(e)
+    return output[2:]
+
+  def saveRDB(self, path):
+    open(path, "w").write(str(self))
+
+  #interpolate values
+  #This may be changed to a higher order interpolation in the future
+  def interpolateValue(self, x0, y0, x1, y1, x):
+    m = (y1 - y0) / (x1 - x0)
+    output = y0 + (x - x0) * m
+    return output
+
+  def rate(self, indep):
+    """ Rate a single value based on linear interpolation. """
+    data = self.data
+    index = len(data) - 2
+    for i in range(len(data) - 1):
+      if indep < float(data[i + 1][0]):
+        index = i
+        break
+    return self.interpolateValue(
+        float(data[index][0]),
+        float(data[index][2]),
+        float(data[index + 1][0]), float(data[index + 1][2]), indep)
+
+  def reverseRate(self, indep):
+    """ Reverse rate a single value based on linear interpolation
+        This switches the domain and range of the RDB and rates it. 
+    """
+    data = self.data
+    index = len(data) - 2
+    for i in range(len(data) - 1):
+      if indep < float(data[i + 1][2]):
+        index = i
+        break
+    return self.interpolateValue(
+        float(data[index][2]),
+        float(data[index][0]),
+        float(data[index + 1][2]), float(data[index + 1][0]), indep)
+
+  def makeRating(self, indepTS, depTS, precision=2, factor=1):
+    """ Creates a rating table based on two timeseries
+        precision is for how many decimal places to consider
+        factor to adjust dependent variable (kcfs->cfs, fudge factor, etc)"""
+    self.data = []
+    fmt = '.' + str(precision) + 'f'
+    ratingMap = {}
+    for t in indepTS.timestamps():
+      indep = indepTS.findValue(t)
+      dep = depTS.findValue(t)
+      if dep != None and indep != None:
+        key = float(format(indep, fmt))
+        if key in ratingMap:
+          ratingMap[key].append(dep * factor)
+        else:
+          ratingMap[key] = [dep * factor]
+    keys = sorted(ratingMap.keys())
+    for key in keys:
+      indep = sum(ratingMap[key]) / float(len(ratingMap[key]))
+      self.data.append([format(key, fmt), "0.00", format(indep, fmt), ""])
+
+  def rateTS(self, ts):
+    """ Generates a new time series with rated values from another """
+    output = []
+    for line in ts.data:
+      output.append([line[0], self.rate(line[1])])
+    return timeseries(output)
+
+  def reverseRateTS(self, ts):
+    """ Generates a new time series with reverse-rated values from another """
+    output = []
+    for line in ts.data:
+      output.append([line[0], self.reverseRate(line[1])])
+    return timeseries(output)
+
+
+class tablegrid:
+  #construtor rewrites a path to a RDB file
+  def __init__(self, path):
+    #initialize with Default Configuration
+    self.status = "OK"
+    self.path = None
+    self.data = self.loadTable(path)
+
+  def loadTable(self, path):
+    output = []
+    try:
+      self.path = path
+      input = open(path, "r")
+      for line in input:
+        if (not "#" in line) and (len(line) > 1):
+          row2 = []
+          if "," in line:
+            row1 = line.strip().split(",")
+          else:
+            row1 = line.strip().split("\t")
+          for n in row1:
+            row2.append(
+                float(n))  #Iterate and convert entries to floating point number
+          output.append(row2)
+    except Exception as e:
+      self.status = "\n%s" % str(e)
+    return output
+
+  def tableLookup(self, arr, colval, rowval):
+    #Seek the coordinates
+    output = 0
+    x = 0
+    y = 0
+    maxcols = len(arr[0]) - 1
+    maxrows = len(arr) - 1
+    x = maxcols - 1  #default to end of list if not found
+    y = maxrows - 1
+    i = 1
+    while (i < maxcols):  #seeking coordinates for column values
+      if arr[0][i + 1] > colval:
+        x = i
+        i = maxcols  #exit loop
+      i += 1
+    i = 1
+    while (i < maxrows):  #seeking coorinate for row values
+      if arr[i + 1][0] > rowval:
+        y = i
+        i = maxrows  #exit loop
+      i += 1
+    if ((x == maxcols) or (y == maxrows)):
+      output = arr[y][x]
+    else:
+      output = self.bilinear(arr[0][x], arr[y][0], arr[0][x + 1], arr[y + 1][0],
+                             arr[y][x], arr[y + 1][x], arr[y][x + 1],
+                             arr[y + 1][x + 1], colval, rowval)
+    return output
+
+  def bilinear(self, x1, y1, x2, y2, fQ11, fQ12, fQ21, fQ22, x, y):
+    retval = (fQ11 / ((x2 - x1) * (y2 - y1))) * (x2 - x) * (y2 - y) + (fQ21 / (
+        (x2 - x1) * (y2 - y1))) * (x - x1) * (y2 - y) + (fQ12 / (
+            (x2 - x1) * (y2 - y1))) * (x2 - x) * (y - y1) + (fQ22 / (
+                (x2 - x1) * (y2 - y1))) * (x - x1) * (y - y1)
+    return retval
+
+  #this takes 2 timseries objects and rates them
+  def rateTS(self, cols, rows):
+    output = []
+    for line in cols.data:
+      rowval = rows.findValue(line[0])
+      if rowval != None:
+        output.append(
+            [line[0], self.tableLookup(self.data, line[1], rowval)])
+    return timeseries(output)