shareddata 6.86.1__tar.gz → 6.86.2__tar.gz

{shareddata-6.86.1/src/shareddata.egg-info → shareddata-6.86.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.86.1
+Version: 6.86.2
 Summary: Memory-mapped record and time-series store with an HTTP API and a distributed scheduler
 Home-page: https://shareddata.net
 Author: shareddata

{shareddata-6.86.1 → shareddata-6.86.2}/setup.py

@@ -66,7 +66,7 @@ test_requires = [
 
 setup(
     name='shareddata',
-    version='6.86.1',
+    version='6.86.2',
     description='Memory-mapped record and time-series store with an HTTP API and a distributed scheduler',
     author='shareddata',
     author_email='admin@shareddata.net',

{shareddata-6.86.1 → shareddata-6.86.2}/src/SharedData/API/apidocs.yml

@@ -2,7 +2,7 @@
 openapi: 3.0.3
 info:
   title: SharedData API
-  version: 6.86.1
+  version: 6.86.2
   description: |
     HTTP API of a SharedData server. The same operations are available from Python
     through `SharedData.IO.ClientAPI` (for example `ClientAPI.get_table(...)`).

{shareddata-6.86.1 → shareddata-6.86.2}/src/SharedData/IO/SaveTables.py

@@ -80,6 +80,29 @@ def _failed(row, error):
             'error': f'{type(error).__name__}: {error}'.replace('\n', ' ')[:300]}
 
 
+def live_daily_partitions(dftables: pd.DataFrame, now: datetime = None) -> pd.Series:
+    """Rows of daily partitions (YYYYMMDD) dated today or later in UTC.
+
+    A realtime writer is still appending to such a partition. Table.write holds the
+    table's mutex while it hashes, uploads and flushes the whole partition (minutes
+    for a 100 GB table), and every writer of the partition waits for that mutex, so
+    a backup would stall live data. The partition is complete the next night, when
+    it is still newer than the remote copy and is saved then.
+    """
+    now = datetime.now(timezone.utc) if now is None else now
+    today = now.astimezone(timezone.utc).strftime('%Y%m%d')
+    partition = dftables['partition'].fillna('').astype(str)
+    return partition.str.fullmatch(r'\d{8}') & (partition >= today)
+
+
+def select_tables(dftables: pd.DataFrame, now: datetime = None) -> pd.Series:
+    """Tables to save: local newer than remote (or no remote), not tiny, not live today."""
+    saveidx = (dftables['last_modified_local'].notnull())
+    saveidx = saveidx & ((dftables['last_modified_local'] > dftables['last_modified_remote']) | (dftables['last_modified_remote'].isnull()))
+    saveidx = saveidx & (dftables['size_local'] > 1000)
+    return saveidx & ~live_daily_partitions(dftables, now)
+
+
 def write_table(dftable_row: Dict[str, Any], retry=False) -> Dict[str, str]:
     """Save one table; return {'table', 'status': saved|failed|skipped, 'error'}."""
     path = table_path(dftable_row)
@@ -307,9 +330,13 @@ def main():
             're-uploading every local table would be catastrophic.')
         sys.exit(1)
 
-    saveidx = (dftables['last_modified_local'].notnull())
-    saveidx = saveidx & ((dftables['last_modified_local'] > dftables['last_modified_remote']) | (dftables['last_modified_remote'].isnull()))
-    saveidx = saveidx & (dftables['size_local'] > 1000)
+    saveidx = select_tables(dftables)
+    live = dftables[live_daily_partitions(dftables)
+                    & (dftables['last_modified_local'].notnull())
+                    & (dftables['size_local'] > 1000)]
+    if len(live):
+        Logger.log.info(f'SaveTables skipped {len(live)} live daily partitions (saved the next night): '
+                        + ', '.join(table_path(r) for r in live.to_dict(orient='records')))
 
     Logger.log.info('Saving the following tables:')
     for idx, row in dftables[saveidx].iterrows():

{shareddata-6.86.1 → shareddata-6.86.2/src/shareddata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.86.1
+Version: 6.86.2
 Summary: Memory-mapped record and time-series store with an HTTP API and a distributed scheduler
 Home-page: https://shareddata.net
 Author: shareddata