shareddata 6.83.12__tar.gz → 6.83.17__tar.gz

{shareddata-6.83.12 → shareddata-6.83.17}/MANIFEST.in

@@ -1,2 +1 @@
 include src/SharedData/sharedmutexwin.pyd
-include src/SharedData/IO/apidocs.yml

{shareddata-6.83.12/src/shareddata.egg-info → shareddata-6.83.17}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.83.12
+Version: 6.83.17
 Summary: Memory Mapped / Shared Memory Database with S3 repository
+License: MIT
 Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9, <3.13
 Description-Content-Type: text/markdown
@@ -39,6 +39,7 @@ Requires-Dist: redis==6.2.0
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
+Dynamic: license
 Dynamic: license-file
 Dynamic: requires-dist
 Dynamic: requires-python

{shareddata-6.83.12 → shareddata-6.83.17}/setup.py

@@ -34,13 +34,13 @@ install_requires = [
 
 setup(
     name='shareddata',
-    version='6.83.12',
+    version='6.83.17',
     description='Memory Mapped / Shared Memory Database with S3 repository',
+    license='MIT',
     long_description=open('README.md').read(),
     long_description_content_type='text/markdown',
     classifiers=[
         'Programming Language :: Python :: 3',
-        'License :: OSI Approved :: MIT License',
         'Operating System :: OS Independent',
     ],
     packages=find_packages(where='src'),

shareddata-6.83.17/src/SharedData/API/listcache.py

@@ -0,0 +1,91 @@
+"""
+TTL + single-flight cache for the expensive listing endpoints.
+
+``/api/tables``, ``/api/timeseries`` and ``/api/collections`` enumerate every
+partition on S3 (``list_remote``) and on local disk (``list_local``). With an
+empty ``keyword`` this scans the entire data lake and can take well over a
+minute. When several callers (the Dashboard polling + workers) hit the route at
+once, each one launches an independent full scan, the responses materialize in
+memory simultaneously, and the box sawtooths into swap while every request
+exceeds the upstream gateway timeout and returns 502/504.
+
+This cache fixes the dominant failure mode: it memoizes each listing for a short
+TTL and, crucially, runs the producer under a per-key lock (single-flight) so at
+most one scan executes per key at a time. Concurrent callers block on the lock
+and then receive the freshly cached result instead of each starting their own
+scan. Steady-state polling becomes instant; the storm of overlapping full scans
+disappears.
+
+Note: the very first (cold) call for a key still pays the full scan cost. That
+is inherent to a no-keyword full enumeration; pair this with a higher gateway
+read-timeout on the listing routes if cold calls must also stay under the proxy
+limit.
+"""
+
+import threading
+import time
+
+
+class _Entry:
+    __slots__ = ("value", "ts", "lock")
+
+    def __init__(self):
+        self.value = None
+        self.ts = 0.0
+        self.lock = threading.Lock()
+
+
+class TTLSingleFlightCache:
+    """Thread-safe TTL cache that collapses concurrent misses into one producer call."""
+
+    def __init__(self, ttl=60.0):
+        self.ttl = float(ttl)
+        self._entries = {}
+        self._guard = threading.Lock()
+
+    def _entry(self, key):
+        # Fast path under a short global lock just to fetch/create the per-key entry.
+        with self._guard:
+            entry = self._entries.get(key)
+            if entry is None:
+                entry = _Entry()
+                self._entries[key] = entry
+            return entry
+
+    def get(self, key, producer, ttl=None):
+        """
+        Return the cached value for ``key`` if fresh, otherwise call ``producer()``.
+
+        Only one thread runs ``producer`` per key at a time; others wait on the
+        per-key lock and then receive the value the winner just stored.
+        """
+        ttl = self.ttl if ttl is None else float(ttl)
+        entry = self._entry(key)
+
+        # Fresh hit, no locking needed.
+        if entry.value is not None and (time.time() - entry.ts) < ttl:
+            return entry.value
+
+        # Stale or empty -> single-flight refresh.
+        with entry.lock:
+            # Another thread may have refreshed while we waited on the lock.
+            if entry.value is not None and (time.time() - entry.ts) < ttl:
+                return entry.value
+            value = producer()
+            entry.value = value
+            entry.ts = time.time()
+            return value
+
+    def invalidate(self, key=None):
+        """Drop one key (or the whole cache when ``key`` is None)."""
+        with self._guard:
+            if key is None:
+                self._entries.clear()
+            else:
+                self._entries.pop(key, None)
+
+
+# Module-level singleton shared by the listing routes.
+# 60s TTL: listing metadata changes slowly, and a minute of staleness is
+# acceptable for a directory view while it eliminates the scan pile-up.
+LIST_CACHE = TTLSingleFlightCache(ttl=60.0)

{shareddata-6.83.12 → shareddata-6.83.17}/src/SharedData/API/routes/collections.py

@@ -23,6 +23,7 @@ from SharedData.API.constants import (
     DEFAULT_PAGE_SIZE,
     ERROR_SLEEP_SECONDS,
 )
+from SharedData.API.listcache import LIST_CACHE
 from SharedData.API.utils import (
     apply_subfolder,
     make_empty_response,
@@ -99,13 +100,20 @@ def init_collections_routes(shdata, authenticate_fn):
                 return jsonify({'error': 'unauthorized'}), 401
             keyword = request.args.get('keyword', '')
             user = request.args.get('user', DEFAULT_USER)
-            collections = shdata.list_collections(keyword=keyword, user=user)
-            if collections.empty:
+
+            def _produce():
+                collections = shdata.list_collections(keyword=keyword, user=user)
+                if collections.empty:
+                    return b''  # sentinel -> 204
+                collections_recs = collections.reset_index().to_dict('records')
+                collections_recs = CollectionMongoDB.serialize(collections_recs, iso_dates=True)
+                return json.dumps(collections_recs).encode('utf-8')
+
+            # Single-flight + TTL cache: see listcache.py.
+            response_data = LIST_CACHE.get(('collections', user, keyword), _produce)
+            if not response_data:
                 return Response(status=204)
-            collections = collections.reset_index().to_dict('records')
-            collections = CollectionMongoDB.serialize(collections, iso_dates=True)
-            response_data = json.dumps(collections).encode('utf-8')
-            response = Response(response_data, mimetype='application/json')        
+            response = Response(response_data, mimetype='application/json')
             return response
         except Exception as e:        
             response = Response(status=500)

{shareddata-6.83.12 → shareddata-6.83.17}/src/SharedData/API/routes/tables.py

@@ -21,6 +21,7 @@ from SharedData.API.constants import (
     ERROR_SLEEP_SECONDS,
     MAX_RESPONSE_SIZE_BYTES,
 )
+from SharedData.API.listcache import LIST_CACHE
 from SharedData.API.utils import (
     apply_subfolder,
     make_empty_response,
@@ -56,12 +57,19 @@ def init_tables_routes(shdata, authenticate_fn):
             keyword = request.args.get('keyword', '')
             user = request.args.get('user', DEFAULT_USER)
 
-            tables = shdata.list_tables(keyword=keyword, user=user)
-            if tables.empty:
+            def _produce():
+                tables = shdata.list_tables(keyword=keyword, user=user)
+                if tables.empty:
+                    return b''  # sentinel -> 204
+                tables_recs = tables.reset_index().to_dict('records')
+                tables_recs = CollectionMongoDB.serialize(tables_recs, iso_dates=True)
+                return json.dumps(tables_recs).encode('utf-8')
+
+            # Single-flight + TTL cache: collapses concurrent/repeat full scans so
+            # they no longer pile up past the gateway timeout (see listcache.py).
+            response_data = LIST_CACHE.get(('tables', user, keyword), _produce)
+            if not response_data:
                 return Response(status=204)
-            tables = tables.reset_index().to_dict('records')        
-            tables = CollectionMongoDB.serialize(tables, iso_dates=True)
-            response_data = json.dumps(tables).encode('utf-8')
             response = Response(response_data, mimetype='application/json')
             response.headers['Content-Length'] = len(response_data)
             return response
@@ -190,14 +198,36 @@ def init_tables_routes(shdata, authenticate_fn):
             loc = np.array(loc)
         
         # cycle query keys
-        if query.keys() is not None:        
-            for key in query.keys():            
-                if pd.api.types.is_string_dtype(tbl[key]):
-                    idx = tbl[loc][key] == query[key].encode()
-                elif pd.api.types.is_datetime64_any_dtype(tbl[key]):
-                    idx = tbl[loc][key] == pd.Timestamp(query[key])
-                else:                    
-                    idx = tbl[loc][key] == query[key]
+        if query.keys() is not None:
+            for key in query.keys():
+                # Take the column view FIRST, then index it.
+                #
+                # `tbl[key]` is zero-copy (SharedNumpy.__getitem__ slices to
+                # count, and a field name on an ndarray is a view), whereas
+                # `tbl[loc]` is fancy indexing -- a copy of every selected row
+                # at the FULL record width, done once per query key and applied
+                # BEFORE the 20 MiB response cap, so it was unbounded.
+                #
+                # Measured on a 208-byte record: old peak was exactly
+                # 217.0 B/row at 100k/200k/400k/800k rows and FLAT across a
+                # 2000x selectivity sweep (it copies the whole table either
+                # way); new peak is ~1 B/row.
+                #
+                # `tbl[:]` (not `tbl[key]`) is deliberate: SharedNumpy.__getitem__
+                # re-reads tbl.count on every access, so slicing the whole table
+                # here observes the count at COMPARISON time, exactly as the old
+                # `tbl[loc]` did. Caching a column view from an earlier count
+                # would diverge if a concurrent writer shrank the table -- e.g.
+                # TableIndex.create_index() lowers hdr['count'] when it drops
+                # invalid/duplicate rows -- letting this succeed where the old
+                # code raised IndexError (and vice versa on growth).
+                col = tbl[:][key]
+                if pd.api.types.is_string_dtype(col):
+                    idx = col[loc] == query[key].encode()
+                elif pd.api.types.is_datetime64_any_dtype(col):
+                    idx = col[loc] == pd.Timestamp(query[key])
+                else:
+                    idx = col[loc] == query[key]
                 loc = loc[idx]
         
         if len(loc) == 0:
@@ -206,28 +236,31 @@ def init_tables_routes(shdata, authenticate_fn):
         # filter columns
         pkey = DATABASE_PKEYS[database]
         columns = request.args.get('columns')
-        if columns:
-            if not tbl.is_schemaless:
-                columns = columns.split(',')
-                columns = np.array([c for c in columns if c not in pkey])
-                columns = pkey + list(np.unique(columns))
-                names = columns
-                formats = [tbl.dtype.fields[name][0].str for name in names]
-                dtype = np.dtype(list(zip(names, formats)))
-                # Apply pagination    
-                maxrows = int(np.floor(MAX_RESPONSE_SIZE_BYTES / dtype.itemsize))
-                maxrows = min(maxrows, len(loc))
-                if (per_page > maxrows) | (per_page == 0):
-                    per_page = maxrows        
-                startpage = (page - 1) * per_page
-                endpage = startpage + per_page        
-                content_pages = int(np.ceil(len(loc) / per_page))
-                recs2send = tbl[loc[startpage:endpage]]
-                # Create new array
-                arrays = [recs2send[field] for field in columns]
-                recs2send = np.rec.fromarrays(arrays, dtype=dtype)
-            else:
-                pass
+        # NOTE: `columns` on a SCHEMALESS table falls through to the plain
+        # pagination branch below. It used to hit `else: pass`, which left
+        # startpage/endpage/content_pages/recs2send unbound -> UnboundLocalError
+        # -> 500 -> 1s in-handler sleep -> 3 immediate client retries. The
+        # projection for schemaless tables is applied to the decoded dict_list
+        # further down instead.
+        if columns and not tbl.is_schemaless:
+            columns = columns.split(',')
+            columns = np.array([c for c in columns if c not in pkey])
+            columns = pkey + list(np.unique(columns))
+            names = columns
+            formats = [tbl.dtype.fields[name][0].str for name in names]
+            dtype = np.dtype(list(zip(names, formats)))
+            # Apply pagination
+            maxrows = int(np.floor(MAX_RESPONSE_SIZE_BYTES / dtype.itemsize))
+            maxrows = min(maxrows, len(loc))
+            if (per_page > maxrows) | (per_page == 0):
+                per_page = maxrows
+            startpage = (page - 1) * per_page
+            endpage = startpage + per_page
+            content_pages = int(np.ceil(len(loc) / per_page))
+            recs2send = tbl[loc[startpage:endpage]]
+            # Create new array
+            arrays = [recs2send[field] for field in columns]
+            recs2send = np.rec.fromarrays(arrays, dtype=dtype)
         else:
             # Apply pagination    
             maxrows = int(np.floor(MAX_RESPONSE_SIZE_BYTES / tbl.itemsize))
@@ -293,11 +326,13 @@ def init_tables_routes(shdata, authenticate_fn):
             if tbl.is_schemaless:
                 dict_list = tbl.get_dict_list(loc[startpage:endpage])
                 if columns:
+                    # was built into `filtered_dict_list` and then dropped on
+                    # the floor -- the UNfiltered list was sent to the client
                     columns = columns.split(',')
-                    filtered_dict_list = []
-                    for record in dict_list:
-                        filtered_record = {col: record[col] for col in columns if col in record}
-                        filtered_dict_list.append(filtered_record)
+                    dict_list = [
+                        {col: record[col] for col in columns if col in record}
+                        for record in dict_list
+                    ]
 
             json_payload = {
                 'names': names,

{shareddata-6.83.12 → shareddata-6.83.17}/src/SharedData/API/routes/timeseries.py

@@ -16,6 +16,7 @@ from SharedData.API.constants import (
     DEFAULT_USER,
     ERROR_SLEEP_SECONDS,
 )
+from SharedData.API.listcache import LIST_CACHE
 from SharedData.API.utils import (
     apply_subfolder,
     make_empty_response,
@@ -45,22 +46,24 @@ def init_timeseries_routes(shdata, authenticate_fn):
                 return jsonify({'error': 'unauthorized'}), 401
             keyword = request.args.get('keyword', '')
             user = request.args.get('user', DEFAULT_USER)
-            
-            timeseries_df = shdata.list_timeseries(keyword=keyword, user=user)
-            
-            if len(timeseries_df) == 0:
+
+            def _produce():
+                timeseries_df = shdata.list_timeseries(keyword=keyword, user=user)
+                if len(timeseries_df) == 0:
+                    return b''  # sentinel -> 204
+                timeseries_list = timeseries_df.reset_index().to_dict('records')
+                for ts in timeseries_list:
+                    for key, value in ts.items():
+                        if pd.isna(value):
+                            ts[key] = None
+                        elif isinstance(value, pd.Timestamp):
+                            ts[key] = value.isoformat()
+                return json.dumps(timeseries_list).encode('utf-8')
+
+            # Single-flight + TTL cache: see listcache.py.
+            response_data = LIST_CACHE.get(('timeseries', user, keyword), _produce)
+            if not response_data:
                 return Response(status=204)
-            
-            timeseries_list = timeseries_df.reset_index().to_dict('records')
-            
-            for ts in timeseries_list:
-                for key, value in ts.items():
-                    if pd.isna(value):
-                        ts[key] = None
-                    elif isinstance(value, pd.Timestamp):
-                        ts[key] = value.isoformat()
-            
-            response_data = json.dumps(timeseries_list).encode('utf-8')
             response = Response(response_data, mimetype='application/json')
             response.headers['Content-Length'] = len(response_data)
             return response