shareddata 6.83.17__tar.gz → 6.83.21__tar.gz

shareddata-6.83.21/MANIFEST.in

@@ -0,0 +1,3 @@
+include src/SharedData/sharedmutexwin.pyd
+include src/SharedData/IO/apidocs.yml
+prune tests

{shareddata-6.83.17/src/shareddata.egg-info → shareddata-6.83.21}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.83.17
+Version: 6.83.21
 Summary: Memory Mapped / Shared Memory Database with S3 repository
-License: MIT
 Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9, <3.13
 Description-Content-Type: text/markdown
@@ -39,7 +39,6 @@ Requires-Dist: redis==6.2.0
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
-Dynamic: license
 Dynamic: license-file
 Dynamic: requires-dist
 Dynamic: requires-python

{shareddata-6.83.17 → shareddata-6.83.21}/setup.py

@@ -34,13 +34,13 @@ install_requires = [
 
 setup(
     name='shareddata',
-    version='6.83.17',
+    version='6.83.21',
     description='Memory Mapped / Shared Memory Database with S3 repository',
-    license='MIT',
     long_description=open('README.md').read(),
     long_description_content_type='text/markdown',
     classifiers=[
         'Programming Language :: Python :: 3',
+        'License :: OSI Approved :: MIT License',
         'Operating System :: OS Independent',
     ],
     packages=find_packages(where='src'),

{shareddata-6.83.17 → shareddata-6.83.21}/src/SharedData/API/__init__.py

@@ -21,6 +21,7 @@ from SharedData.API.routes.timeseries import timeseries_bp, init_timeseries_rout
 from SharedData.API.routes.workers import workers_bp, init_workers_routes
 from SharedData.API.routes.metadata import metadata_bp, init_metadata_routes
 from SharedData.API.routes.cache import cache_bp, init_cache_routes
+from SharedData.API.routes.downloads import init_downloads_routes
 from SharedData.API.routes.system import (
     system_bp, 
     init_system_routes, 
@@ -85,6 +86,7 @@ def create_app(shdata=None, quiet=True):
     init_workers_routes(shdata, authenticate_fn)
     init_metadata_routes(shdata, authenticate_fn)
     init_cache_routes(shdata, authenticate_fn)
+    downloads_bp = init_downloads_routes(shdata, authenticate_fn)
     init_system_routes(shdata, authenticate_fn)
 
     # Register blueprints with /api prefix
@@ -94,6 +96,7 @@ def create_app(shdata=None, quiet=True):
     app.register_blueprint(workers_bp, url_prefix='/api')
     app.register_blueprint(metadata_bp, url_prefix='/api')
     app.register_blueprint(cache_bp, url_prefix='/api')
+    app.register_blueprint(downloads_bp, url_prefix='/api')
     app.register_blueprint(system_bp, url_prefix='/api')
 
     # Register request lifecycle hooks

{shareddata-6.83.17 → shareddata-6.83.21}/src/SharedData/API/constants.py

@@ -29,3 +29,10 @@ FORMAT_JSON = 'json'
 FORMAT_CSV = 'csv'
 FORMAT_BIN = 'bin'
 FORMAT_BSON = 'bson'
+
+# Downloads explorer (/api/list, /api/download)
+DOWNLOAD_SLOT_LIMIT = 5
+DOWNLOAD_SLOT_TTL_SECONDS = 6 * 60 * 60  # leak-safety if a worker dies mid-transfer
+LIST_DEFAULT_DEPTH = 1
+LIST_MAX_DEPTH = 3
+LIST_MAX_ENTRIES = 5000

shareddata-6.83.21/src/SharedData/API/download_paths.py

@@ -0,0 +1,126 @@
+"""
+Path safety and directory listing for the DOWNLOADS_FOLDER explorer.
+"""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any
+
+from SharedData.API.constants import LIST_MAX_ENTRIES
+
+
+class PathEscapeError(ValueError):
+    """Requested path is outside DOWNLOADS_FOLDER."""
+
+
+class DownloadsRootError(RuntimeError):
+    """DOWNLOADS_FOLDER is missing or not a directory."""
+
+
+def downloads_root() -> Path:
+    raw = os.environ.get('DOWNLOADS_FOLDER')
+    if not raw:
+        raise DownloadsRootError('DOWNLOADS_FOLDER is not set')
+    root = Path(raw).resolve()
+    if not root.is_dir():
+        raise DownloadsRootError(f'DOWNLOADS_FOLDER is not a directory: {root}')
+    return root
+
+
+def resolve_under_downloads(relpath: str | None) -> Path:
+    """
+    Resolve a client-supplied relative path so it stays inside DOWNLOADS_FOLDER.
+
+    Leading slashes are stripped. ``.``, empty, and ``/`` mean the folder root.
+    """
+    root = downloads_root()
+    rel = (relpath or '').strip()
+    if rel in ('', '.', '/'):
+        return root
+    if '\x00' in rel:
+        raise PathEscapeError('invalid path')
+    rel = rel.lstrip('/')
+    candidate = (root / rel).resolve()
+    try:
+        candidate.relative_to(root)
+    except ValueError as exc:
+        raise PathEscapeError('path escapes DOWNLOADS_FOLDER') from exc
+    return candidate
+
+
+def _relative_posix(path: Path, root: Path) -> str:
+    if path == root:
+        return ''
+    return path.relative_to(root).as_posix()
+
+
+def _skip_name(name: str) -> bool:
+    return name.startswith('.')
+
+
+def list_entries(
+    relpath: str | None,
+    depth: int,
+    max_entries: int = LIST_MAX_ENTRIES,
+) -> dict[str, Any]:
+    """
+    List files and directories under ``relpath`` up to ``depth`` levels.
+
+    ``depth=1`` is the requested directory's immediate children only.
+    Hidden names (leading ``.``) are omitted. Escaping symlinks are skipped.
+    """
+    root = downloads_root()
+    base = resolve_under_downloads(relpath)
+    if not base.exists():
+        raise FileNotFoundError(f'path not found: {relpath}')
+    if not base.is_dir():
+        raise NotADirectoryError(f'not a directory: {relpath}')
+
+    entries: list[dict[str, Any]] = []
+    truncated = False
+
+    def walk(dirpath: Path, remaining: int) -> None:
+        nonlocal truncated
+        if truncated:
+            return
+        try:
+            children = sorted(dirpath.iterdir(), key=lambda p: p.name.lower())
+        except OSError:
+            return
+        for child in children:
+            if truncated:
+                return
+            if _skip_name(child.name):
+                continue
+            try:
+                resolved = child.resolve()
+                resolved.relative_to(root)
+            except (OSError, ValueError):
+                continue
+            try:
+                st = child.stat()
+            except OSError:
+                continue
+            is_dir = child.is_dir()
+            entries.append({
+                'name': child.name,
+                'path': _relative_posix(resolved if not is_dir else child.resolve(), root),
+                'type': 'dir' if is_dir else 'file',
+                'size': None if is_dir else int(st.st_size),
+                'mtime': int(st.st_mtime),
+            })
+            if len(entries) >= max_entries:
+                truncated = True
+                return
+            if is_dir and remaining > 1:
+                walk(child, remaining - 1)
+
+    walk(base, depth)
+    return {
+        'path': _relative_posix(base, root),
+        'depth': depth,
+        'truncated': truncated,
+        'count': len(entries),
+        'entries': entries,
+    }

shareddata-6.83.21/src/SharedData/API/download_slots.py

@@ -0,0 +1,142 @@
+"""
+Global concurrent-download limiter.
+
+Gunicorn runs multiple worker processes (production: 8), so an in-memory
+counter would allow limit × workers. This limiter is process- and host-safe:
+
+- Redis (default when REDIS_CLUSTER_NODES is set): one counter for the
+  whole SharedData deployment.
+- File lock under DOWNLOADS_FOLDER: fallback for tests / Redis-unavailable,
+  still global across hosts that share that NFS folder.
+"""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from filelock import FileLock
+
+from SharedData.API.constants import DOWNLOAD_SLOT_LIMIT, DOWNLOAD_SLOT_TTL_SECONDS
+from SharedData.API.download_paths import downloads_root
+from SharedData.Logger import Logger
+
+_REDIS_SLOT_KEY = '{shareddata}:download_slots'
+
+_ACQUIRE_LUA = r"""
+local n = tonumber(redis.call('GET', KEYS[1]) or '0')
+if n >= tonumber(ARGV[1]) then
+  return -1
+end
+n = redis.call('INCR', KEYS[1])
+redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2]))
+return n
+"""
+
+_RELEASE_LUA = r"""
+local n = tonumber(redis.call('GET', KEYS[1]) or '0')
+if n <= 0 then
+  redis.call('SET', KEYS[1], 0)
+  return 0
+end
+return redis.call('DECR', KEYS[1])
+"""
+
+_limiter = None
+
+
+class DownloadSlotLimiter:
+    def acquire(self) -> bool:
+        raise NotImplementedError
+
+    def release(self) -> None:
+        raise NotImplementedError
+
+    def active(self) -> int:
+        raise NotImplementedError
+
+
+class FileDownloadSlotLimiter(DownloadSlotLimiter):
+    def __init__(self, root: Path | None = None, limit: int = DOWNLOAD_SLOT_LIMIT):
+        self.limit = limit
+        self.root = root if root is not None else downloads_root()
+        self.count_path = self.root / '.shareddata_download_slots.count'
+        self.lock_path = str(self.root / '.shareddata_download_slots.lock')
+
+    def _read(self) -> int:
+        try:
+            return max(0, int(self.count_path.read_text().strip() or '0'))
+        except (OSError, ValueError):
+            return 0
+
+    def acquire(self) -> bool:
+        with FileLock(self.lock_path):
+            n = self._read()
+            if n >= self.limit:
+                return False
+            self.count_path.write_text(str(n + 1))
+            return True
+
+    def release(self) -> None:
+        with FileLock(self.lock_path):
+            n = self._read()
+            self.count_path.write_text(str(max(0, n - 1)))
+
+    def active(self) -> int:
+        with FileLock(self.lock_path):
+            return self._read()
+
+
+class RedisDownloadSlotLimiter(DownloadSlotLimiter):
+    def __init__(self, limit: int = DOWNLOAD_SLOT_LIMIT, ttl: int = DOWNLOAD_SLOT_TTL_SECONDS):
+        from SharedData.CacheRedis import _build_sync_client
+
+        self.limit = limit
+        self.ttl = ttl
+        self._client = _build_sync_client()
+        self._acquire = self._client.register_script(_ACQUIRE_LUA)
+        self._release = self._client.register_script(_RELEASE_LUA)
+
+    def acquire(self) -> bool:
+        result = self._acquire(keys=[_REDIS_SLOT_KEY], args=[self.limit, self.ttl])
+        return int(result) > 0
+
+    def release(self) -> None:
+        self._release(keys=[_REDIS_SLOT_KEY], args=[])
+
+    def active(self) -> int:
+        raw = self._client.get(_REDIS_SLOT_KEY)
+        if raw is None:
+            return 0
+        try:
+            return max(0, int(raw))
+        except (TypeError, ValueError):
+            return 0
+
+
+def _make_limiter() -> DownloadSlotLimiter:
+    backend = os.environ.get('SHAREDDATA_DOWNLOAD_SLOT_BACKEND', '').strip().lower()
+    if backend == 'file':
+        return FileDownloadSlotLimiter()
+    if backend == 'redis' or os.environ.get('REDIS_CLUSTER_NODES'):
+        try:
+            return RedisDownloadSlotLimiter()
+        except Exception as exc:
+            Logger.log.warning(
+                'download slot limiter: Redis unavailable (%s); using file backend',
+                exc,
+            )
+            return FileDownloadSlotLimiter()
+    return FileDownloadSlotLimiter()
+
+
+def get_limiter() -> DownloadSlotLimiter:
+    global _limiter
+    if _limiter is None:
+        _limiter = _make_limiter()
+    return _limiter
+
+
+def reset_limiter() -> None:
+    """Drop the cached limiter (tests only)."""
+    global _limiter
+    _limiter = None

shareddata-6.83.21/src/SharedData/API/routes/downloads.py

@@ -0,0 +1,147 @@
+"""
+Downloads explorer blueprint: GET /api/list and GET /api/download.
+"""
+from __future__ import annotations
+
+import time
+
+from flask import Blueprint, Response, jsonify, request, stream_with_context
+
+from SharedData.API.constants import (
+    DOWNLOAD_SLOT_LIMIT,
+    ERROR_SLEEP_SECONDS,
+    LIST_DEFAULT_DEPTH,
+    LIST_MAX_DEPTH,
+)
+from SharedData.API.download_paths import (
+    DownloadsRootError,
+    PathEscapeError,
+    list_entries,
+    resolve_under_downloads,
+)
+from SharedData.API.download_slots import get_limiter
+from SharedData.API.utils import make_error_response, make_json_response, parse_int_param
+from SharedData.Logger import Logger
+
+
+def _clamp_depth(raw: str | None) -> int:
+    depth = parse_int_param(raw, LIST_DEFAULT_DEPTH)
+    if depth < 1:
+        return 1
+    if depth > LIST_MAX_DEPTH:
+        return LIST_MAX_DEPTH
+    return depth
+
+
+def init_downloads_routes(shdata, authenticate_fn):
+    """
+    Initialize downloads routes with required dependencies.
+
+    Returns a fresh blueprint so create_app / tests can register it more
+    than once in the same process.
+
+    Parameters:
+        shdata: SharedData instance.
+        authenticate_fn: Authentication function.
+    """
+    downloads_bp = Blueprint('downloads', __name__)
+
+    @downloads_bp.route('/list', methods=['GET'])
+    def list_downloads():
+        """GET /api/list — list files under DOWNLOADS_FOLDER up to a depth cap."""
+        try:
+            if not authenticate_fn(request, shdata):
+                return jsonify({'error': 'unauthorized'}), 401
+
+            depth = _clamp_depth(request.args.get('depth'))
+            payload = list_entries(request.args.get('path'), depth)
+            return make_json_response(payload)
+        except PathEscapeError as e:
+            return make_json_response({'error': str(e)}, status=400)
+        except DownloadsRootError as e:
+            return make_json_response({'error': str(e)}, status=503)
+        except FileNotFoundError as e:
+            return make_json_response({'error': str(e)}, status=404)
+        except NotADirectoryError as e:
+            return make_json_response({'error': str(e)}, status=400)
+        except Exception as e:
+            Logger.log.error(f'list_downloads error: {e}')
+            time.sleep(ERROR_SLEEP_SECONDS)
+            return make_error_response(e)
+
+    @downloads_bp.route('/download', methods=['GET'])
+    def download_file():
+        """GET /api/download — stream one regular file; 429 if 5 transfers are active."""
+        try:
+            if not authenticate_fn(request, shdata):
+                return jsonify({'error': 'unauthorized'}), 401
+
+            target = resolve_under_downloads(request.args.get('path'))
+            if not target.exists():
+                return make_json_response(
+                    {'error': f'path not found: {request.args.get("path")}'},
+                    status=404,
+                )
+            if not target.is_file():
+                return make_json_response(
+                    {'error': 'not a file'},
+                    status=400,
+                )
+
+            limiter = get_limiter()
+            if not limiter.acquire():
+                return make_json_response(
+                    {
+                        'error': 'download limit reached',
+                        'limit': DOWNLOAD_SLOT_LIMIT,
+                        'active': limiter.active(),
+                    },
+                    status=429,
+                )
+
+            released = False
+
+            def _release_once():
+                nonlocal released
+                if released:
+                    return
+                released = True
+                limiter.release()
+
+            def _stream():
+                try:
+                    with target.open('rb') as fh:
+                        while True:
+                            chunk = fh.read(1024 * 1024)
+                            if not chunk:
+                                break
+                            yield chunk
+                finally:
+                    _release_once()
+
+            try:
+                response = Response(
+                    stream_with_context(_stream()),
+                    mimetype='application/octet-stream',
+                    direct_passthrough=True,
+                )
+                response.headers['Content-Disposition'] = (
+                    f'attachment; filename="{target.name}"'
+                )
+                response.headers['Content-Length'] = str(target.stat().st_size)
+                response.call_on_close(_release_once)
+            except Exception:
+                _release_once()
+                raise
+
+            return response
+        except PathEscapeError as e:
+            return make_json_response({'error': str(e)}, status=400)
+        except DownloadsRootError as e:
+            return make_json_response({'error': str(e)}, status=503)
+        except Exception as e:
+            Logger.log.error(f'download_file error: {e}')
+            time.sleep(ERROR_SLEEP_SECONDS)
+            return make_error_response(e)
+
+    return downloads_bp

{shareddata-6.83.17 → shareddata-6.83.21}/src/SharedData/IO/ClientAPI.py

@@ -3054,3 +3054,66 @@ class ClientAPI:
         response = requests.post(url, params=params)
         ClientAPI.raise_for_status(response)
         return response.json().get('deleted', 0)
+
+    ################# DOWNLOADS #################
+
+    @staticmethod
+    def list_downloads(path: str = '', depth: int = 1,
+                       endpoint=None, token=None):
+        """
+        GET /api/list — list files and folders under DOWNLOADS_FOLDER.
+
+        Parameters:
+            path (str): Relative path under DOWNLOADS_FOLDER. Empty lists the root.
+            depth (int): Directory depth to include (server caps at 3).
+            endpoint (str, optional): API base URL. Defaults to SHAREDDATA_ENDPOINT.
+            token (str, optional): Auth token. Defaults to SHAREDDATA_TOKEN.
+
+        Returns:
+            dict: {path, depth, truncated, count, entries}
+        """
+        url = os.environ['SHAREDDATA_ENDPOINT'] if not endpoint else endpoint
+        url += '/api/list'
+        params = {'path': path, 'depth': depth}
+        if token is not None:
+            params['token'] = token
+        else:
+            params['token'] = os.environ.get('SHAREDDATA_TOKEN')
+            if not params['token']:
+                raise Exception('SHAREDDATA_TOKEN not found in environment variables')
+        response = requests.get(url, params=params, headers={'Accept': 'application/json'})
+        ClientAPI.raise_for_status(response)
+        return response.json()
+
+    @staticmethod
+    def download_file(path: str, dest, endpoint=None, token=None):
+        """
+        GET /api/download — stream one file from DOWNLOADS_FOLDER to ``dest``.
+
+        Parameters:
+            path (str): Relative file path under DOWNLOADS_FOLDER.
+            dest: Local filesystem path to write.
+            endpoint (str, optional): API base URL. Defaults to SHAREDDATA_ENDPOINT.
+            token (str, optional): Auth token. Defaults to SHAREDDATA_TOKEN.
+
+        Returns:
+            str: Absolute destination path.
+        """
+        url = os.environ['SHAREDDATA_ENDPOINT'] if not endpoint else endpoint
+        url += '/api/download'
+        params = {'path': path}
+        if token is not None:
+            params['token'] = token
+        else:
+            params['token'] = os.environ.get('SHAREDDATA_TOKEN')
+            if not params['token']:
+                raise Exception('SHAREDDATA_TOKEN not found in environment variables')
+        response = requests.get(url, params=params, stream=True)
+        ClientAPI.raise_for_status(response)
+        dest_path = os.path.abspath(str(dest))
+        os.makedirs(os.path.dirname(dest_path) or '.', exist_ok=True)
+        with open(dest_path, 'wb') as fh:
+            for chunk in response.iter_content(chunk_size=1024 * 1024):
+                if chunk:
+                    fh.write(chunk)
+        return dest_path

{shareddata-6.83.17 → shareddata-6.83.21}/src/SharedData/IO/MongoDBClient.py

@@ -260,4 +260,4 @@ class MongoDBClient:
             if pymongo.helpers._index_list(index_data['key']) == target_index:
                 return  # Index already exists
 
-        coll.create_index(index_fields, **kwargs)
+        coll.create_index(index_fields, **kwargs)

{shareddata-6.83.17 → shareddata-6.83.21}/src/SharedData/Logger.py

@@ -340,18 +340,14 @@ class Logger:
         Returns:
             pandas.DataFrame: The updated DataFrame containing processed log entries.
         """
-        # Serialize cursor + dflogs mutation: gunicorn gthread workers run
-        # concurrent request threads that would otherwise race _logfileposition
-        # / dflogs and serve inconsistent tails to UI polls.
-        with Logger.lock:
-            dfnewlines_sorted = Logger.readLogs()
-            if dfnewlines_sorted is None or len(dfnewlines_sorted) == 0:
-                return Logger.dflogs
-
-            Logger.getLastLog(dfnewlines_sorted)
-            Logger.getStatus(dfnewlines_sorted)
-            Logger.handleHeartbeats(keep_latest_heartbeat)
+        dfnewlines_sorted = Logger.readLogs()
+        if dfnewlines_sorted is None or len(dfnewlines_sorted) == 0:
             return Logger.dflogs
+        
+        Logger.getLastLog(dfnewlines_sorted)
+        Logger.getStatus(dfnewlines_sorted)
+        Logger.handleHeartbeats(keep_latest_heartbeat)
+        return Logger.dflogs
     
     @staticmethod
     def readLogs():

shareddata-6.83.21/src/SharedData/Routines/UpdateActiveJobs.py

@@ -0,0 +1,53 @@
+import faulthandler
+import time
+
+# Capture a Python-level traceback if a C extension (pandas_parser, numpy,
+# pyarrow, ...) crashes the interpreter with SIGSEGV/SIGABRT/SIGFPE/SIGBUS.
+# Without this, a crash leaves only a kernel segfault line with a raw IP.
+faulthandler.enable()
+
+# PROPRIETARY LIBS
+from SharedData.Routines.WorkerPool import WorkerPool
+from SharedData.Logger import Logger
+from SharedData.SharedData import SharedData
+shdata = SharedData('SharedData.Routines.UpdateActiveJobs', user='master')
+
+Logger.log.info('ROUTINE STARTED!')
+
+# Standalone routine form of the promotion sweep formerly run as an
+# in-process Worker thread (WorkerPool.update_active_jobs). Splitting it out
+# lets it be killed/restarted independently via the normal workerpool
+# commands without tearing down the Worker's API server and every other
+# child routine on the host — see docs/INCIDENT_WORKERPOOL_JOBS_INDEX_20260724.md.
+MAX_ERRORS = 10
+error_count = 0
+lastheartbeat = time.time()
+
+while True:
+    try:
+        if time.time() - lastheartbeat > 15:
+            lastheartbeat = time.time()
+            Logger.log.debug('#heartbeat#errors=%d' % error_count)
+
+        retry_delay = 15
+        try:
+            WorkerPool.update_active_jobs_once(shdata)
+            error_count = 0
+        except Exception as e:
+            Logger.log.error('Error in update_active_jobs_once: %s' % (str(e)))
+            error_count += 1
+            retry_delay = 60
+            if error_count >= MAX_ERRORS:
+                Logger.log.error(
+                    'Max errors (%d) reached, exiting' % (MAX_ERRORS))
+                raise SystemExit(1)
+
+        # Never hold the JOBS mutex during the retry backoff —
+        # update_active_jobs_once() always releases it before raising or
+        # returning.
+        time.sleep(retry_delay)
+    except SystemExit:
+        raise
+    except Exception as e:
+        Logger.log.error('UpdateActiveJobs Loop Exception: %s' % (str(e)))
+        time.sleep(60)

{shareddata-6.83.17 → shareddata-6.83.21}/src/SharedData/Routines/Worker.py

@@ -330,4 +330,4 @@ while True:
         )
         # Logger.log.debug(f'#heartbeat# {nroutines}routines,{running_batch_jobs}/{MAX_BATCH_JOBS}jobs,{completed_batch_jobs}completed,{error_batch_jobs}errors,{SHAREDDATA_VERSION}')
     
-    time.sleep(SLEEP_TIME * np.random.rand())
+    time.sleep(SLEEP_TIME * np.random.rand())

{shareddata-6.83.17 → shareddata-6.83.21}/src/SharedData/Routines/WorkerLib.py

@@ -42,7 +42,7 @@ def get_own_ip():
         s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
         try:
             # Doesn't need to be reachable, just used to determine the interface
-            s.connect(('10.255.255.255', 1))
+            s.connect(('192.0.2.1', 1))
             own_ip = s.getsockname()[0]
         except Exception:
             own_ip = '127.0.0.1'