shareddata 6.86.1__tar.gz → 6.94.0__tar.gz

{shareddata-6.86.1/src/shareddata.egg-info → shareddata-6.94.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.86.1
+Version: 6.94.0
 Summary: Memory-mapped record and time-series store with an HTTP API and a distributed scheduler
 Home-page: https://shareddata.net
 Author: shareddata

{shareddata-6.86.1 → shareddata-6.94.0}/setup.py

@@ -66,7 +66,7 @@ test_requires = [
 
 setup(
     name='shareddata',
-    version='6.86.1',
+    version='6.94.0',
     description='Memory-mapped record and time-series store with an HTTP API and a distributed scheduler',
     author='shareddata',
     author_email='admin@shareddata.net',
@@ -92,6 +92,7 @@ setup(
     extras_require={'test': test_requires, 's3': [], 'kafka': [], 'redis': [], 'all': []},
     include_package_data=True,
     package_data={
+        'SharedData.Analytics': ['data/cdie/*.csv', 'data/cdie/*.json'],
         'SharedData.API': ['apidocs.yml', 'apidocs-dark.css', 'install/*'],
         'SharedData.Web': [
             'templates/*', 'templates/**/*',
@@ -105,6 +106,7 @@ setup(
             'shdata-web=SharedData.Web.server:main',
             'shdata-server=SharedData.server:main',
             'shareddata-mcp=SharedData.MCP.server:main',
+            'shareddata-research-mcp=SharedData.ResearchMCP:main',
         ],
     },
 )

{shareddata-6.86.1 → shareddata-6.94.0}/src/SharedData/API/__init__.py

@@ -111,12 +111,16 @@ def create_app(shdata=None, quiet=True, app=None):
 
     @app.before_request
     def auth_availability():
+        from flask import g
         from SharedData.Auth.store import StoreUnavailable
         # Worker install scripts and enrollment take a one-time install code instead of a token.
         public = request.path in ('/api/health', '/api/heartbeat', '/api/workers/enroll') or \
             request.path.startswith(('/api/auth/', '/api/web/', '/api/workers/install/'))
         from SharedData.API.auth import local_log_token_ok
-        if request.path.startswith('/api/') and not public and not local_log_token_ok(request):
+        if local_log_token_ok(request):
+            g.client_user = 'local-log'
+            return
+        if request.path.startswith('/api/') and not public:
             try:
                 if not authenticate_fn(request, shdata):
                     return jsonify(error='unauthorized'), 401
@@ -126,6 +130,9 @@ def create_app(shdata=None, quiet=True, app=None):
     # Register request lifecycle hooks
     register_request_hooks(app)
 
+    from SharedData.API import access_log
+    access_log.register(app, shdata)
+
     # Store shdata on app for access in routes
     app.shdata = shdata
 

shareddata-6.94.0/src/SharedData/API/access_log.py

@@ -0,0 +1,306 @@
+"""
+access_log.py — per-client API access logging, two-tier, never on the request path.
+
+Tier 1 (always): per-minute aggregate counters keyed by
+(user, route, status_class) — requests, bytes_in, bytes_out, total_ms, max_ms.
+Small docs; powers bandwidth / usage / route-preference analytics.
+
+Tier 2 (events): one document per request, but only for security-relevant or
+sampled traffic — status >= 300, failed authentication, unmatched paths, plus a
+configurable sample of successful requests.
+
+Flush: a daemon thread drains both tiers every FLUSH_INTERVAL_S seconds and
+writes through the in-process SharedData handle — Requests/RT/API/ACCESS_AGG
+and Requests/RT/API/ACCESS_EVT (hasindex=False collections → insert_many).
+No HTTP loopback: a ClientAPI.post_collection call would itself be a request
+and self-log. Each server worker process flushes its own buffer (worker_pid).
+
+Overhead: the request path does one dict update under a short lock and, for
+events, a put_nowait — no I/O, no body reads. A full event queue drops the
+record (counted in dropped_events). The request path never blocks on Mongo.
+
+Secrets: the token itself is never logged — only token_fp (first 8 chars of the
+server-side token_hash, stashed on flask.g by API/auth.py). The raw query
+string is never logged either (it can carry ?token=); only the allow-listed
+data-path arguments are kept.
+
+Client IP: X-Real-IP / X-Forwarded-For are trusted only when the direct peer
+is a known proxy; a LAN client hitting the API port directly cannot spoof its
+address. Built in: loopback only (a same-host proxy). A proxy on another
+address, or one that reaches the API by its LAN address, is listed in env
+SHAREDDATA_ACCESSLOG_TRUSTED (comma-separated, replaces the built-in list).
+"""
+
+import os
+import queue
+import random
+import socket
+import threading
+import time
+from collections import defaultdict
+from datetime import datetime, timezone
+
+from flask import g, request
+
+from SharedData.Logger import Logger
+
+# ─── Configuration ──────────────────────────────────────────────────────────
+
+FLUSH_INTERVAL_S = float(os.environ.get('SHAREDDATA_ACCESSLOG_FLUSH_S', 30))
+MAX_QUEUE_SIZE   = int(os.environ.get('SHAREDDATA_ACCESSLOG_QUEUE', 10_000))
+EVENT_SAMPLE     = float(os.environ.get('SHAREDDATA_ACCESSLOG_SAMPLE', 0.01))
+DISABLED         = os.environ.get('SHAREDDATA_ACCESSLOG', '1') in ('0', 'false', 'off')
+
+DATABASE   = 'Requests'
+PERIOD     = 'RT'
+SOURCE     = 'API'
+AGG_TABLE  = 'ACCESS_AGG'
+EVT_TABLE  = 'ACCESS_EVT'
+
+# Peers allowed to supply client addresses in X-Real-IP / X-Forwarded-For:
+# loopback (same-host proxy), unless the deployment lists its own proxies.
+_TRUSTED_PROXIES = frozenset(
+    part.strip() for part in (os.environ.get('SHAREDDATA_ACCESSLOG_TRUSTED') or '127.0.0.1,::1').split(',')
+    if part.strip()
+)
+
+# Public/polled endpoints and noise that is not worth a counter.
+_SKIP_PREFIXES = (
+    '/api/health',
+    '/api/heartbeat',
+    '/flasgger_static',
+    '/favicon.ico',
+)
+
+# Data-path arguments worth keeping on event records (the object a client
+# read/wrote). Everything else — most of all 'token' — stays out.
+_DATA_ARGS = ('database', 'period', 'source', 'tablename', 'tablesubfolder',
+              'timeseries', 'symbol', 'user')
+
+_PATH_MAX = 200
+_UA_MAX   = 160
+
+# ─── Internal state ─────────────────────────────────────────────────────────
+
+_evt_queue: 'queue.Queue[dict]' = queue.Queue(maxsize=MAX_QUEUE_SIZE)
+
+# {(minute_epoch, user, route, status_class): [requests, bytes_in, bytes_out, total_ms, max_ms]}
+_agg = defaultdict(lambda: [0, 0, 0, 0.0, 0.0])
+_agg_lock = threading.Lock()
+
+_pending_evts: list = []   # evt docs retained across a failed flush
+_pending_lock = threading.Lock()
+dropped_events = 0         # events lost to a full queue / retained-buffer cap
+
+_shdata = None
+_sink = None               # fn(collection_name, docs); injectable for tests
+_HOSTNAME = socket.gethostname()
+
+
+def _client_ip() -> str:
+    """The real client address: the peer itself, unless the peer is a known
+    proxy that supplied X-Real-IP / X-Forwarded-For (original client first)."""
+    peer = request.remote_addr or ''
+    if peer in _TRUSTED_PROXIES:
+        real = request.headers.get('X-Real-IP')
+        if real:
+            return real.split(',')[0].strip()
+        xff = request.headers.get('X-Forwarded-For')
+        if xff:
+            return xff.split(',')[0].strip()
+    return peer or 'unknown'
+
+
+def _user() -> str:
+    """Auth identity resolved by API/auth.authenticate (or 'local-log' set by
+    the auth_availability hook). 'invalid-token' when a token was presented
+    and rejected, 'anonymous' when none was needed/presented."""
+    user = getattr(g, 'client_user', None)
+    if user:
+        return user
+    if getattr(g, 'auth_ok', None) is False:
+        return 'invalid-token'
+    return 'anonymous'
+
+
+def _route() -> str:
+    rule = request.url_rule
+    return rule.rule if rule is not None else '<unmatched>'
+
+
+def _record_event(response, ms, bytes_in, bytes_out):
+    rec = {
+        'kind':        'evt',
+        'ts':          datetime.now(timezone.utc).isoformat(),
+        'host':        _HOSTNAME,
+        'worker_pid':  os.getpid(),
+        'remote_ip':   _client_ip(),
+        'user':        _user(),
+        'method':      request.method,
+        'route':       _route(),
+        'status':      response.status_code,
+        'ms':          round(ms, 3),
+        'bytes_in':    bytes_in,
+        'bytes_out':   bytes_out,
+    }
+    token_fp = getattr(g, 'token_fp', None)
+    if token_fp:
+        rec['token_fp'] = token_fp
+    if request.url_rule is None or response.status_code >= 300:
+        rec['path'] = request.path[:_PATH_MAX]          # raw path: probes/errors only
+    ua = request.headers.get('User-Agent')
+    if ua:
+        rec['user_agent'] = ua[:_UA_MAX]
+    args = {k: request.args.get(k) for k in _DATA_ARGS if request.args.get(k) is not None}
+    if args:
+        rec['args'] = args
+    global dropped_events
+    try:
+        _evt_queue.put_nowait(rec)
+    except queue.Full:
+        dropped_events += 1
+
+
+# ─── Flask hooks ────────────────────────────────────────────────────────────
+
+def before_request():
+    g._accesslog_start = time.perf_counter()
+
+
+def after_request(response):
+    try:
+        path = request.path
+        if any(path.startswith(p) for p in _SKIP_PREFIXES):
+            return response
+
+        start = getattr(g, '_accesslog_start', None)
+        ms = (time.perf_counter() - start) * 1000.0 if start is not None else 0.0
+        status = response.status_code
+
+        content_length = response.calculate_content_length()
+        bytes_out = content_length if content_length is not None else 0
+        bytes_in = g.get('request_bytes', request.content_length or 0)
+
+        minute = int(time.time() // 60) * 60
+        key = (minute, _user(), _route(), f'{status // 100}xx')
+        with _agg_lock:
+            bucket = _agg[key]
+            bucket[0] += 1
+            bucket[1] += bytes_in
+            bucket[2] += bytes_out
+            bucket[3] += ms
+            if ms > bucket[4]:
+                bucket[4] = ms
+
+        if status >= 300 or getattr(g, 'auth_ok', None) is False \
+                or random.random() < EVENT_SAMPLE:
+            _record_event(response, ms, bytes_in, bytes_out)
+
+    except Exception as exc:                       # logging must never break the app
+        try:
+            Logger.log.warning(f'access_log after_request: {exc}')
+        except Exception:
+            pass
+
+    return response
+
+
+# ─── Flush ──────────────────────────────────────────────────────────────────
+
+def _agg_docs(snapshot):
+    docs = []
+    for (minute, user, route, status_class), (n, bi, bo, total_ms, max_ms) in snapshot.items():
+        docs.append({
+            'kind':         'agg',
+            'ts':           datetime.fromtimestamp(minute, tz=timezone.utc).isoformat(),
+            'host':         _HOSTNAME,
+            'worker_pid':   os.getpid(),
+            'user':         user,
+            'route':        route,
+            'status_class': status_class,
+            'requests':     n,
+            'bytes_in':     bi,
+            'bytes_out':    bo,
+            'total_ms':     round(total_ms, 3),
+            'max_ms':       round(max_ms, 3),
+        })
+    return docs
+
+
+def _flush_once(sink):
+    """Drain both tiers and hand them to sink(name, docs). Returns doc counts.
+    On sink failure event docs are retained for the next cycle (capped);
+    aggregate buckets are minute-resolution and simply lost on failure."""
+    global dropped_events, _agg
+    with _agg_lock:
+        snapshot = _agg
+        _agg = defaultdict(lambda: [0, 0, 0, 0.0, 0.0])
+    with _pending_lock:
+        evts = list(_pending_evts)
+        _pending_evts.clear()
+    while True:
+        try:
+            evts.append(_evt_queue.get_nowait())
+        except queue.Empty:
+            break
+
+    agg_docs = _agg_docs(snapshot)
+    if not agg_docs and not evts:
+        return 0, 0
+    try:
+        if agg_docs:
+            sink(AGG_TABLE, agg_docs)
+        if evts:
+            sink(EVT_TABLE, evts)
+        return len(agg_docs), len(evts)
+    except Exception as exc:
+        Logger.log.error(f'access_log flush error: {exc}')
+        with _pending_lock:
+            keep = _pending_evts + evts
+            if len(keep) > MAX_QUEUE_SIZE:
+                dropped_events += len(keep) - MAX_QUEUE_SIZE
+                keep = keep[-MAX_QUEUE_SIZE:]
+            _pending_evts[:] = keep
+        # aggs are minute-buckets; on failure they are simply lost (bounded)
+        return 0, 0
+
+
+def _default_sink(name, docs):
+    coll = _shdata.collection(DATABASE, PERIOD, SOURCE, name,
+                              hasindex=False, user='master')
+    coll.extend(docs)
+
+
+def _flush_worker():
+    while True:
+        time.sleep(FLUSH_INTERVAL_S)
+        _flush_once(_sink or _default_sink)
+
+
+# ─── Registration ───────────────────────────────────────────────────────────
+
+def register(flask_app, shdata_instance, sink=None):
+    """Attach access logging to a Flask app; starts the flush thread once.
+
+    Parameters
+    ----------
+    flask_app       : Flask application instance
+    shdata_instance : SharedData handle for the direct collection write
+    sink            : optional fn(name, docs) override (tests)
+    """
+    global _shdata, _sink
+    if DISABLED:
+        Logger.log.info('access_log: disabled (SHAREDDATA_ACCESSLOG=0)')
+        return
+    if flask_app.extensions.get('shareddata_access_log'):
+        return
+    _shdata = shdata_instance
+    if sink is not None:
+        _sink = sink
+    flask_app.before_request(before_request)
+    flask_app.after_request(after_request)
+    flask_app.extensions['shareddata_access_log'] = True
+    threading.Thread(target=_flush_worker, daemon=True,
+                     name='api-access-log-flush').start()
+    Logger.log.info('access_log: registered (flush every %ds, event sample %.3f)',
+                    FLUSH_INTERVAL_S, EVENT_SAMPLE)

{shareddata-6.86.1 → shareddata-6.94.0}/src/SharedData/API/auth.py

@@ -120,17 +120,21 @@ def check_permissions(reqpath: list, permissions: dict, method: str, identity=No
 def authenticate(request, shdata) -> bool:
     """
     Authenticate an incoming HTTP request by validating a token and verifying permissions.
-    
+
     Parameters:
         request (flask.Request): The incoming HTTP request object.
         shdata: SharedData instance for accessing user table.
-    
+
     Returns:
         bool: True if authenticated and authorized; False otherwise.
     """
+    from flask import g
     from SharedData.Auth.store import get_store, StoreUnavailable, token_hash
     token = get_token_from_request(request)
+    if token:
+        g.token_fp = token_hash(token)[:8]
     if not token:
+        g.auth_ok = False
         time.sleep(AUTH_FAILURE_SLEEP_SECONDS)
         return False
     try:
@@ -138,8 +142,12 @@ def authenticate(request, shdata) -> bool:
     except Exception as exc:
         raise StoreUnavailable('auth store lookup failed') from exc
     if not userdata:
+        g.auth_ok = False
+        g.client_user = 'invalid-token'
         time.sleep(AUTH_FAILURE_SLEEP_SECONDS)
         return False
+    g.auth_ok = True
+    g.client_user = userdata.get('email') or userdata.get('symbol') or 'unknown'
     with user_by_token_lock:
         if len(user_by_token) >= 4096:
             user_by_token.clear()

{shareddata-6.86.1 → shareddata-6.94.0}/src/SharedData/API/routes/data.py

@@ -14,7 +14,7 @@ from SharedData.Auth.store import StoreUnavailable
 from SharedData.DataAPI import service
 from SharedData.DataAPI.errors import DataAPIError
 
-KEYED = ('symbol', 'marketdata', 'signals', 'portfolios', 'riskfactors', 'vol')
+KEYED = ('symbol', 'marketdata', 'signals', 'portfolios', 'riskfactors', 'vol', 'cdie')
 
 
 def token_permissions(request):

{shareddata-6.86.1 → shareddata-6.94.0}/src/SharedData/API/routes/tables.py

@@ -16,6 +16,7 @@ from SharedData.CollectionMongoDB import CollectionMongoDB
 from SharedData.Database import DATABASE_PKEYS
 from SharedData.Table import TableNotFound
 from SharedData.SharedNumpy import IndexNotAvailable
+from SharedData.TableVector import scalar_names
 from SharedData.TableRead import generation_read
 from SharedData.Listing import ListingBudget, ListingTimeout, ListingTooLarge
 
@@ -218,9 +219,16 @@ def init_tables_routes(shdata, authenticate_fn):
             since = np.datetime64(since.value, 'ns')
         plain = not (startdate or enddate or symbols or portfolios or tags or query)
 
+        if not tbl.count:
+            _require_filter_indexes(tbl, startdate, enddate, symbols, portfolios, tags)
+            return make_empty_response(204)
+
         loc = None
 
-        if startdate is not None and enddate is not None:
+        if symbols is not None:
+            # symbol first: the symbols' own rows filtered by date, instead of every row in the date range
+            loc = tbl.get_symbols_date_loc(symbols.split(','), startdate, enddate)
+        elif startdate is not None and enddate is not None:
             loc = tbl.get_date_loc_gte_lte(startdate, enddate)
         elif startdate is not None:
             loc = tbl.get_date_loc_gte(startdate)
@@ -238,18 +246,6 @@ def init_tables_routes(shdata, authenticate_fn):
             _require_filter_indexes(tbl, startdate, enddate, symbols, portfolios, tags)
             return make_empty_response(204)
 
-        # filter data by symbols    
-        if symbols is not None:
-            symbols = symbols.split(',')
-            symbolloc = []
-            for symbol in symbols:
-                symbolloc.extend(tbl.get_symbol_loc(symbol))
-            symbolloc = np.array(symbolloc)
-            if len(symbolloc) > 0:
-                loc = np.intersect1d(loc, symbolloc)
-            else:
-                loc = np.array([])
-
         # filter data by portfolios
         if portfolios is not None:
             portfolios = portfolios.split(',')
@@ -278,8 +274,13 @@ def init_tables_routes(shdata, authenticate_fn):
             loc = np.array(loc)
         
         # cycle query keys
-        if query.keys() is not None:
+        if not isinstance(query, dict):
+            return jsonify({'error': 'query must be a JSON object'}), 400
+        if query:
+            scalars = set(scalar_names(tbl.dtype))
             for key in query.keys():
+                if key not in scalars:
+                    return jsonify({'error': f'Unknown filter column: {key}'}), 400
                 # Take the column view FIRST, then index it.
                 #
                 # `tbl[key]` is zero-copy (SharedNumpy.__getitem__ slices to
@@ -342,8 +343,11 @@ def init_tables_routes(shdata, authenticate_fn):
         # projection for schemaless tables is applied to the decoded dict_list
         # further down instead.
         if columns and not tbl.is_schemaless:
-            columns = columns.split(',')
-            columns = np.array([c for c in columns if c not in pkey])
+            requested = [c for c in columns.split(',') if c and (c not in tbl.dtype.fields or tbl.dtype[c].subdtype is None)]
+            bad = [c for c in requested if c not in tbl.dtype.fields]
+            if bad:
+                return jsonify({'error': f'Unknown column: {bad[0]}'}), 400
+            columns = np.array([c for c in requested if c not in pkey])
             columns = pkey + list(np.unique(columns))
             names = columns
             formats = [tbl.dtype.fields[name][0].str for name in names]
@@ -423,6 +427,10 @@ def init_tables_routes(shdata, authenticate_fn):
                 response.headers['Content-Length'] = len(response_data_json)
                 return sent(response)
         else:  # output_format=='bin'
+            if any(recs2send.dtype[name].subdtype is not None for name in recs2send.dtype.names):
+                # a vector column stays local: the wire layout carries one value per field per row
+                from numpy.lib import recfunctions
+                recs2send = recfunctions.repack_fields(recs2send[scalar_names(recs2send.dtype)])
             names = list(recs2send.dtype.names)
             formats = [recs2send.dtype.fields[name][0].str for name in names]
 

shareddata-6.94.0/src/SharedData/Analytics/__init__.py

@@ -0,0 +1 @@
+"""Pure analytical calculations; importing this package does not connect to services."""