shareddata 6.86.2__tar.gz → 6.99.0__tar.gz

{shareddata-6.86.2/src/shareddata.egg-info → shareddata-6.99.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.86.2
+Version: 6.99.0
 Summary: Memory-mapped record and time-series store with an HTTP API and a distributed scheduler
 Home-page: https://shareddata.net
 Author: shareddata

{shareddata-6.86.2 → shareddata-6.99.0}/setup.py

@@ -66,7 +66,7 @@ test_requires = [
 
 setup(
     name='shareddata',
-    version='6.86.2',
+    version='6.99.0',
     description='Memory-mapped record and time-series store with an HTTP API and a distributed scheduler',
     author='shareddata',
     author_email='admin@shareddata.net',
@@ -92,6 +92,7 @@ setup(
     extras_require={'test': test_requires, 's3': [], 'kafka': [], 'redis': [], 'all': []},
     include_package_data=True,
     package_data={
+        'SharedData.Charts': ['templates/*'],
         'SharedData.API': ['apidocs.yml', 'apidocs-dark.css', 'install/*'],
         'SharedData.Web': [
             'templates/*', 'templates/**/*',
@@ -105,6 +106,7 @@ setup(
             'shdata-web=SharedData.Web.server:main',
             'shdata-server=SharedData.server:main',
             'shareddata-mcp=SharedData.MCP.server:main',
+            'shareddata-research-mcp=SharedData.ResearchMCP:main',
         ],
     },
 )

{shareddata-6.86.2 → shareddata-6.99.0}/src/SharedData/API/__init__.py

@@ -111,12 +111,16 @@ def create_app(shdata=None, quiet=True, app=None):
 
     @app.before_request
     def auth_availability():
+        from flask import g
         from SharedData.Auth.store import StoreUnavailable
         # Worker install scripts and enrollment take a one-time install code instead of a token.
         public = request.path in ('/api/health', '/api/heartbeat', '/api/workers/enroll') or \
             request.path.startswith(('/api/auth/', '/api/web/', '/api/workers/install/'))
         from SharedData.API.auth import local_log_token_ok
-        if request.path.startswith('/api/') and not public and not local_log_token_ok(request):
+        if local_log_token_ok(request):
+            g.client_user = 'local-log'
+            return
+        if request.path.startswith('/api/') and not public:
             try:
                 if not authenticate_fn(request, shdata):
                     return jsonify(error='unauthorized'), 401
@@ -126,6 +130,9 @@ def create_app(shdata=None, quiet=True, app=None):
     # Register request lifecycle hooks
     register_request_hooks(app)
 
+    from SharedData.API import access_log
+    access_log.register(app, shdata)
+
     # Store shdata on app for access in routes
     app.shdata = shdata
 

shareddata-6.99.0/src/SharedData/API/access_log.py

@@ -0,0 +1,306 @@
+"""
+access_log.py — per-client API access logging, two-tier, never on the request path.
+
+Tier 1 (always): per-minute aggregate counters keyed by
+(user, route, status_class) — requests, bytes_in, bytes_out, total_ms, max_ms.
+Small docs; powers bandwidth / usage / route-preference analytics.
+
+Tier 2 (events): one document per request, but only for security-relevant or
+sampled traffic — status >= 300, failed authentication, unmatched paths, plus a
+configurable sample of successful requests.
+
+Flush: a daemon thread drains both tiers every FLUSH_INTERVAL_S seconds and
+writes through the in-process SharedData handle — Requests/RT/API/ACCESS_AGG
+and Requests/RT/API/ACCESS_EVT (hasindex=False collections → insert_many).
+No HTTP loopback: a ClientAPI.post_collection call would itself be a request
+and self-log. Each server worker process flushes its own buffer (worker_pid).
+
+Overhead: the request path does one dict update under a short lock and, for
+events, a put_nowait — no I/O, no body reads. A full event queue drops the
+record (counted in dropped_events). The request path never blocks on Mongo.
+
+Secrets: the token itself is never logged — only token_fp (first 8 chars of the
+server-side token_hash, stashed on flask.g by API/auth.py). The raw query
+string is never logged either (it can carry ?token=); only the allow-listed
+data-path arguments are kept.
+
+Client IP: X-Real-IP / X-Forwarded-For are trusted only when the direct peer
+is a known proxy; a LAN client hitting the API port directly cannot spoof its
+address. Built in: loopback only (a same-host proxy). A proxy on another
+address, or one that reaches the API by its LAN address, is listed in env
+SHAREDDATA_ACCESSLOG_TRUSTED (comma-separated, replaces the built-in list).
+"""
+
+import os
+import queue
+import random
+import socket
+import threading
+import time
+from collections import defaultdict
+from datetime import datetime, timezone
+
+from flask import g, request
+
+from SharedData.Logger import Logger
+
+# ─── Configuration ──────────────────────────────────────────────────────────
+
+FLUSH_INTERVAL_S = float(os.environ.get('SHAREDDATA_ACCESSLOG_FLUSH_S', 30))
+MAX_QUEUE_SIZE   = int(os.environ.get('SHAREDDATA_ACCESSLOG_QUEUE', 10_000))
+EVENT_SAMPLE     = float(os.environ.get('SHAREDDATA_ACCESSLOG_SAMPLE', 0.01))
+DISABLED         = os.environ.get('SHAREDDATA_ACCESSLOG', '1') in ('0', 'false', 'off')
+
+DATABASE   = 'Requests'
+PERIOD     = 'RT'
+SOURCE     = 'API'
+AGG_TABLE  = 'ACCESS_AGG'
+EVT_TABLE  = 'ACCESS_EVT'
+
+# Peers allowed to supply client addresses in X-Real-IP / X-Forwarded-For:
+# loopback (same-host proxy), unless the deployment lists its own proxies.
+_TRUSTED_PROXIES = frozenset(
+    part.strip() for part in (os.environ.get('SHAREDDATA_ACCESSLOG_TRUSTED') or '127.0.0.1,::1').split(',')
+    if part.strip()
+)
+
+# Public/polled endpoints and noise that is not worth a counter.
+_SKIP_PREFIXES = (
+    '/api/health',
+    '/api/heartbeat',
+    '/flasgger_static',
+    '/favicon.ico',
+)
+
+# Data-path arguments worth keeping on event records (the object a client
+# read/wrote). Everything else — most of all 'token' — stays out.
+_DATA_ARGS = ('database', 'period', 'source', 'tablename', 'tablesubfolder',
+              'timeseries', 'symbol', 'user')
+
+_PATH_MAX = 200
+_UA_MAX   = 160
+
+# ─── Internal state ─────────────────────────────────────────────────────────
+
+_evt_queue: 'queue.Queue[dict]' = queue.Queue(maxsize=MAX_QUEUE_SIZE)
+
+# {(minute_epoch, user, route, status_class): [requests, bytes_in, bytes_out, total_ms, max_ms]}
+_agg = defaultdict(lambda: [0, 0, 0, 0.0, 0.0])
+_agg_lock = threading.Lock()
+
+_pending_evts: list = []   # evt docs retained across a failed flush
+_pending_lock = threading.Lock()
+dropped_events = 0         # events lost to a full queue / retained-buffer cap
+
+_shdata = None
+_sink = None               # fn(collection_name, docs); injectable for tests
+_HOSTNAME = socket.gethostname()
+
+
+def _client_ip() -> str:
+    """The real client address: the peer itself, unless the peer is a known
+    proxy that supplied X-Real-IP / X-Forwarded-For (original client first)."""
+    peer = request.remote_addr or ''
+    if peer in _TRUSTED_PROXIES:
+        real = request.headers.get('X-Real-IP')
+        if real:
+            return real.split(',')[0].strip()
+        xff = request.headers.get('X-Forwarded-For')
+        if xff:
+            return xff.split(',')[0].strip()
+    return peer or 'unknown'
+
+
+def _user() -> str:
+    """Auth identity resolved by API/auth.authenticate (or 'local-log' set by
+    the auth_availability hook). 'invalid-token' when a token was presented
+    and rejected, 'anonymous' when none was needed/presented."""
+    user = getattr(g, 'client_user', None)
+    if user:
+        return user
+    if getattr(g, 'auth_ok', None) is False:
+        return 'invalid-token'
+    return 'anonymous'
+
+
+def _route() -> str:
+    rule = request.url_rule
+    return rule.rule if rule is not None else '<unmatched>'
+
+
+def _record_event(response, ms, bytes_in, bytes_out):
+    rec = {
+        'kind':        'evt',
+        'ts':          datetime.now(timezone.utc).isoformat(),
+        'host':        _HOSTNAME,
+        'worker_pid':  os.getpid(),
+        'remote_ip':   _client_ip(),
+        'user':        _user(),
+        'method':      request.method,
+        'route':       _route(),
+        'status':      response.status_code,
+        'ms':          round(ms, 3),
+        'bytes_in':    bytes_in,
+        'bytes_out':   bytes_out,
+    }
+    token_fp = getattr(g, 'token_fp', None)
+    if token_fp:
+        rec['token_fp'] = token_fp
+    if request.url_rule is None or response.status_code >= 300:
+        rec['path'] = request.path[:_PATH_MAX]          # raw path: probes/errors only
+    ua = request.headers.get('User-Agent')
+    if ua:
+        rec['user_agent'] = ua[:_UA_MAX]
+    args = {k: request.args.get(k) for k in _DATA_ARGS if request.args.get(k) is not None}
+    if args:
+        rec['args'] = args
+    global dropped_events
+    try:
+        _evt_queue.put_nowait(rec)
+    except queue.Full:
+        dropped_events += 1
+
+
+# ─── Flask hooks ────────────────────────────────────────────────────────────
+
+def before_request():
+    g._accesslog_start = time.perf_counter()
+
+
+def after_request(response):
+    try:
+        path = request.path
+        if any(path.startswith(p) for p in _SKIP_PREFIXES):
+            return response
+
+        start = getattr(g, '_accesslog_start', None)
+        ms = (time.perf_counter() - start) * 1000.0 if start is not None else 0.0
+        status = response.status_code
+
+        content_length = response.calculate_content_length()
+        bytes_out = content_length if content_length is not None else 0
+        bytes_in = g.get('request_bytes', request.content_length or 0)
+
+        minute = int(time.time() // 60) * 60
+        key = (minute, _user(), _route(), f'{status // 100}xx')
+        with _agg_lock:
+            bucket = _agg[key]
+            bucket[0] += 1
+            bucket[1] += bytes_in
+            bucket[2] += bytes_out
+            bucket[3] += ms
+            if ms > bucket[4]:
+                bucket[4] = ms
+
+        if status >= 300 or getattr(g, 'auth_ok', None) is False \
+                or random.random() < EVENT_SAMPLE:
+            _record_event(response, ms, bytes_in, bytes_out)
+
+    except Exception as exc:                       # logging must never break the app
+        try:
+            Logger.log.warning(f'access_log after_request: {exc}')
+        except Exception:
+            pass
+
+    return response
+
+
+# ─── Flush ──────────────────────────────────────────────────────────────────
+
+def _agg_docs(snapshot):
+    docs = []
+    for (minute, user, route, status_class), (n, bi, bo, total_ms, max_ms) in snapshot.items():
+        docs.append({
+            'kind':         'agg',
+            'ts':           datetime.fromtimestamp(minute, tz=timezone.utc).isoformat(),
+            'host':         _HOSTNAME,
+            'worker_pid':   os.getpid(),
+            'user':         user,
+            'route':        route,
+            'status_class': status_class,
+            'requests':     n,
+            'bytes_in':     bi,
+            'bytes_out':    bo,
+            'total_ms':     round(total_ms, 3),
+            'max_ms':       round(max_ms, 3),
+        })
+    return docs
+
+
+def _flush_once(sink):
+    """Drain both tiers and hand them to sink(name, docs). Returns doc counts.
+    On sink failure event docs are retained for the next cycle (capped);
+    aggregate buckets are minute-resolution and simply lost on failure."""
+    global dropped_events, _agg
+    with _agg_lock:
+        snapshot = _agg
+        _agg = defaultdict(lambda: [0, 0, 0, 0.0, 0.0])
+    with _pending_lock:
+        evts = list(_pending_evts)
+        _pending_evts.clear()
+    while True:
+        try:
+            evts.append(_evt_queue.get_nowait())
+        except queue.Empty:
+            break
+
+    agg_docs = _agg_docs(snapshot)
+    if not agg_docs and not evts:
+        return 0, 0
+    try:
+        if agg_docs:
+            sink(AGG_TABLE, agg_docs)
+        if evts:
+            sink(EVT_TABLE, evts)
+        return len(agg_docs), len(evts)
+    except Exception as exc:
+        Logger.log.error(f'access_log flush error: {exc}')
+        with _pending_lock:
+            keep = _pending_evts + evts
+            if len(keep) > MAX_QUEUE_SIZE:
+                dropped_events += len(keep) - MAX_QUEUE_SIZE
+                keep = keep[-MAX_QUEUE_SIZE:]
+            _pending_evts[:] = keep
+        # aggs are minute-buckets; on failure they are simply lost (bounded)
+        return 0, 0
+
+
+def _default_sink(name, docs):
+    coll = _shdata.collection(DATABASE, PERIOD, SOURCE, name,
+                              hasindex=False, user='master')
+    coll.extend(docs)
+
+
+def _flush_worker():
+    while True:
+        time.sleep(FLUSH_INTERVAL_S)
+        _flush_once(_sink or _default_sink)
+
+
+# ─── Registration ───────────────────────────────────────────────────────────
+
+def register(flask_app, shdata_instance, sink=None):
+    """Attach access logging to a Flask app; starts the flush thread once.
+
+    Parameters
+    ----------
+    flask_app       : Flask application instance
+    shdata_instance : SharedData handle for the direct collection write
+    sink            : optional fn(name, docs) override (tests)
+    """
+    global _shdata, _sink
+    if DISABLED:
+        Logger.log.info('access_log: disabled (SHAREDDATA_ACCESSLOG=0)')
+        return
+    if flask_app.extensions.get('shareddata_access_log'):
+        return
+    _shdata = shdata_instance
+    if sink is not None:
+        _sink = sink
+    flask_app.before_request(before_request)
+    flask_app.after_request(after_request)
+    flask_app.extensions['shareddata_access_log'] = True
+    threading.Thread(target=_flush_worker, daemon=True,
+                     name='api-access-log-flush').start()
+    Logger.log.info('access_log: registered (flush every %ds, event sample %.3f)',
+                    FLUSH_INTERVAL_S, EVENT_SAMPLE)

{shareddata-6.86.2 → shareddata-6.99.0}/src/SharedData/API/apidocs.yml

@@ -2,7 +2,7 @@
 openapi: 3.0.3
 info:
   title: SharedData API
-  version: 6.86.2
+  version: 6.99.0
   description: |
     HTTP API of a SharedData server. The same operations are available from Python
     through `SharedData.IO.ClientAPI` (for example `ClientAPI.get_table(...)`).
@@ -128,6 +128,95 @@ components:
       in: query
       name: token
       description: API token in the URL (avoid).
+  schemas:
+    ScheduleRoutine:
+      type: object
+      properties:
+        routine:
+          type: string
+          description: user@COMPUTER:script, the routine's name
+        user:
+          type: string
+        computer:
+          type: string
+        script:
+          type: string
+          description: Repo/folder/code, Repo#branch/folder/code or SharedData.Module
+        group:
+          type: string
+        realtime:
+          type: boolean
+        external:
+          type: boolean
+        runtimes:
+          type: string
+          description: HH:MM,HH:MM (server local time); empty for none
+        dependencies:
+          type: string
+          description: Comma-separated routine names
+        args:
+          type: string
+          description: Masked when a flag's name suggests a secret
+        args_masked:
+          type: boolean
+        health_url:
+          type: string
+    ScheduleRoutineEdit:
+      type: object
+      additionalProperties: false
+      required:
+      - revision
+      properties:
+        revision:
+          type: string
+          description: The revision GET /api/schedules/{name}/routines answered (64 hex characters)
+        user:
+          type: string
+          maxLength: 100
+          pattern: ^[A-Za-z0-9_.+-]+$
+          description: Required for POST and PUT
+        computer:
+          type: string
+          maxLength: 100
+          pattern: ^[A-Za-z0-9_.+-]+$
+          description: Required for POST and PUT
+        script:
+          type: string
+          maxLength: 600
+          description: Required for POST and PUT. Repo/folder/code[.py], Repo#branch/folder/code or SharedData.Module;
+            a path inside its repository
+        group:
+          type: string
+          maxLength: 2000
+        realtime:
+          type: boolean
+        external:
+          type: boolean
+        runtimes:
+          type: string
+          description: HH:MM, distinct, comma-separated; empty for none
+        dependencies:
+          oneOf:
+          - type: string
+          - type: array
+            items:
+              type: string
+          description: Routine names (user@COMPUTER:script, _HHMM for one run time of a routine with several), each
+            in the schedule, distinct, not later than this routine, no cycle
+        args:
+          type: string
+          maxLength: 20000
+          description: One line, single spaces between arguments (the Worker does not interpret quotes). Left out,
+            or sent back masked exactly as read, keeps the stored arguments
+        health_url:
+          type: string
+          description: Realtime routines only; http(s) on localhost or a private IP, no credentials, query or fragment
+        routine:
+          type: string
+          description: Read-only; may be sent back, then it must be the routine edited
+        args_masked:
+          type: boolean
+          description: Read-only; ignored
   parameters:
     Token:
       in: query
@@ -1727,6 +1816,29 @@ paths:
           description: No such schedule here
       description: 'What the SCHED page shows, for a script: the routines in run order, each with its status as the
         Scheduler keeps it, a level, its last message and how many seconds ago it spoke (`age`).'
+    delete:
+      tags:
+      - Workers
+      summary: Remove every routine of a schedule; it stays kept, empty (master token)
+      parameters:
+      - name: revision
+        in: query
+        description: The revision read, when it is not sent as {"revision"} in the body
+        schema:
+          type: string
+      responses:
+        '200':
+          description: '{schedule, revision, kept, routines: []}'
+        '400':
+          description: Revision missing
+        '403':
+          description: Master token required
+        '404':
+          description: No such schedule here
+        '409':
+          description: The schedule changed since its revision was read
+      description: 'The schedule is kept empty, so its old copy in the metadata store is never imported again.
+        Running processes are not stopped.'
   /api/workers:
     get:
       tags:
@@ -1874,6 +1986,136 @@ paths:
       description: 'Takes effect within a second and stays until switched back, also across a restart of the
         Scheduler. While a schedule is on `observe` the Scheduler goes on keeping the status of its routines; a
         routine can still be restarted or stopped by a person. Master token.'
+  /api/schedules/{name}/routines:
+    parameters:
+    - name: name
+      in: path
+      required: true
+      description: Schedule name, e.g. MASTER
+      schema:
+        type: string
+    get:
+      tags:
+      - Workers
+      summary: The definition of a schedule, every routine with the revision an edit sends back (master token)
+      responses:
+        '200':
+          description: '{schedule, revision, kept, routines: [ScheduleRoutine]}. kept=false: no such schedule
+            yet, its first routine creates it.'
+        '400':
+          description: Invalid schedule name
+        '403':
+          description: Master token required
+    post:
+      tags:
+      - Workers
+      summary: Add a routine to a schedule; the first one creates the schedule (master token)
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/ScheduleRoutineEdit'
+      responses:
+        '201':
+          description: '{schedule, revision, routine: ScheduleRoutine}'
+        '400':
+          description: A field is wrong (the error says which), revision missing, or a dependency is missing,
+            later than the routine or makes a cycle. Nothing was written.
+        '403':
+          description: Master token required
+        '409':
+          description: The schedule changed since its revision was read, or the routine already exists
+      description: 'Every field is checked before the store is touched. The Scheduler takes the routine up on its
+        next tick; nothing is started by the edit itself.'
+  /api/schedules/{name}/routines/{routine}:
+    parameters:
+    - name: name
+      in: path
+      required: true
+      description: Schedule name, e.g. MASTER
+      schema:
+        type: string
+    - name: routine
+      in: path
+      required: true
+      description: The routine's name, user@COMPUTER:script, e.g. user@HOST:Repo/folder/code (percent-encode '#')
+      schema:
+        type: string
+    get:
+      tags:
+      - Workers
+      summary: One routine of a schedule (master token)
+      responses:
+        '200':
+          description: '{schedule, revision, routine: ScheduleRoutine}'
+        '403':
+          description: Master token required
+        '404':
+          description: No such routine in the schedule
+    put:
+      tags:
+      - Workers
+      summary: The routine becomes these fields; one left out takes its default, except args (master token)
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/ScheduleRoutineEdit'
+      responses:
+        '200':
+          description: '{schedule, revision, routine: ScheduleRoutine}; a new user, computer or script renames it'
+        '400':
+          description: A field is wrong, revision missing, or the edit breaks a dependency. Nothing was written.
+        '403':
+          description: Master token required
+        '404':
+          description: No such schedule or routine
+        '409':
+          description: The schedule changed since its revision was read, or the new name is taken
+    patch:
+      tags:
+      - Workers
+      summary: Only the fields sent change; the routine is checked whole afterwards (master token)
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/ScheduleRoutineEdit'
+      responses:
+        '200':
+          description: '{schedule, revision, routine: ScheduleRoutine}'
+        '400':
+          description: A field is wrong, nothing to change, revision missing, or the edit breaks a dependency
+        '403':
+          description: Master token required
+        '404':
+          description: No such schedule or routine
+        '409':
+          description: The schedule changed since its revision was read, or the new name is taken
+    delete:
+      tags:
+      - Workers
+      summary: Remove a routine from a schedule; its running process is not stopped (master token)
+      parameters:
+      - name: revision
+        in: query
+        description: The revision read, when it is not sent as {"revision"} in the body
+        schema:
+          type: string
+      responses:
+        '200':
+          description: '{schedule, revision, deleted}'
+        '400':
+          description: Revision missing, or another routine depends on this one
+        '403':
+          description: Master token required
+        '404':
+          description: No such schedule or routine
+        '409':
+          description: The schedule changed since its revision was read
   /api/routes:
     get:
       tags:

{shareddata-6.86.2 → shareddata-6.99.0}/src/SharedData/API/auth.py

@@ -120,17 +120,21 @@ def check_permissions(reqpath: list, permissions: dict, method: str, identity=No
 def authenticate(request, shdata) -> bool:
     """
     Authenticate an incoming HTTP request by validating a token and verifying permissions.
-    
+
     Parameters:
         request (flask.Request): The incoming HTTP request object.
         shdata: SharedData instance for accessing user table.
-    
+
     Returns:
         bool: True if authenticated and authorized; False otherwise.
     """
+    from flask import g
     from SharedData.Auth.store import get_store, StoreUnavailable, token_hash
     token = get_token_from_request(request)
+    if token:
+        g.token_fp = token_hash(token)[:8]
     if not token:
+        g.auth_ok = False
         time.sleep(AUTH_FAILURE_SLEEP_SECONDS)
         return False
     try:
@@ -138,8 +142,12 @@ def authenticate(request, shdata) -> bool:
     except Exception as exc:
         raise StoreUnavailable('auth store lookup failed') from exc
     if not userdata:
+        g.auth_ok = False
+        g.client_user = 'invalid-token'
         time.sleep(AUTH_FAILURE_SLEEP_SECONDS)
         return False
+    g.auth_ok = True
+    g.client_user = userdata.get('email') or userdata.get('symbol') or 'unknown'
     with user_by_token_lock:
         if len(user_by_token) >= 4096:
             user_by_token.clear()