shareddata 6.85.0__tar.gz → 6.85.1__tar.gz

{shareddata-6.85.0/src/shareddata.egg-info → shareddata-6.85.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.0
+Version: 6.85.1
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata

{shareddata-6.85.0 → shareddata-6.85.1}/setup.py

@@ -66,7 +66,7 @@ test_requires = [
 
 setup(
     name='shareddata',
-    version='6.85.0',
+    version='6.85.1',
     description='Memory Mapped / Shared Memory Database with S3 repository',
     author='shareddata',
     author_email='admin@shareddata.net',

{shareddata-6.85.0 → shareddata-6.85.1}/src/SharedData/API/apidocs.yml

@@ -2,7 +2,7 @@
 openapi: 3.0.3
 info:
   title: SharedData API
-  version: 6.85.0
+  version: 6.85.1
   description: |
     HTTP API of a SharedData server. The same operations are available from Python
     through `SharedData.IO.ClientAPI` (for example `ClientAPI.get_table(...)`).

{shareddata-6.85.0 → shareddata-6.85.1}/src/SharedData/DataAPI/resolve.py

@@ -164,16 +164,24 @@ class SymbolRegistry:
 
     def identify(self, query, remote=True):
         """Identity of *query*: a registry symbol, a ticker or vendor alias of exactly one
-        registry symbol, or a provider symbol of the registry collection (*remote*)."""
+        registry symbol, or a provider symbol of the registry collection (*remote*).
+        Active alias matches take precedence across fields; field order breaks ties.
+        Exact registry symbols and aliases with no active match retain their behavior."""
         query = str(query or '').strip()
         if not query:
             raise BadRequest('a symbol is required')
         self._load()
         if query in self._rows:
             return self._known(query, query, self._facts(self._rows[query]), 'metadata')
+        matches = [(name, lookup.get(query, [])) for name, lookup in self._reverse.items()]
+        if self._records is not None and 'isactive' in self._records.dtype.names:
+            active = {position for _, positions in matches for position in positions
+                      if self._records['isactive'][position] == 1}
+            if active:
+                matches = [(name, [position for position in positions if position in active])
+                           for name, positions in matches]
         candidates = []
-        for name, lookup in self._reverse.items():
-            positions = lookup.get(query, [])
+        for name, positions in matches:
             if len(positions) == 1:
                 return self._known(query, self._symbol(positions[0]), self._facts(positions[0]), f'alias:{name}')
             candidates.extend(self._symbol(position) for position in positions)

{shareddata-6.85.0 → shareddata-6.85.1}/src/SharedData/Redaction.py

@@ -912,9 +912,38 @@ class RedactingStream:
         return getattr(self.stream, name)
 
 
+class _ChildDebugFilter(logging.Filter):
+    """Keep Flask's level-less default handler from printing propagated DEBUG records."""
+    def __init__(self):
+        super().__init__()
+        self.names = set()
+
+    def filter(self, record):
+        return record.levelno > logging.DEBUG or record.name in self.names
+
+
+def _quiet_flask_default_handler(logger):
+    """Reading ``app.logger`` makes Flask attach its default handler, which has no
+    level, to a logger named after the app (``SharedData.API``). Every child logger
+    under that dotted name (the server's own, set to DEBUG with an INFO console
+    handler) propagates into it, so each ``#heartbeat#`` debug record was printed
+    again in Flask's format. Only the app logger's own DEBUG records stay visible;
+    INFO and above from children are untouched.
+    """
+    from flask.logging import default_handler
+    if default_handler not in logger.handlers:
+        return
+    quiet = getattr(default_handler, '_shareddata_child_debug', None)
+    if quiet is None:
+        quiet = default_handler._shareddata_child_debug = _ChildDebugFilter()
+        default_handler.addFilter(quiet)
+    quiet.names.add(logger.name)
+
+
 def install_flask_redaction(app):
     """Protect Flask logging and its request-local wsgi.errors writer."""
     install_redaction(app.logger)
+    _quiet_flask_default_handler(app.logger)
     if not getattr(app, '_shareddata_redaction', False):
         original = app.wsgi_app
         def redacted_wsgi(environ, start_response):

{shareddata-6.85.0 → shareddata-6.85.1}/src/SharedData/Table.py

@@ -17,6 +17,8 @@ import asyncio
 import importlib
 import bson
 import mmap
+import stat
+import uuid
 
 from SharedData.Logger import Logger
 from SharedData.LockWaiters import TableBusy
@@ -635,14 +637,18 @@ class Table:
         Raises:
             Exception: If the table does not exist and cannot be created, or if the index cannot be properly created.
         """
-        if self.type == 1 and self.filepath.exists():
+        replace_image = self.type == 1 and self.filepath.exists()
+        if replace_image and not self.overwrite:
             from SharedData.TableRewrite import RewriteError
             raise RewriteError('overwriting an existing disk image requires staged publication')
         if (not self.names is None) \
             & (not self.formats is None)\
                 & (not self.size is None):
-            self.create_header()            
-            self.malloc(create=True)
+            self.create_header()
+            if replace_image:
+                self._publish_empty_image()
+            else:
+                self.malloc(create=True)
 
         elif (not self.exists) | (self.overwrite):
             if not self.exists:
@@ -656,6 +662,52 @@ class Table:
         if self.is_schemaless:
             self.init_bson_mmap()
 
+    def _publish_empty_image(self):
+        """
+        Replace the existing disk image with a new empty one (overwrite=True).
+
+        The new image is built complete under a stage name beside data.bin and
+        installed by one os.replace, inside the caller's STRUCT section, so a
+        failure or crash before the replace leaves the old image untouched.
+        The mappings made on the stage stay valid after the rename (same inode);
+        other processes reopen by path when the epoch is published on exit from
+        the structure section. The index and BSON files only describe the old
+        records, so they are removed and rebuilt from the new empty header.
+        """
+        from SharedData.TableRewrite import (RewriteError, fsync_directory,
+                                             require_rewrite_support)
+        require_rewrite_support()
+        final = self.filepath
+        old = os.stat(final)
+        # The writer lock is held: any earlier stage is from a build that died.
+        for leftover in final.parent.glob('data.bin.overwrite.*'):
+            leftover.unlink()
+        stage = final.parent / ('data.bin.overwrite.' + uuid.uuid4().hex)
+        self.filepath = stage
+        try:
+            self.malloc(create=True)
+            os.chmod(stage, stat.S_IMODE(old.st_mode))
+            try:
+                os.chown(stage, old.st_uid, old.st_gid)
+            except OSError:
+                pass
+            os.replace(stage, final)
+        except BaseException:
+            self.filepath = final
+            try:
+                stage.unlink()
+            except FileNotFoundError:
+                pass
+            raise
+        self.filepath = final
+        fsync_directory(final.parent)
+        for name in ('bson.bin', 'dateidx.bin', 'pkey.bin', 'symbolidx.bin', 'portidx.bin'):
+            try:
+                (final.parent / name).unlink()
+            except FileNotFoundError:
+                pass
+        fsync_directory(final.parent)
+
     def create_header(self):
 
         """

{shareddata-6.85.0 → shareddata-6.85.1}/src/SharedData/Web/config.py

@@ -216,6 +216,7 @@ STANDALONE_COMMANDS = {
     'ROUTES': {'route': '/admin/routes', 'description': 'Other web services opened from this site', 'type': 'command', 'group': 'Admin', 'master': True},
     'WORKERS': {'route': '/admin/workers', 'description': 'Add workers, join and command tokens, Git credentials', 'type': 'command', 'group': 'Admin', 'master': True},
     'SCHED': {'route': '/admin/sched', 'description': 'Schedules: status of every routine, restart and stop', 'type': 'command', 'group': 'System', 'master': True},
+    'PROC': {'route': '/admin/proc', 'description': 'Processes: what each routine running on this server uses, restart and stop', 'type': 'command', 'group': 'System', 'master': True},
     'LOGS': {'route': '/admin/logs', 'description': 'Live tail of the logs', 'type': 'command', 'group': 'System', 'master': True},
     'WORKERPOOL': {'route': '/admin/workerpool', 'description': 'Worker pool: a terminal to the workers, and the repositories they can run', 'type': 'command', 'group': 'System', 'master': True},
     # The services SharedData can use: listed whether or not one is configured, since each page is also where its connection is set.

shareddata-6.85.1/src/SharedData/Web/proc.py

@@ -0,0 +1,170 @@
+"""The PROC page's data: the routines whose processes run on this server, and the two commands
+a master may send from it (restart, stop).
+
+A routine is one line however many processes it has (a routine with several workers is one
+card). Its numbers are those of all its processes together. Platform processes (the Worker, the
+Scheduler, this server) are shown but are not restarted or stopped from here: their Worker keeps
+them running, and stopping the server would stop the page. So are processes that are not a script
+the Worker runs (an installed program such as vllm or rtxclaw): a restart would try to start that
+file as a Python script.
+"""
+import hashlib
+import os
+import threading
+import time
+from datetime import datetime, timezone
+
+import psutil
+
+from SharedData.Routines.RoutineCommand import is_scheduler
+from SharedData.Web import sched
+
+PIDS_SHOWN = 20          # process ids listed on a card
+SAMPLE_FRESH = 30        # seconds a CPU reading can serve as the start of the next one
+SAMPLE_WAIT = 0.25       # seconds the first reading of a process is taken over
+BAD_STATES = ('zombie', 'dead', 'stopped', 'stop')
+# What the page asks for -> the worker's job (its own 'stop' does nothing).
+JOBS = {job: sched.JOBS[job] for job in ('restart', 'stop')}
+
+_samples = {}            # (pid, start) -> (monotonic seconds, cpu seconds), kept between two readings
+_lock = threading.Lock()
+
+
+def _script(command):
+    """What follows 'user@COMPUTER:' in a routine's name, as the SCHED page writes it."""
+    repo, routine = command.get('repo') or '', command.get('routine') or ''
+    if repo == 'SharedData':
+        return 'SharedData.' + routine
+    branch = command.get('branch')
+    return repo + ('#' + branch if branch and branch != 'nan' else '') + '/' + (routine[:-3] if routine.endswith('.py') else routine)
+
+
+def _key(command):
+    parts = [str(command.get(name) or '') for name in ('repo', 'branch', 'routine', 'args')]
+    return hashlib.sha1('\0'.join(parts).encode()).hexdigest()[:12]
+
+
+def _cpu_seconds(process):
+    times = process.cpu_times()
+    return times.user + times.system
+
+
+def _read_cpu(processes):
+    """{(pid, start): percent of one core since the previous reading of that process}. A process
+    without a recent reading is measured over one short wait, shared by all of them; the page
+    asks every few seconds, so the later readings do not wait."""
+    first = time.monotonic()
+    by_key, spent = {}, {}
+    for process in processes:
+        try:
+            key = (process.pid, process.create_time())
+            by_key[key], spent[key] = process, _cpu_seconds(process)
+        except psutil.Error:
+            continue
+    with _lock:
+        known = {key: _samples[key] for key in by_key if key in _samples and first - _samples[key][0] <= SAMPLE_FRESH}
+    percent = {key: max(0.0, (spent[key] - used) / max(first - when, 1e-3) * 100) for key, (when, used) in known.items()}
+    new = [key for key in by_key if key not in known]
+    if new:
+        time.sleep(SAMPLE_WAIT)
+        later = time.monotonic()
+        for key in new:
+            try:
+                percent[key] = max(0.0, (_cpu_seconds(by_key[key]) - spent[key]) / max(later - first, 1e-3) * 100)
+            except psutil.Error:
+                percent[key] = 0.0
+    with _lock:
+        for key in [key for key in _samples if key not in by_key]:
+            del _samples[key]
+        for key in by_key:
+            _samples[key] = (first, spent[key])
+    return percent
+
+
+def _iso(epoch):
+    return datetime.fromtimestamp(epoch, timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
+
+
+def _routines():
+    from SharedData.Routines.WorkerLib import update_routines
+    found = []
+    update_routines(found)
+    return found
+
+
+def rows():
+    """The routines running here, one dict each. '_args' and '_branch' are the real values and
+    never leave this module (an argument can be a secret)."""
+    found = _routines()
+    processes = [routine['process'] for routine in found if routine.get('process') is not None]
+    cpu = _read_cpu(processes)
+    grouped = {}
+    for routine in found:
+        process, command = routine.get('process'), routine.get('command') or {}
+        try:
+            with process.oneshot():
+                start = process.create_time()
+                item = dict(pid=routine['pid'], cpu=cpu.get((process.pid, start), 0.0), memory=process.memory_info().rss,
+                            threads=process.num_threads(), state=process.status(), started=start)
+                try:
+                    item['files'] = process.num_fds() if hasattr(process, 'num_fds') else len(process.open_files())
+                except psutil.Error:
+                    item['files'] = None
+        except Exception:                      # the process ended while it was being read
+            continue
+        key = _key(command)
+        row = grouped.setdefault(key, dict(
+            key=key, script=_script(command), repo=command.get('repo') or '', branch=command.get('branch') or '',
+            routine=command.get('routine') or '', args=sched.mask_args(command.get('args')), _args=command.get('args') or '',
+            processes=0, pids=[], cpu=0.0, memory_mb=0.0, threads=0, files=0, states=set(), started=None,
+            platform=command.get('repo') == 'SharedData', script_run=str(command.get('routine') or '').endswith('.py')))
+        row['processes'] += 1
+        row['pids'].append(item['pid'])
+        row['cpu'] += item['cpu']
+        row['memory_mb'] += item['memory'] / (1 << 20)
+        row['threads'] += item['threads']
+        row['files'] = None if row['files'] is None or item['files'] is None else row['files'] + item['files']
+        row['states'].add(item['state'])
+        row['started'] = item['started'] if row['started'] is None else min(row['started'], item['started'])
+    out = []
+    for row in grouped.values():
+        states = sorted(row.pop('states'))
+        row['states'] = states
+        bad = [state for state in states if state in BAD_STATES]
+        script_run = row.pop('script_run')
+        row['kind'] = 'platform' if row['platform'] else 'routine' if script_run else 'external'
+        row.update(state=(bad or states or ['unknown'])[0], level='error' if bad else 'running',
+                   cpu=round(row['cpu'], 1), memory_mb=int(row['memory_mb']), pids=sorted(row['pids'])[:PIDS_SHOWN],
+                   started=_iso(row['started']), commandable=row['kind'] == 'routine')
+        out.append(row)
+    return sorted(out, key=lambda row: (row['script'].lower(), row['key']))
+
+
+def view():
+    """Everything the page shows."""
+    from SharedData.Web import workerpool
+    found = rows()
+    shown = [{key: value for key, value in row.items() if not key.startswith('_')} for row in found]
+    totals = dict(routines=len(shown), processes=sum(row['processes'] for row in shown), cpu=round(sum(row['cpu'] for row in shown), 1),
+                  memory_mb=sum(row['memory_mb'] for row in shown), failing=sum(row['level'] == 'error' for row in shown))
+    return dict(worker=workerpool.local_worker().upper(), read_at=_iso(time.time()), viewer=sched.viewer(), rows=shown, totals=totals)
+
+
+def command(key, job):
+    """The worker command for *job* on the routine whose card has this key. LookupError when it
+    no longer runs here, PermissionError for a platform process, ValueError for a wrong job."""
+    from SharedData.Web import workerpool
+    if not isinstance(job, str) or job not in JOBS:
+        raise ValueError('job must be one of: ' + ', '.join(JOBS))
+    found = [row for row in rows() if row['key'] == key] if isinstance(key, str) else []
+    if not found:
+        raise LookupError('that routine no longer runs on this server')
+    row = found[0]
+    if row['kind'] != 'routine' or is_scheduler(row['script']):
+        raise PermissionError('this process is not a routine of the Worker (a platform process, or an installed program): it is not restarted or stopped from here')
+    command = {'sender': 'MASTER', 'job': JOBS[job], 'target': workerpool.local_worker(), 'repo': row['repo'], 'routine': row['routine']}
+    if row['branch'] and row['branch'] != 'nan':
+        command['branch'] = row['branch']
+    if row['_args'] and str(row['_args']) != 'nan':
+        command['args'] = str(row['_args'])
+    return command

{shareddata-6.85.0 → shareddata-6.85.1}/src/SharedData/Web/standalone.py

@@ -31,7 +31,7 @@ def csrf_valid():
     return bool(provided and expected and hmac.compare_digest(provided.encode(), expected.encode()))
 
 
-ADMIN_PAGES = {'users': 'Users', 'data': 'Data', 'routes': 'Routes', 'workers': 'Workers', 'sched': 'Schedules', 'logs': 'Logs',
+ADMIN_PAGES = {'users': 'Users', 'data': 'Data', 'routes': 'Routes', 'workers': 'Workers', 'sched': 'Schedules', 'proc': 'Processes', 'logs': 'Logs',
                'workerpool': 'Worker pool', 'redis': 'Redis', 'kafka': 'Kafka', 'mongo': 'MongoDB'}
 
 
@@ -575,6 +575,36 @@ def init_standalone_routes(app, shdata):
         Logger.log.info('SCHED: %s of %s sent by %s' % (data.get('job'), data.get('routine'), g.get('current_identity')))
         return jsonify(sent={key: value for key, value in command.items() if key != 'args'})   # args may hold a secret
 
+    # PROC: the routines whose processes run on this server, and restart/stop of one.
+    @bp.get('/api/web/proc')
+    def proc_state():
+        from SharedData.Web import proc
+        denied = route_admin()
+        if denied:
+            return denied
+        return jsonify(proc.view())
+
+    @bp.post('/api/web/proc/command')
+    def proc_command():
+        from SharedData.Logger import Logger
+        from SharedData.Routines.WorkerPool import WorkerPool
+        from SharedData.Web import proc
+        denied = route_admin()
+        if denied:
+            return denied
+        data = request.get_json(silent=True) or {}
+        try:
+            command = proc.command(data.get('key'), data.get('job'))
+        except LookupError as exc:
+            return jsonify(error=str(exc)), 404
+        except PermissionError as exc:
+            return jsonify(error=str(exc)), 409
+        except ValueError as exc:
+            return jsonify(error=str(exc)), 400
+        WorkerPool.post_commands(shdata, [dict(command)])
+        Logger.log.info('PROC: %s of %s sent by %s' % (data.get('job'), command['repo'] + '/' + command['routine'], g.get('current_identity')))
+        return jsonify(sent={key: value for key, value in command.items() if key != 'args'})   # args may hold a secret
+
     def _jobs_list(args):
         """What the Jobs tab's addresses have in common: ?month=YYYYMM ?status=A,B ?q=text."""
         return {'month': args.get('month') or None, 'statuses': [s for s in args.get('status', '').split(',') if s], 'text': args.get('q') or None}

{shareddata-6.85.0 → shareddata-6.85.1}/src/SharedData/Web/static/standalone.css

@@ -385,6 +385,19 @@ body.resizing { cursor: col-resize; user-select: none; }
   .sched-fact .hint.inline { flex-basis: 100%; }
 }
 
+/* PROC: a card for each routine running here, in SCHED's colours */
+.sched-main.proc-main { grid-template-columns: .6rem 5.4rem minmax(10rem, 1fr) 4.4rem 5.2rem 2.6rem 3.6rem; }
+.proc-num { color: var(--ink); font-size: .82rem; text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; }
+.proc-cpu[data-hot] { color: #ffd479; font-weight: 600; }
+@media (min-width: 721px) { .proc-row { grid-template-columns: minmax(0, 1fr) 8.2rem; } .proc-row .sched-actions { justify-content: flex-end; } }
+.proc-platform { font-size: .72rem; }
+.proc-procs { color: var(--muted); }
+@media (max-width: 720px) {
+  .sched-main.proc-main { grid-template-columns: .6rem auto minmax(0, 1fr) auto; grid-template-areas: "dot status name up" "dot cpu ram procs"; gap: .1rem .45rem; }
+  .proc-main .sched-dot { grid-area: dot; } .proc-main .sched-status { grid-area: status; } .proc-main .sched-name { grid-area: name; } .proc-main .proc-up { grid-area: up; }
+  .proc-main .proc-cpu { grid-area: cpu; text-align: left; } .proc-main .proc-ram { grid-area: ram; text-align: left; } .proc-main .proc-procs { grid-area: procs; text-align: left; }
+}
+
 /* WORKERS: how each worker is doing, from its heartbeat */
 /* WORKERS, Status: a card for each worker, as the Dashboard's workers tab shows them. */
 .worker-hosts { display: grid; gap: .35rem; }

{shareddata-6.85.0 → shareddata-6.85.1}/src/SharedData/Web/static/standalone.js

@@ -1544,6 +1544,138 @@ if (schedList) {
   if (view.tab === 'jobs') loadJobs();
 }
 
+// PROC: what each routine running on this server uses, read again every 5 seconds; restart and stop of a routine.
+const procList = document.querySelector('#proc-list');
+if (procList) {
+  const KEY = 'shareddata.proc', SORTS = ['name', 'cpu', 'memory', 'started'];
+  const saved = (() => { try { return JSON.parse(localStorage.getItem(KEY)) || {}; } catch (_) { return {}; } })();
+  const view = {q: '', failing: saved.failing === true, sort: SORTS.includes(saved.sort) ? saved.sort : 'name', open: new Set(), filters: saved.filters === true};
+  let data = null, timer;
+  const sent = {};
+  const summary = document.querySelector('#proc-summary'), tools = document.querySelector('#proc-tools'), filter = document.querySelector('#proc-q');
+  const pills = document.querySelector('#proc-levels'), sorting = document.querySelector('#proc-sort'), empty = document.querySelector('#proc-empty');
+  const failed = document.querySelector('#proc-error'), filtersBtn = document.querySelector('#proc-filters');
+  const remember = () => localStorage.setItem(KEY, JSON.stringify({failing: view.failing, sort: view.sort, filters: view.filters}));
+  const clock = iso => iso ? new Date(iso).toLocaleTimeString([], {hour12: false}) : '';
+  const memory = mb => mb >= 1024 ? (mb / 1024).toFixed(1) + ' GB' : mb + ' MB';
+  const up = row => spanText((Date.now() - Date.parse(row.started)) / 1000);
+  function paintFilters() {
+    tools.hidden = !view.filters;
+    filtersButton(filtersBtn, view.filters, (view.q ? 1 : 0) + (view.failing ? 1 : 0) + (view.sort !== 'name' ? 1 : 0));
+  }
+  function openFilters(open) { view.filters = open; remember(); paintFilters(); }
+  filtersBtn.addEventListener('click', () => { openFilters(!view.filters); if (view.filters) filter.focus(); });
+  function matches(row) {
+    if (view.failing && row.level !== 'error') return false;
+    return !view.q || [row.script, row.repo, row.branch, row.args, row.routine].some(text => String(text).toLowerCase().includes(view.q));
+  }
+  const ORDER = {
+    name: (a, b) => a.script.localeCompare(b.script, undefined, {sensitivity: 'base'}),
+    cpu: (a, b) => b.cpu - a.cpu || ORDER.name(a, b),
+    memory: (a, b) => b.memory_mb - a.memory_mb || ORDER.name(a, b),
+    started: (a, b) => Date.parse(b.started) - Date.parse(a.started) || ORDER.name(a, b)};
+  function name(row) {              // 'Repo#branch/folder/' muted, the code's own name strong
+    const cut = row.script.includes('/') ? row.script.lastIndexOf('/') + 1 : 0;
+    return el('span', {class: 'sched-name'}, cut ? el('span', {class: 'sched-path'}, row.script.slice(0, cut)) : null, el('strong', {}, row.script.slice(cut)));
+  }
+  function ask(row, job) {
+    const verb = job === 'restart' ? 'Restart' : 'Stop';
+    const box = el('dialog', {class: 'log-popup sched-confirm'},
+      el('p', {}, el('strong', {}, `${verb} this routine?`)),
+      el('p', {class: 'sched-routine'}, row.script + (row.args ? ' ' + row.args : '')),
+      el('p', {class: 'hint'}, `on ${data.worker}` + (row.processes > 1 ? ` · ${row.processes} processes` : '')),
+      el('p', {class: 'hint'}, job === 'restart' ? 'The worker ends the routine and starts it again with the newest code.'
+        : 'The worker ends the routine. The Scheduler starts a realtime routine again once it has been silent for a while (45 seconds to 5 minutes): to keep it stopped, take it out of the schedule.'),
+      data.viewer ? el('p', {class: 'hint', 'data-viewer': ''}, 'This server is a viewer: the command is recorded here and reaches no worker, so the routine goes on as it is.') : null,
+      el('div', {class: 'actions'},
+        el('button', {type: 'button', autofocus: true, onclick: () => box.close()}, 'Cancel'),
+        el('button', {type: 'button', class: 'danger', 'data-confirm': job, onclick: async event => {
+          event.target.disabled = true;
+          try {
+            await send('POST', '/api/web/proc/command', {key: row.key, job});
+            sent[row.key] = {job, at: Date.now(), started: row.started};
+            result.textContent = `${verb} sent: ${row.script}` + (data.viewer ? ' (on this viewer only: no worker receives it)' : '');
+          } catch (error) { failure(error); }
+          box.close(); paint();
+        }}, verb)));
+    box.addEventListener('click', event => { if (event.target === box) box.close(); });
+    box.addEventListener('close', () => box.remove());
+    document.body.append(box); box.showModal();
+  }
+  function details(row) {
+    const fact = (label, value) => value || value === 0 ? el('div', {class: 'sched-fact'}, el('span', {class: 'hint inline'}, label), el('span', {}, String(value))) : null;
+    const more = row.processes - row.pids.length;
+    return el('div', {class: 'sched-details'},
+      fact('Repository', row.repo + (row.branch ? ' · branch ' + row.branch : '')), fact('Code', row.routine), fact('Arguments', row.args),
+      fact(row.processes > 1 ? 'Processes' : 'Process', row.pids.join(', ') + (more > 0 ? ` and ${more} more` : '')),
+      fact('Threads', row.threads), fact('Open files', row.files), fact('State', row.states.join(', ')), fact('Started', clock(row.started)),
+      row.commandable ? null : el('p', {class: 'hint'}, row.kind === 'platform' ? 'A platform process: its Worker keeps it running, so it is not restarted or stopped from here.'
+        : 'An installed program, not a script the Worker runs: it is listed for what it uses and is not restarted or stopped from here.'),
+      el('div', {class: 'actions'}, el('button', {type: 'button', onclick: async event => {
+        try { await navigator.clipboard.writeText(row.script); event.target.textContent = 'Copied'; } catch (_) { event.target.textContent = row.script; }
+      }}, 'Copy routine name')));
+  }
+  function line(row) {
+    const open = view.open.has(row.key), mark = sent[row.key];
+    const main = el('div', {class: 'sched-main proc-main', role: 'button', tabindex: '0', 'aria-expanded': String(open),
+      onclick: () => { open ? view.open.delete(row.key) : view.open.add(row.key); paint(); },
+      onkeydown: event => { if (event.key === 'Enter' || event.key === ' ') { event.preventDefault(); event.currentTarget.click(); } }},
+      el('span', {class: 'sched-dot', 'aria-hidden': 'true'}),
+      el('span', {class: 'sched-status'}, row.level === 'error' ? row.state.toUpperCase() : 'RUNNING'),
+      name(row),
+      el('span', {class: 'proc-num proc-cpu', title: 'CPU: percent of one core', 'data-hot': row.cpu >= 80 ? '' : null}, row.cpu.toFixed(1) + '%'),
+      el('span', {class: 'proc-num proc-ram', title: 'Memory in use'}, memory(row.memory_mb)),
+      el('span', {class: 'proc-num proc-procs', title: 'Processes'}, row.processes > 1 ? '×' + row.processes : ''),
+      el('span', {class: 'sched-age proc-up', 'data-started': row.started, title: 'Running for'}, up(row)));
+    return el('div', {class: 'sched-row proc-row', 'data-level': row.level, 'data-key': row.key}, main,
+      el('span', {class: 'sched-actions'}, mark ? el('span', {class: 'sched-sent'}, `${mark.job} sent ${clock(new Date(mark.at).toISOString())}`) : null,
+        ...(row.commandable ? [el('button', {type: 'button', 'data-job': 'restart', onclick: () => ask(row, 'restart')}, 'Restart'),
+                               el('button', {type: 'button', 'data-job': 'stop', onclick: () => ask(row, 'stop')}, 'Stop')] : [el('span', {class: 'hint proc-platform'}, row.kind)])),
+      open ? details(row) : null);
+  }
+  function paint() {
+    if (!data) return;
+    const rows = data.rows, totals = data.totals, bad = rows.filter(row => row.level === 'error').length;
+    summary.textContent = `${data.worker} · ${totals.routines} ${totals.routines === 1 ? 'routine' : 'routines'} · ${totals.processes} ${totals.processes === 1 ? 'process' : 'processes'} · CPU ${totals.cpu.toFixed(0)}% · ${memory(totals.memory_mb)}` + (bad ? ` · ${bad} failing` : '');
+    pills.replaceChildren(...(bad || view.failing ? [el('button', {type: 'button', class: 'pill sched-level', 'data-level': 'error', 'aria-pressed': String(view.failing),
+      onclick: () => { view.failing = !view.failing; remember(); paint(); }}, `Failing ${bad}`)] : []));
+    paintFilters();
+    sorting.value = view.sort;
+    const shown = rows.filter(matches).sort(ORDER[view.sort]);
+    procList.replaceChildren(...shown.map(line));
+    empty.hidden = shown.length > 0;
+    empty.textContent = !rows.length ? 'No routine is running on this server.' : 'No routine matches the filter.';
+  }
+  function times() {              // once a second: uptimes go on counting between two readings
+    for (const node of procList.querySelectorAll('.proc-up[data-started]')) node.textContent = spanText((Date.now() - Date.parse(node.dataset.started)) / 1000);
+  }
+  async function load() {
+    clearTimeout(timer);
+    try {
+      const next = await call('/api/web/proc');
+      data = next; failed.hidden = true;
+      for (const [key, mark] of Object.entries(sent)) {       // a mark stays until the routine has started again, or a minute
+        const row = data.rows.find(candidate => candidate.key === key);
+        if (!row || row.started !== mark.started || Date.now() - mark.at > 60000) delete sent[key];
+      }
+      if (!String(getSelection()) || !procList.contains(getSelection().anchorNode)) paint();
+    } catch (error) { failed.hidden = false; failed.textContent = 'Cannot read the processes: ' + error.message; }
+    if (!document.hidden) timer = setTimeout(load, 5000);
+  }
+  filter.addEventListener('input', () => { view.q = filter.value.trim().toLowerCase(); paint(); });
+  sorting.addEventListener('change', () => { view.sort = sorting.value; remember(); paint(); });
+  document.addEventListener('keydown', event => {
+    if (event.ctrlKey || event.metaKey || event.altKey || document.querySelector('dialog[open]')) return;
+    if (/^(INPUT|TEXTAREA|SELECT)$/.test(event.target.tagName)) { if (event.key === 'Escape') event.target.blur(); return; }
+    if (event.key === '/') { event.preventDefault(); openFilters(true); filter.focus(); }
+    else if (event.key.toLowerCase() === 'f') pills.querySelector('[data-level=error]')?.click();
+  });
+  document.addEventListener('visibilitychange', () => { if (!document.hidden) load(); });
+  setInterval(times, 1000);
+  paintFilters();
+  load();
+}
+
 // Shared by WORKERPOOL and REDIS: a node's children without the ones that are not there, and
 // sizes and spans as a person reads them.
 function fill(node, ...children) { node.replaceChildren(...children.filter(child => child != null)); }

{shareddata-6.85.0 → shareddata-6.85.1}/src/SharedData/Web/templates/standalone.html

@@ -94,6 +94,13 @@
 <p><button id="jobs-more" type="button" hidden>Show more</button></p>
 </div>
 </section>{% endif %}
+{% if section == 'proc' %}<section id="page-proc" class="page tight sched proc">
+<div class="head"><h2>Processes</h2><button id="proc-filters" class="filters-toggle" type="button" aria-expanded="false">Filters</button><span id="proc-summary" class="hint" role="status"></span></div>
+<div id="proc-tools" class="toolbar sched-toolbar" hidden><input id="proc-q" type="search" placeholder="Filter: routine, repository, branch or arguments" aria-label="Filter routines"><span id="proc-levels" class="pills" role="group" aria-label="States shown"></span><select id="proc-sort" aria-label="Sort by"><option value="name">By name</option><option value="cpu">By CPU</option><option value="memory">By memory</option><option value="started">Newest first</option></select></div>
+<p id="proc-error" class="sched-stale" role="alert" hidden></p>
+<div id="proc-list" class="sched-list" aria-live="off"></div>
+<p id="proc-empty" class="hint" hidden></p>
+</section>{% endif %}
 {% if section == 'workerpool' %}<section id="page-workerpool" class="page tight sched workerpool">
 <div class="head"><h2>Worker pool</h2><span id="wp-count" class="hint" role="status"></span><button id="wp-repos-btn" type="button" aria-expanded="false" aria-controls="wp-repos">Repositories</button><button id="wp-procs-btn" type="button" aria-expanded="false" aria-controls="wp-procs">Processes</button></div>
 <div id="wp-workers" class="wp-workers" role="listbox" aria-label="Where the commands go"></div>

{shareddata-6.85.0 → shareddata-6.85.1/src/shareddata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.0
+Version: 6.85.1
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata

{shareddata-6.85.0 → shareddata-6.85.1}/src/shareddata.egg-info/SOURCES.txt

@@ -132,6 +132,7 @@ src/SharedData/Web/containers.py
 src/SharedData/Web/identity.py
 src/SharedData/Web/kafka_view.py
 src/SharedData/Web/mongo_view.py
+src/SharedData/Web/proc.py
 src/SharedData/Web/redis_view.py
 src/SharedData/Web/route_health.py
 src/SharedData/Web/route_registry.py