shareddata 6.85.14__tar.gz → 6.85.15__tar.gz

{shareddata-6.85.14/src/shareddata.egg-info → shareddata-6.85.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.85.14
+Version: 6.85.15
 Summary: Memory Mapped / Shared Memory Database with S3 repository
 Home-page: https://shareddata.net
 Author: shareddata

{shareddata-6.85.14 → shareddata-6.85.15}/setup.py

@@ -66,7 +66,7 @@ test_requires = [
 
 setup(
     name='shareddata',
-    version='6.85.14',
+    version='6.85.15',
     description='Memory Mapped / Shared Memory Database with S3 repository',
     author='shareddata',
     author_email='admin@shareddata.net',

{shareddata-6.85.14 → shareddata-6.85.15}/src/SharedData/Routines/JobStore.py

@@ -285,7 +285,7 @@ def active(now=None):
 
 
 # What a list of jobs can be grouped by: the columns (SQL) that make a group, by name.
-GROUPS = {'time': ('minute', 'repo', 'branch', 'routine'), 'routine': ('repo', 'branch', 'routine'), 'status': ('status',), 'computer': ('computer',)}
+GROUPS = {'time': ('minute', 'repo', 'branch', 'routine'), 'routine': ('repo', 'branch', 'routine'), 'repo': ('repo',), 'status': ('status',), 'computer': ('computer',)}
 _GROUP_SQL = {'minute': 'substr(date, 1, 16)', 'repo': "COALESCE(repo, '')", 'branch': "COALESCE(branch, '')", 'routine': "COALESCE(routine, '')",
               'status': 'status', 'computer': "COALESCE(target, computer, '')"}
 
@@ -333,7 +333,7 @@ def find(month=None, statuses=None, text=None, limit=200, offset=0, now=None, gr
 
 def groups(by='time', month=None, statuses=None, text=None, limit=500, now=None):
     """The groups of a list without their jobs: how many jobs of each status every group has.
-    by: 'time' (the minute a job is due, with its routine), 'routine', 'status' or 'computer'.
+    by: 'time' (the minute a job is due, with its routine), 'routine', 'repo', 'status' or 'computer'.
     -> {'groups': [{'key': {...}, 'counts': {status: n}, 'total': n}], 'groups_total': n,
         'total': n, 'counts': {status: n}, 'months': [...]}; oldest minute first, then by name."""
     if by not in GROUPS:

{shareddata-6.85.14 → shareddata-6.85.15}/src/SharedData/Routines/WorkerLib.py

@@ -859,6 +859,31 @@ def install_repo(command, quiet=False):
             installed_repo['isinstalling'] = False
 
 
+def _git_text(args, timeout=30):
+    """stdout of a git command, or None when it failed or timed out (a missing or unreachable server)."""
+    try:
+        done = subprocess.run(['git', *args], capture_output=True, text=True, timeout=timeout)
+    except (OSError, subprocess.SubprocessError):
+        return None
+    return done.stdout.strip() if done.returncode == 0 else None
+
+
+def _repo_is_current(repo_path, url, ref):
+    """True when the clone is on `ref` at exactly the commit the server has for it.
+
+    One `git ls-remote` (a reference list, no objects) replaces the checkout and the pull of every start:
+    code is exactly as new as before, and nothing runs, logs or changes when nothing was pushed. Anything
+    that cannot be proven (server unreachable, branch gone, other branch checked out) is not 'current'
+    and takes the full checkout + pull path, with its retries and its fallback to the installed copy.
+    """
+    remote = _git_text(['ls-remote', url, 'refs/heads/' + ref])
+    if not remote:
+        return False
+    remote_sha = remote.split()[0]
+    return (_git_text(['-C', str(repo_path), 'rev-parse', 'HEAD']) == remote_sha
+            and _git_text(['-C', str(repo_path), 'rev-parse', '--abbrev-ref', 'HEAD']) == ref)
+
+
 def _install_repo(command, quiet=False):
 
     """
@@ -901,7 +926,19 @@ def _install_repo(command, quiet=False):
             os.environ['GIT_ACRONYM']+'/'+command['repo']
 
         # GIT PULL OR GIT CLONE
-        if repo_exists:
+        current = (repo_exists and venv_exists and requirements_path.is_file()
+                   and _repo_is_current(repo_path, GIT_URL, command['branch'] if hasbranch else 'main'))
+        if repo_exists and current:
+            Logger.log.debug('Repo %s is up to date' % (command['repo']))
+            pulled = True                           # the code is as new as the server's: the requirements check below still applies
+            runroutine = True
+            install_requirements = False
+            if requirements_path.is_file():
+                venv_root = repo_path / "venv"
+                new_hash = hash_file(requirements_path)
+                stored_hash = load_last_hash(venv_root)
+                install_requirements = (new_hash is None or stored_hash is None or new_hash != stored_hash)
+        elif repo_exists:
             if not quiet:
                 Logger.log.info('Pulling repo %s' % (command['repo']))
             requirements_lastmod = 0

shareddata-6.85.15/src/SharedData/Web/mongo_view.py

@@ -0,0 +1,399 @@
+"""The MongoDB this server uses, as the MONGO page shows it: every server it is told of (and,
+behind a router, every router of the cluster), the members of its replica set or the shards
+behind its routers, its databases, and what is wrong.
+
+Read-only commands, on connections of the page's own with short limits. Each server is asked
+directly and at the same time (one connection to that server only), so the numbers of a row are
+that server's, and a server that is down is reported instead of waited for.
+"""
+import hashlib
+import ipaddress
+import re
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, TimeoutError as Late, wait as wait_for
+from datetime import timezone
+
+WAIT = 7
+BEHIND = 60             # seconds a secondary may be behind before it is pointed out
+SERVER_SECONDS = 2.5    # a server that takes longer is reported as not answering
+FORGOTTEN = 3600        # a router the cluster has not heard of for longer is taken as retired
+PORT = 27017
+MAX_PENDING = 64        # includes running and queued work across all page requests
+MAX_SERVERS = 64        # bound cluster-controlled discovery and candidate scans
+
+
+def configured():
+    return bool(_env().get('MONGODB_HOST'))
+
+
+def _env():
+    from SharedData.Web import services
+    return services.values('mongodb')
+
+
+def _seeds(env):
+    """host:port of every server MONGODB_HOST names, in its order."""
+    seeds = []
+    for part in (env.get('MONGODB_HOST') or '').split(','):
+        part = part.strip()
+        if not part:
+            continue
+        host, sep, port = part.rpartition(':')
+        address = part if sep and host and port.isdigit() else '%s:%d' % (part, PORT)
+        if address not in seeds:
+            seeds.append(address)
+    return seeds
+
+
+def _server_uri(env, address):
+    """The address of one server alone. Without replicaSet: with it the driver finds the set
+    again and sends the commands to its primary, whichever server was asked for."""
+    from SharedData.Web.services import mongo_uri
+    one = {name: value for name, value in env.items() if name != 'MONGODB_REPLICA_SET'}
+    one['MONGODB_HOST'] = address
+    uri = mongo_uri(one)
+    return uri + ('&' if '?' in uri else '?') + 'directConnection=true'
+
+
+def _named(host, address):
+    """name:port as a server gives its own name (serverStatus says the name without the port)."""
+    if not host:
+        return ''
+    return host if ':' in host else '%s:%s' % (host, address.rpartition(':')[2])
+
+
+def _ask(address, env):
+    from pymongo import timeout
+    with timeout(SERVER_SECONDS):  # one budget for selection, hello and serverStatus together
+        return _ask_server(address, env)
+
+
+def _ask_server(address, env):
+    """What one server says of itself, or why it does not answer. -> {'address', 'host', 'role',
+    'set', 'answers', 'error', 'seconds', 'version', 'uptime', 'connections', 'connections_free',
+    'memory_mb', 'operations'}"""
+    from pymongo import MongoClient
+    started = time.time()
+    out = {'address': address, 'host': '', 'role': '', 'set': None, 'answers': False, 'error': None, 'seconds': None, 'version': None, 'uptime': None,
+           'connections': None, 'connections_free': None, 'memory_mb': None, 'operations': {}}
+    limit = int(SERVER_SECONDS * 1000)
+    try:
+        client = MongoClient(_server_uri(env, address), serverSelectionTimeoutMS=limit, connectTimeoutMS=limit, socketTimeoutMS=limit)
+        try:
+            hello = client.admin.command('hello')
+            out.update(answers=True, set=hello.get('setName'),
+                       role='router' if hello.get('msg') == 'isdbgrid' else 'primary' if hello.get('setName') and hello.get('isWritablePrimary')
+                       else 'secondary' if hello.get('secondary') else 'arbiter' if hello.get('arbiterOnly') else 'member' if hello.get('setName') else 'single')
+            try:
+                status = client.admin.command('serverStatus')
+                out.update(host=_named(status.get('host'), address), version=status.get('version'), uptime=status.get('uptime'),
+                           connections=status.get('connections', {}).get('current'), connections_free=status.get('connections', {}).get('available'),
+                           memory_mb=status.get('mem', {}).get('resident'),
+                           operations={name: status.get('opcounters', {}).get(name) for name in ('insert', 'query', 'update', 'delete', 'command')})
+            except Exception as exc:                    # it answers, but this user may not read its numbers
+                out['error'] = 'its numbers cannot be read (%s: %s)' % (type(exc).__name__, str(exc)[:120])
+        finally:
+            client.close()
+    except Exception as exc:
+        out['error'] = '%s: %s' % (type(exc).__name__, str(exc).split(' (configured timeouts')[0].split(', Timeout:')[0][:160])
+    out['seconds'] = round(time.time() - started, 3)
+    return out
+
+
+def _locate(name, candidates, env):
+    """Router names are labels, never network destinations. Probe only the validated cluster
+    addresses passed by _servers and require the server's own identity and router role."""
+    from pymongo import timeout
+    started = time.monotonic()
+    with timeout(SERVER_SECONDS):
+        for address in candidates[:MAX_SERVERS]:
+            if time.monotonic() - started >= SERVER_SECONDS:
+                break
+            reply = _ask(address, env)
+            if reply.get('answers') and reply.get('host') == name and reply.get('role') == 'router':
+                return reply
+    return {'address': '', 'host': name, 'role': 'router', 'set': None, 'answers': False, 'seconds': round(time.monotonic() - started, 3), 'version': None, 'uptime': None,
+            'connections': None, 'connections_free': None, 'memory_mb': None, 'operations': {},
+            'error': 'no approved address of the cluster answers as this router'}
+
+
+def _candidates(name, hosts, seeds):
+    # config.mongos is writable metadata: it cannot choose a port, URI, DNS target or IP.
+    # getShardMap comes from the authenticated cluster command, not from config.mongos.
+    # Auto-discovery is restricted to numeric private unicast addresses and configured ports.
+    if not isinstance(name, str) or not re.fullmatch(r'[A-Za-z0-9_.-]{1,253}:[0-9]{1,5}', name):
+        return ()
+    port = name.rpartition(':')[2]
+    if port not in {seed.rpartition(':')[2] for seed in seeds} or not 0 < int(port) < 65536:
+        return ()
+    addresses = []
+    for host in hosts[:MAX_SERVERS]:
+        try:
+            ip = ipaddress.ip_address(host)
+        except ValueError:
+            continue
+        if (ip.version != 4 or not ip.is_private or ip.is_loopback or ip.is_link_local
+                or ip.is_multicast or ip.is_unspecified or ip.is_reserved):
+            continue
+        address = '%s:%s' % (ip, port)
+        if address not in addresses and address not in seeds:
+            addresses.append(address)
+    return tuple(addresses)
+
+
+def _identity(env):
+    # Separate in-flight work and cached identities when credentials/settings change.
+    return hashlib.sha256(repr(sorted(env.items())).encode()).hexdigest()
+
+
+_pool = ThreadPoolExecutor(max_workers=16, thread_name_prefix='mongo-view')
+_pending = {}           # what is asked -> a question still without an answer (never asked twice at once)
+_pending_lock = threading.Lock()
+_names = {}             # address -> the name it last answered as, so a server that stops answering keeps its name
+_located = {}           # a router's name that does not resolve here -> the address that answered as it
+
+
+def _submit(questions):
+    """{key: (function, args)} -> {key: future}, a question already on its way not asked again."""
+    with _pending_lock:
+        for key, future in list(_pending.items()):
+            if future.done():
+                _pending.pop(key, None)
+        asked = {}
+        for key, (function, args) in questions.items():
+            future = _pending.get(key)
+            if future is None and len(_pending) < MAX_PENDING:
+                future = _pool.submit(function, *args)
+                _pending[key] = future
+            asked[key] = future   # saturated: an explicit no-answer, no new task
+    return asked
+
+
+def _collect(asked, until):
+    """The replies in by *until* (time.monotonic()); a question not answered by then is reported as
+    such and left to end by itself. -> {key: reply}"""
+    wait_for([future for future in asked.values() if future is not None], timeout=max(until - time.monotonic(), 0))
+    replies = {}
+    with _pending_lock:
+        for key, future in asked.items():
+            if future is not None and future.done():
+                if _pending.get(key) is future:
+                    _pending.pop(key, None)
+                try:
+                    replies[key] = future.result()
+                except Exception as exc:
+                    replies[key] = {'answers': False, 'error': '%s: %s' % (type(exc).__name__, str(exc)[:160])}
+            else:
+                replies[key] = {'answers': False, 'error': 'no answer in %g seconds' % SERVER_SECONDS}
+    return replies
+
+
+def _read(seconds):
+    from pymongo import timeout
+    with timeout(seconds):  # all cluster commands and dbStats share this deadline
+        return _read_cluster(seconds)
+
+
+def _read_cluster(seconds):
+    """What the cluster says of itself, through the connection SharedData uses."""
+    from pymongo import MongoClient
+    from SharedData.Web.services import mongo_uri
+    limit = seconds * 1000
+    client = MongoClient(mongo_uri(_env()), serverSelectionTimeoutMS=limit, connectTimeoutMS=limit, socketTimeoutMS=limit)
+    try:
+        hello = client.admin.command('hello')
+        out = {'kind': 'sharded' if hello.get('msg') == 'isdbgrid' else 'replica set' if hello.get('setName') else 'single', 'set': hello.get('setName'),
+               'members': [], 'shards': [], 'routers': [], 'cluster_hosts': []}
+        if out['kind'] == 'replica set':
+            try:
+                members = client.admin.command('replSetGetStatus').get('members', [])
+                primary = next((member for member in members if member.get('stateStr') == 'PRIMARY'), None)
+                for member in members:
+                    lag = None
+                    if primary and member.get('stateStr') == 'SECONDARY' and member.get('optimeDate') and primary.get('optimeDate'):
+                        lag = max((primary['optimeDate'] - member['optimeDate']).total_seconds(), 0)
+                    out['members'].append({'name': member.get('name'), 'state': member.get('stateStr'), 'healthy': member.get('health') == 1, 'lag': lag,
+                                           'uptime': member.get('uptime'), 'copies_from': member.get('syncSourceHost') or '', 'message': member.get('lastHeartbeatMessage') or ''})
+            except Exception:                           # this user may not read the set's status: what the server says of it
+                out['members'] = [{'name': name, 'state': 'PRIMARY' if name == hello.get('primary') else 'SECONDARY' if name in hello.get('hosts', []) else 'ARBITER',
+                                   'healthy': True, 'lag': None, 'uptime': None, 'copies_from': '', 'message': ''}
+                                  for name in list(hello.get('hosts', [])) + list(hello.get('arbiters', []))]
+                out['limited'] = True
+        elif out['kind'] == 'sharded':
+            out['shards'] = [{'name': shard.get('_id'), 'servers': shard.get('host'), 'ready': shard.get('state', 1) == 1} for shard in client.admin.command('listShards').get('shards', [])]
+            try:                                        # every router of the cluster, as the routers report themselves
+                now = time.time()                       # the driver gives the ping in UTC, without its zone
+                out['routers'] = [{'name': router['_id'], 'heard': max(now - router['ping'].replace(tzinfo=timezone.utc).timestamp(), 0) if router.get('ping') else None,
+                                   'version': router.get('mongoVersion')} for router in client.config.mongos.find({}, {'_id': 1, 'ping': 1, 'mongoVersion': 1}).limit(MAX_SERVERS)]
+            except Exception:
+                out['routers'] = []
+            try:                                        # where the cluster's servers are, for a router whose name does not resolve here
+                hosts = set()
+                for servers in client.admin.command('getShardMap').get('map', {}).values():
+                    for address in servers.rpartition('/')[2].split(','):
+                        if address:
+                            hosts.add(address.rpartition(':')[0])
+                out['cluster_hosts'] = sorted(hosts)
+            except Exception:
+                out['cluster_hosts'] = sorted({address.rpartition(':')[0] for shard in out['shards'] for address in str(shard['servers'] or '').rpartition('/')[2].split(',') if address})
+        databases = []
+        for database in client.admin.command('listDatabases').get('databases', []):
+            row = {'name': database['name'], 'size': database.get('sizeOnDisk'), 'collections': None, 'documents': None, 'indexes': None}
+            if len(databases) < 40:
+                try:
+                    stats = client[database['name']].command('dbStats')
+                    row.update(collections=stats.get('collections'), documents=stats.get('objects'), indexes=stats.get('indexes'), data=stats.get('dataSize'), index_size=stats.get('indexSize'))
+                except Exception:
+                    pass
+            databases.append(row)
+        out['databases'] = sorted(databases, key=lambda row: -(row['size'] or 0))
+        return out
+    finally:
+        client.close()
+
+
+def _row(reply, address, host, listed, heard):
+    row = {'address': address, 'host': host, 'role': '', 'set': None, 'answers': False, 'error': None, 'seconds': None, 'version': None, 'uptime': None,
+           'connections': None, 'connections_free': None, 'memory_mb': None, 'operations': {}}
+    row.update({name: value for name, value in reply.items() if name in row})
+    row['address'], row['host'] = row['address'] or address, row['host'] or host
+    row.update(listed=listed, heard=None if heard is None else round(heard))
+    return row
+
+
+def _servers(env, seeds, asked, found, started):
+    """One row per server: those MONGODB_HOST names, then the cluster's other routers. A router
+    the cluster has not heard of for an hour is left out (config.mongos keeps every router that
+    ever ran). Unlocated names may belong to unnamed down seeds, so they are diagnostics
+    rather than extra server counts in that case. -> (rows, retired count, unlocated names)"""
+    replies = _collect(asked, started + SERVER_SECONDS + 1.5)
+    identity = _identity(env)
+    routers = {router['name']: router for router in (found or {}).get('routers', [])[:MAX_SERVERS] if isinstance(router.get('name'), str)}
+    rows = []
+    for address in seeds:
+        reply = replies[(identity, address)]
+        host = reply.get('host') or _names.get((identity, address), '')
+        if reply.get('answers') and reply.get('host'):
+            _names[(identity, address)] = reply['host']
+        row = _row(reply, address, host, 'configured', routers.get(host, {}).get('heard'))
+        same = next((i for i, other in enumerate(rows) if host and other['host'] == host), None)
+        if same is None:
+            rows.append(row)
+        elif row['answers'] and not rows[same]['answers']:
+            rows[same] = row
+    known = {row['host'] for row in rows if row['host']}
+    others = [router for name, router in sorted(routers.items()) if name not in known]
+    left_out = [router for router in others if router['heard'] is not None and router['heard'] > FORGOTTEN]
+    others = [router for router in others if router not in left_out]
+    unlocated = []
+    if others:
+        hosts = (found or {}).get('cluster_hosts', [])
+        questions = {}
+        for router in others:
+            candidates = list(_candidates(router['name'], hosts, seeds))
+            located_key = (identity, router['name'])
+            if _located.get(located_key) in candidates:          # where it answered the last time first
+                candidates.remove(_located[located_key])
+                candidates.insert(0, _located[located_key])
+            questions[(identity, 'router|' + router['name'])] = (_locate, (router['name'], tuple(candidates), env))
+        replies = _collect(_submit(questions), started + WAIT - 0.25)
+        for router in others:
+            reply = replies[(identity, 'router|' + router['name'])]
+            if not reply.get('answers') and any(not row['host'] for row in rows):
+                unlocated.append(router['name'])
+                continue   # identity unknown: do not count a down seed twice or invent an address
+            if reply.get('answers') and reply.get('address') and reply['address'] != router['name']:
+                _located[(identity, router['name'])] = reply['address']
+            rows.append(_row(dict(reply, host=router['name'], role=reply.get('role') or 'router'), reply.get('address') or _located.get((identity, router['name']), ''),
+                             router['name'], 'cluster', router['heard']))
+    return rows, len(left_out), unlocated
+
+
+def view():
+    """-> {'configured', 'state', 'kind', 'summary', 'members', 'shards', 'servers', 'databases', 'problems', 'seconds'}
+    'servers' is one row per server, as that server answers for itself: {'address' (where it is
+    asked), 'host' (the name it gives), 'role', 'set', 'answers', 'error', 'seconds', 'version',
+    'uptime', 'connections', 'connections_free', 'memory_mb', 'operations', 'listed'
+    ('configured': MONGODB_HOST names it, 'cluster': a router the cluster lists), 'heard'
+    (seconds since the cluster last heard of that router)}."""
+    started = time.monotonic()
+    empty = {'kind': '', 'summary': {}, 'members': [], 'shards': [], 'servers': [], 'databases': [], 'problems': []}
+    if not configured():
+        return dict(empty, configured=False, state='none', seconds=0)
+    env = _env()
+    seeds = _seeds(env)
+    identity = _identity(env)
+    cluster_key = (identity, 'cluster')
+    cluster = _submit({cluster_key: (_read, (WAIT - 3,))})[cluster_key]
+    asked = _submit({(identity, address): (_ask, (address, env)) for address in seeds})
+    try:
+        if cluster is None:
+            raise Late()
+        found = cluster.result(timeout=max(0, started + WAIT - 3 - time.monotonic()))
+    except Late:
+        why = 'no answer in %g seconds' % (WAIT - 3)
+        found = None
+    except Exception as exc:
+        why, found = '%s: %s' % (type(exc).__name__, str(exc).split(', Timeout:')[0][:200]), None
+    finally:
+        with _pending_lock:
+            if cluster is not None and cluster.done() and _pending.get(cluster_key) is cluster:
+                _pending.pop(cluster_key, None)
+    servers, left_out, unlocated = _servers(env, seeds, asked, found, started)
+    answering = [row for row in servers if row['answers']]
+    counts = {'addresses': env.get('MONGODB_HOST'), 'servers': len(servers), 'answering': len(answering),
+              'versions': sorted({row['version'] for row in answering if row['version']})}
+    if found is None:
+        return dict(empty, configured=True, state='down', seconds=round(time.monotonic() - started, 3), summary=counts, servers=servers,
+                    problems=[{'level': 'error', 'text': 'MongoDB does not answer at %s (%s)' % (env.get('MONGODB_HOST'), why)}])
+    problems = []
+    if found['kind'] == 'replica set':
+        if not any(member['state'] == 'PRIMARY' for member in found['members']):
+            problems.append({'level': 'error', 'text': 'the replica set has no primary: nothing can be written'})
+        for member in found['members']:
+            if not member['healthy'] or member['state'] not in ('PRIMARY', 'SECONDARY', 'ARBITER'):
+                problems.append({'level': 'warning', 'text': '%s is %s%s' % (member['name'], (member['state'] or 'not healthy').lower(), ': ' + member['message'] if member['message'] else '')})
+            elif member['lag'] is not None and member['lag'] > BEHIND:
+                problems.append({'level': 'warning', 'text': '%s is %d seconds behind the primary' % (member['name'], member['lag'])})
+        holding = [member for member in found['members'] if member['state'] in ('PRIMARY', 'SECONDARY')]
+        if len(holding) == 1 and not found.get('limited'):
+            problems.append({'level': 'note', 'text': 'the data is on one member only (%s): the set has no second copy' % holding[0]['name']})
+    for shard in found['shards']:
+        if not shard['ready']:
+            problems.append({'level': 'error', 'text': 'shard %s is not ready' % shard['name']})
+    for row in servers:
+        name = row['host'] or row['address']
+        at = ' at ' + row['address'] if row['host'] and row['address'] and row['address'] != row['host'] else ''
+        if not row['answers']:
+            problems.append({'level': 'error', 'text': '%s%s does not answer (%s)' % (name, at, row['error'])})
+        elif row['error']:
+            problems.append({'level': 'warning', 'text': '%s answers, but %s' % (name, row['error'])})
+        free = row['connections_free']
+        if free is not None and row['connections'] and free < row['connections'] * 0.1:
+            problems.append({'level': 'warning', 'text': '%s has few connections left: %d in use, %d free' % (name, row['connections'], free)})
+    if left_out:
+        problems.append({'level': 'note', 'text': '%d %s the cluster lists %s not been heard of for over an hour: taken as retired and not shown'
+                         % (left_out, 'router' if left_out == 1 else 'routers', 'has' if left_out == 1 else 'have')})
+    if unlocated:
+        problems.append({'level': 'warning', 'text': 'Router identities could not be located: %s. They may be the unnamed servers that do not answer; not counted twice.' % ', '.join(unlocated)})
+    state = 'degraded' if any(problem['level'] == 'error' for problem in problems) else 'ok'
+    user = [row for row in found['databases'] if row['name'] not in ('admin', 'local', 'config')]
+    summary = dict(counts, set=found.get('set'), members=len(found['members']), shards=len(found['shards']), databases=len(user),
+                   size=sum(row['size'] or 0 for row in found['databases']), collections=sum(row['collections'] or 0 for row in user),
+                   version=', '.join(counts['versions']) or None)
+    return {'configured': True, 'state': state, 'kind': found['kind'], 'summary': summary, 'members': found['members'], 'shards': found['shards'], 'servers': servers,
+            'databases': found['databases'], 'problems': problems, 'limited': bool(found.get('limited')), 'seconds': round(time.monotonic() - started, 3)}
+
+
+def health():
+    found = view()
+    if not found['configured']:
+        return None
+    summary = found['summary']
+    worst = next((problem['text'] for problem in found['problems'] if problem['level'] in ('error', 'warning')), '')
+    state = found['state'] if found['state'] != 'ok' or not any(problem['level'] == 'warning' for problem in found['problems']) else 'warning'
+    text = ('does not answer' if found['state'] == 'down' else
+            '%d of %d servers answer' % (summary['answering'], summary['servers']) if summary['answering'] < summary['servers'] else
+            '%d members' % summary['members'] if found['kind'] == 'replica set' else '%d shards' % summary['shards'] if found['kind'] == 'sharded' else 'one server')
+    return {'name': 'MongoDB', 'state': state, 'text': text + (' · ' + worst if worst and found['state'] != 'down' else '')}

{shareddata-6.85.14 → shareddata-6.85.15}/src/SharedData/Web/proc.py

@@ -3,10 +3,18 @@ a master may send from it (restart, stop).
 
 A routine is one line however many processes it has (a routine with several workers is one
 card). Its numbers are those of all its processes together. Platform processes (the Worker, the
-Scheduler, this server) are shown but are not restarted or stopped from here: their Worker keeps
-them running, and stopping the server would stop the page. So are processes that are not a script
-the Worker runs (an installed program such as vllm or rtxclaw): a restart would try to start that
-file as a Python script.
+Scheduler, this server) are restarted and stopped like the others, with one exception: the
+Scheduler is not commanded from here, because its Worker starts it again the moment it ends — a
+stop of it would only be a delayed restart. Processes that are not a script the Worker runs (an
+installed program such as vllm or rtxclaw) are not commanded either: a restart would try to
+start that file as a Python script.
+
+A restart is the Worker's own restart job: it ends the process and starts it again (the
+Worker's own row restarts with the reset job). A stop is its kill job: the Worker ends the
+process and does not start it again itself. What else may start it again (the Scheduler, for a
+realtime routine of a schedule kept here) and what goes with it (this page, the API every host
+uses, every process of the Worker's service) is what the page's confirmation says, row by row:
+a stop is never silently undone.
 """
 import hashlib
 import os
@@ -23,6 +31,10 @@ PIDS_SHOWN = 20          # process ids listed on a card
 SAMPLE_FRESH = 30        # seconds a CPU reading can serve as the start of the next one
 SAMPLE_WAIT = 0.25       # seconds the first reading of a process is taken over
 BAD_STATES = ('zombie', 'dead', 'stopped', 'stop')
+WORKER = 'SharedData.Routines.Worker'      # the Worker program itself, when it is among the routines
+WEB = 'SharedData.Web.server'              # the server that serves this page, when it runs as a routine
+API = 'SharedData.API.ServerGunicorn'       # the SharedData API of this server, which every host and routine of it uses
+SUPERVISED_KEPT = 10                       # seconds the schedule's realtime scripts are kept between readings
 # What the page asks for -> the worker's job (its own 'stop' does nothing).
 JOBS = {job: sched.JOBS[job] for job in ('restart', 'stop')}
 
@@ -92,6 +104,49 @@ def _routines():
     return found
 
 
+def _worker_pid():
+    """The Worker that owns this WebUI, not another Worker's process on the same host.
+    A reset is addressed to local_worker(), so it must never be offered on another row."""
+    try:
+        for parent in psutil.Process().parents():
+            words = parent.cmdline()
+            if any(words[i:i + 2] == ['-m', WORKER] for i in range(len(words) - 1)):
+                return parent.pid
+    except psutil.Error:
+        pass
+    return None
+
+
+_supervised = (0.0, frozenset())           # (when, scripts the Scheduler keeps running here)
+
+
+def _supervised_scripts():
+    """The scripts the Scheduler of this server keeps running: the realtime routines of the
+    schedules kept here, for this host. A killed one is started again once it has been silent,
+    so a stop of it is not durable. The set is kept a few seconds: the page asks every few."""
+    global _supervised
+    when, found = _supervised
+    if time.time() - when < SUPERVISED_KEPT:
+        return found
+    from SharedData.Web import workerpool
+    host = str(workerpool.local_worker() or '').upper()
+    scripts = set()
+    try:
+        from SharedData.Routines import ScheduleStore
+        day = datetime.now().strftime('%Y%m%d')
+        for name in ScheduleStore.names():
+            for record in ScheduleStore.state(name, day).to_dict('records'):
+                if record.get('isrealtime') and str(record.get('computer') or '').upper() == host:
+                    script = str(record.get('script') or '')
+                    if script:
+                        scripts.add(script)
+    except Exception:                      # unknown is not evidence that a stop will hold
+        _supervised = (time.time(), None)
+        return None
+    _supervised = (time.time(), frozenset(scripts))
+    return _supervised[1]
+
+
 def rows():
     """The routines running here, one dict each. '_args' and '_branch' are the real values and
     never leave this module (an argument can be a secret)."""
@@ -127,15 +182,23 @@ def rows():
         row['states'].add(item['state'])
         row['started'] = item['started'] if row['started'] is None else min(row['started'], item['started'])
     out = []
+    supervised = _supervised_scripts()
+    worker_pid = _worker_pid()
     for row in grouped.values():
         states = sorted(row.pop('states'))
         row['states'] = states
         bad = [state for state in states if state in BAD_STATES]
         script_run = row.pop('script_run')
         row['kind'] = 'platform' if row['platform'] else 'routine' if script_run else 'external'
+        row['role'] = ('' if row['kind'] != 'platform' else 'scheduler' if is_scheduler(row['script'])
+                       else 'worker' if row['script'] == WORKER else 'web' if row['script'] == WEB else 'api' if row['script'] == API else '')
+        row['_own_worker'] = row['processes'] == 1 and worker_pid in row['pids']
+        row['supervised'] = (None if supervised is None else row['script'] in supervised) if row['kind'] == 'platform' else False
         row.update(state=(bad or states or ['unknown'])[0], level='error' if bad else 'running',
                    cpu=round(row['cpu'], 1), memory_mb=int(row['memory_mb']), pids=sorted(row['pids'])[:PIDS_SHOWN],
-                   started=_iso(row['started']), commandable=row['kind'] == 'routine')
+                   started=_iso(row['started']),
+                   commandable=row['kind'] == 'routine' or (row['kind'] == 'platform' and row['role'] != 'scheduler'
+                                                          and (row['role'] != 'worker' or row['_own_worker'])))
         out.append(row)
     return sorted(out, key=lambda row: (row['script'].lower(), row['key']))
 
@@ -152,7 +215,13 @@ def view():
 
 def command(key, job):
     """The worker command for *job* on the routine whose card has this key. LookupError when it
-    no longer runs here, PermissionError for a platform process, ValueError for a wrong job."""
+    no longer runs here, PermissionError for the Scheduler and for an installed program,
+    ValueError for a wrong job.
+
+    A restart is the Worker's restart job (it ends the process and starts it again); the Worker's
+    own row restarts with reset instead, because a restart job would end the Worker itself before
+    the thread that starts it again could run. A stop is the Worker's kill job: the Worker ends
+    the process and does not start it again itself; what else may start it, the page says."""
     from SharedData.Web import workerpool
     if not isinstance(job, str) or job not in JOBS:
         raise ValueError('job must be one of: ' + ', '.join(JOBS))
@@ -160,9 +229,17 @@ def command(key, job):
     if not found:
         raise LookupError('that routine no longer runs on this server')
     row = found[0]
-    if row['kind'] != 'routine' or is_scheduler(row['script']):
-        raise PermissionError('this process is not a routine of the Worker (a platform process, or an installed program): it is not restarted or stopped from here')
-    command = {'sender': 'MASTER', 'job': JOBS[job], 'target': workerpool.local_worker(), 'repo': row['repo'], 'routine': row['routine']}
+    if row['kind'] == 'external':
+        raise PermissionError('this process is an installed program, not a script the Worker runs: it is not restarted or stopped from here')
+    if is_scheduler(row['script']):
+        raise PermissionError('the Scheduler is not restarted or stopped from here: its Worker keeps it running '
+                              '(end its process on that host and the Worker starts it again)')
+    target = workerpool.local_worker()
+    if row['role'] == 'worker' and not row['_own_worker']:
+        raise PermissionError('this is not the single Worker process that owns this WebUI; it cannot be commanded from here')
+    if row['role'] == 'worker' and job == 'restart':
+        return {'sender': 'MASTER', 'job': 'reset', 'target': target, 'reset_nginx': False}  # only the Worker and its children
+    command = {'sender': 'MASTER', 'job': JOBS[job], 'target': target, 'repo': row['repo'], 'routine': row['routine']}
     if row['branch'] and row['branch'] != 'nan':
         command['branch'] = row['branch']
     if row['_args'] and str(row['_args']) != 'nan':