shareddata 6.83.22__tar.gz → 6.85.0__tar.gz

shareddata-6.85.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022-2026 The SharedData Authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

shareddata-6.85.0/MANIFEST.in

@@ -0,0 +1,5 @@
+include src/SharedData/sharedmutexwin.pyd
+include src/SharedData/API/apidocs.yml
+include src/SharedData/API/apidocs-dark.css
+include src/SharedData/API/install/*
+prune tests

{shareddata-6.83.22/src/shareddata.egg-info → shareddata-6.85.0}/PKG-INFO

@@ -1,45 +1,74 @@
 Metadata-Version: 2.4
 Name: shareddata
-Version: 6.83.22
+Version: 6.85.0
 Summary: Memory Mapped / Shared Memory Database with S3 repository
+Home-page: https://shareddata.net
+Author: shareddata
+Author-email: admin@shareddata.net
+License: MIT
+Project-URL: Homepage, https://shareddata.net
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.9, <3.13
+Requires-Python: >=3.11, <3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: ipykernel==6.23.3
-Requires-Dist: boto3==1.26.160
-Requires-Dist: python-json-logger==2.0.7
-Requires-Dist: python-dotenv==1.0.1
+Requires-Dist: ipykernel<8,>=7.4
+Requires-Dist: python-json-logger<5,>=4.2
+Requires-Dist: python-dotenv<2,>=1.2.4
 Requires-Dist: pandas<3,>=2.2
-Requires-Dist: scipy==1.14.0
-Requires-Dist: numpy==1.26.2
+Requires-Dist: scipy<1.18,>=1.14
+Requires-Dist: numpy<1.27,>=1.26.4
 Requires-Dist: numba==0.59.1
-Requires-Dist: XlsxWriter==3.1.2
-Requires-Dist: openpyxl==3.1.2
-Requires-Dist: tqdm==4.65.0
-Requires-Dist: cffi==1.17.1
-Requires-Dist: tzlocal==5.0.1
-Requires-Dist: websockets==12.0
-Requires-Dist: cryptography==41.0.7
-Requires-Dist: lz4==4.3.3
-Requires-Dist: flask==3.0.0
-Requires-Dist: waitress==3.0.0
-Requires-Dist: requests==2.31.0
+Requires-Dist: XlsxWriter<4,>=3.2.9
+Requires-Dist: openpyxl<4,>=3.1.5
+Requires-Dist: tqdm<5,>=4.70.1
+Requires-Dist: cffi<3,>=2.1
+Requires-Dist: tzlocal<6,>=5.4.4
+Requires-Dist: websockets<18,>=17.1
+Requires-Dist: lz4<5,>=4.4.5
+Requires-Dist: flask<4,>=3.1.3
+Requires-Dist: waitress<4,>=3.0.2
+Requires-Dist: requests<3,>=2.34.2
 Requires-Dist: flasgger==0.9.7.1
-Requires-Dist: pymongo==4.8.0
-Requires-Dist: setuptools==74.1.2
-Requires-Dist: filelock==3.18.0
-Requires-Dist: gunicorn==23.0.0
-Requires-Dist: confluent-kafka==2.10.0
-Requires-Dist: aiokafka==0.12.0
-Requires-Dist: aiohttp==3.12.13
-Requires-Dist: redis==6.2.0
+Requires-Dist: pymongo<5,>=4.18.2
+Requires-Dist: setuptools>=84
+Requires-Dist: filelock<5,>=4.0.9
+Requires-Dist: gunicorn<27,>=26.2
+Requires-Dist: aiohttp<4,>=3.14.3
+Requires-Dist: boto3<2,>=1.43
+Requires-Dist: confluent-kafka<3,>=2.15.1
+Requires-Dist: aiokafka<1,>=0.14
+Requires-Dist: redis<9,>=8.1
+Requires-Dist: Authlib<2,>=1.8
+Requires-Dist: PyJWT<3,>=2.8
+Requires-Dist: argon2-cffi<26,>=25.1
+Requires-Dist: flask-jwt-extended<5,>=4.6
+Requires-Dist: cryptography<51,>=50.0.2
+Requires-Dist: segno<2,>=1.6
+Requires-Dist: psutil<8,>=5.9
+Requires-Dist: mcp>=1.27
+Requires-Dist: httpx>=0.27
+Provides-Extra: test
+Requires-Dist: pytest<10,>=8.3.4; extra == "test"
+Requires-Dist: pytest-asyncio<2,>=0.24; extra == "test"
+Requires-Dist: playwright<2,>=1.63; extra == "test"
+Provides-Extra: s3
+Provides-Extra: kafka
+Provides-Extra: redis
+Provides-Extra: all
+Dynamic: author
+Dynamic: author-email
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
 Dynamic: license-file
+Dynamic: project-url
+Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary
@@ -71,7 +100,7 @@ SharedData provides a comprehensive set of features for high-performance financi
 - **📋 Comprehensive Logging System** - Enterprise-grade logging with multiple destinations
 - **🌐 Remote API Client** - REST API for remote data access and operations
 
-**📚 [Read the complete Core Features guide →](docs/CORE_FEATURES.md)**
+**📚 [Read the complete Core Features guide →](https://shareddata.net/docs/CORE_FEATURES.md)**
 
 ## ⚡ Quick Start
 
@@ -111,7 +140,7 @@ data = tbl.loc['2025-01-05':, 'AAPL']  # Fast symbol lookup
 print(f"Retrieved {data.shape[0]} rows")
 ```
 
-**📚 [Read the complete Quick Start guide →](docs/QUICK_START.md)**
+**📚 [Read the complete Quick Start guide →](https://shareddata.net/docs/QUICK_START.md)**
 
 ## 🔧 Configuration
 
@@ -127,7 +156,7 @@ MONGODB_HOST="your-mongodb-host"
 SHAREDDATA_ENDPOINT="http://your-server:port"
 ```
 
-**📚 [Read the complete Configuration guide →](docs/CONFIGURATION.md)**
+**📚 [Read the complete Configuration guide →](https://shareddata.net/docs/CONFIGURATION.md)**
 
 ## 🚀 Advanced Usage
 
@@ -140,7 +169,26 @@ For power users and complex scenarios:
 - **Production Deployment** - High availability and monitoring
 - **Monitoring & Debugging** - Performance profiling and health checks
 
-**📚 [Read the complete Advanced Usage guide →](docs/ADVANCED_USAGE.md)**
+**📚 [Read the complete Advanced Usage guide →](https://shareddata.net/docs/ADVANCED_USAGE.md)**
+
+## 🧭 Data by Symbol
+
+Fetch data by what it is instead of where it is stored. The server picks the source, the
+container and the partitions, and every answer says where its rows came from:
+
+```python
+bars = shdata.dataframe('marketdata', 'AAPL', period='M15', start='2026-10-05')
+bars.attrs['provenance']['dataset']        # which container answered
+
+from SharedData.IO.ClientAPI import ClientAPI
+surface = ClientAPI.get_vol('AAPL', product='tenor', tenor='1M')
+```
+
+Market data, signals, portfolios, risk factors and volatility are served this way over
+`GET /api/data/...`. This is a convenience, not the fast path: producers and other
+performance-critical code keep using `table()`, `timeseries()` and `collection()`.
+
+**📚 [Read the Data by Symbol guide →](https://shareddata.net/docs/data-api.md)**
 
 ## 🔄 Development & Documentation
 
@@ -164,11 +212,11 @@ make all
 python -m pytest tests/
 ```
 
-**📚 [Read the complete Development guide →](docs/DEVELOPMENT.md)**
+**📚 [Read the complete Development guide →](https://shareddata.net/docs/DEVELOPMENT.md)**
 
 ## 📄 License
 
-See [LICENSE](LICENSE) file for details.
+MIT: free to use, change and distribute, commercially too. See the [LICENSE](https://shareddata.net/LICENSE) file.
 
 ## 🤝 Contributing
 
@@ -178,17 +226,17 @@ See [LICENSE](LICENSE) file for details.
 4. Update documentation: `make gitea`
 5. Submit a pull request
 
-**📚 [Read the complete Contributing guide →](docs/DEVELOPMENT.md#contributing-guidelines)**
+**📚 [Read the complete Contributing guide →](https://shareddata.net/docs/DEVELOPMENT.md#contributing-guidelines)**
 
 ---
 
 ## 📚 Documentation Index
 
-- **[🏗️ Core Features](docs/CORE_FEATURES.md)** - Comprehensive overview of all SharedData capabilities
-- **[⚡ Quick Start](docs/QUICK_START.md)** - Get started with SharedData in minutes
-- **[🔧 Configuration](docs/CONFIGURATION.md)** - Complete configuration reference
-- **[🚀 Advanced Usage](docs/ADVANCED_USAGE.md)** - Advanced patterns and optimization
-- **[🔄 Development](docs/DEVELOPMENT.md)** - Contributing and development guidelines
+- **[🏗️ Core Features](https://shareddata.net/docs/CORE_FEATURES.md)** - Comprehensive overview of all SharedData capabilities
+- **[⚡ Quick Start](https://shareddata.net/docs/QUICK_START.md)** - Get started with SharedData in minutes
+- **[🔧 Configuration](https://shareddata.net/docs/CONFIGURATION.md)** - Complete configuration reference
+- **[🚀 Advanced Usage](https://shareddata.net/docs/ADVANCED_USAGE.md)** - Advanced patterns and optimization
+- **[🔄 Development](https://shareddata.net/docs/DEVELOPMENT.md)** - Contributing and development guidelines
 
 ---
 

{shareddata-6.83.22 → shareddata-6.85.0}/README.md

@@ -25,7 +25,7 @@ SharedData provides a comprehensive set of features for high-performance financi
 - **📋 Comprehensive Logging System** - Enterprise-grade logging with multiple destinations
 - **🌐 Remote API Client** - REST API for remote data access and operations
 
-**📚 [Read the complete Core Features guide →](docs/CORE_FEATURES.md)**
+**📚 [Read the complete Core Features guide →](https://shareddata.net/docs/CORE_FEATURES.md)**
 
 ## ⚡ Quick Start
 
@@ -65,7 +65,7 @@ data = tbl.loc['2025-01-05':, 'AAPL']  # Fast symbol lookup
 print(f"Retrieved {data.shape[0]} rows")
 ```
 
-**📚 [Read the complete Quick Start guide →](docs/QUICK_START.md)**
+**📚 [Read the complete Quick Start guide →](https://shareddata.net/docs/QUICK_START.md)**
 
 ## 🔧 Configuration
 
@@ -81,7 +81,7 @@ MONGODB_HOST="your-mongodb-host"
 SHAREDDATA_ENDPOINT="http://your-server:port"
 ```
 
-**📚 [Read the complete Configuration guide →](docs/CONFIGURATION.md)**
+**📚 [Read the complete Configuration guide →](https://shareddata.net/docs/CONFIGURATION.md)**
 
 ## 🚀 Advanced Usage
 
@@ -94,7 +94,26 @@ For power users and complex scenarios:
 - **Production Deployment** - High availability and monitoring
 - **Monitoring & Debugging** - Performance profiling and health checks
 
-**📚 [Read the complete Advanced Usage guide →](docs/ADVANCED_USAGE.md)**
+**📚 [Read the complete Advanced Usage guide →](https://shareddata.net/docs/ADVANCED_USAGE.md)**
+
+## 🧭 Data by Symbol
+
+Fetch data by what it is instead of where it is stored. The server picks the source, the
+container and the partitions, and every answer says where its rows came from:
+
+```python
+bars = shdata.dataframe('marketdata', 'AAPL', period='M15', start='2026-10-05')
+bars.attrs['provenance']['dataset']        # which container answered
+
+from SharedData.IO.ClientAPI import ClientAPI
+surface = ClientAPI.get_vol('AAPL', product='tenor', tenor='1M')
+```
+
+Market data, signals, portfolios, risk factors and volatility are served this way over
+`GET /api/data/...`. This is a convenience, not the fast path: producers and other
+performance-critical code keep using `table()`, `timeseries()` and `collection()`.
+
+**📚 [Read the Data by Symbol guide →](https://shareddata.net/docs/data-api.md)**
 
 ## 🔄 Development & Documentation
 
@@ -118,11 +137,11 @@ make all
 python -m pytest tests/
 ```
 
-**📚 [Read the complete Development guide →](docs/DEVELOPMENT.md)**
+**📚 [Read the complete Development guide →](https://shareddata.net/docs/DEVELOPMENT.md)**
 
 ## 📄 License
 
-See [LICENSE](LICENSE) file for details.
+MIT: free to use, change and distribute, commercially too. See the [LICENSE](https://shareddata.net/LICENSE) file.
 
 ## 🤝 Contributing
 
@@ -132,17 +151,17 @@ See [LICENSE](LICENSE) file for details.
 4. Update documentation: `make gitea`
 5. Submit a pull request
 
-**📚 [Read the complete Contributing guide →](docs/DEVELOPMENT.md#contributing-guidelines)**
+**📚 [Read the complete Contributing guide →](https://shareddata.net/docs/DEVELOPMENT.md#contributing-guidelines)**
 
 ---
 
 ## 📚 Documentation Index
 
-- **[🏗️ Core Features](docs/CORE_FEATURES.md)** - Comprehensive overview of all SharedData capabilities
-- **[⚡ Quick Start](docs/QUICK_START.md)** - Get started with SharedData in minutes
-- **[🔧 Configuration](docs/CONFIGURATION.md)** - Complete configuration reference
-- **[🚀 Advanced Usage](docs/ADVANCED_USAGE.md)** - Advanced patterns and optimization
-- **[🔄 Development](docs/DEVELOPMENT.md)** - Contributing and development guidelines
+- **[🏗️ Core Features](https://shareddata.net/docs/CORE_FEATURES.md)** - Comprehensive overview of all SharedData capabilities
+- **[⚡ Quick Start](https://shareddata.net/docs/QUICK_START.md)** - Get started with SharedData in minutes
+- **[🔧 Configuration](https://shareddata.net/docs/CONFIGURATION.md)** - Complete configuration reference
+- **[🚀 Advanced Usage](https://shareddata.net/docs/ADVANCED_USAGE.md)** - Advanced patterns and optimization
+- **[🔄 Development](https://shareddata.net/docs/DEVELOPMENT.md)** - Contributing and development guidelines
 
 ---
 

{shareddata-6.83.22 → shareddata-6.85.0}/pyproject.toml

@@ -9,6 +9,7 @@ build-backend = "setuptools.build_meta"
 # Configure pytest to suppress asyncio warnings
 asyncio_default_fixture_loop_scope = "function"
 pythonpath = ["src"]
+testpaths = ["tests"]
 filterwarnings = [
     "ignore::pytest.PytestDeprecationWarning",
     "ignore::pytest.PytestReturnNotNoneWarning"

shareddata-6.85.0/setup.py

@@ -0,0 +1,110 @@
+import sys
+from setuptools import setup, find_packages
+
+install_requires = [
+    'ipykernel>=7.4,<8',
+    'python-json-logger>=4.2,<5',
+    'python-dotenv>=1.2.4,<2',
+    # pandas, numba and numpy stay where table and timeseries are tested:
+    # numba 0.59.1 caps numpy below 1.27, which in turn caps scipy at 1.17.
+    'pandas>=2.2,<3',
+    'scipy>=1.14,<1.18',
+    'numpy>=1.26.4,<1.27',
+    'numba==0.59.1',
+    'XlsxWriter>=3.2.9,<4',
+    'openpyxl>=3.1.5,<4',
+    'tqdm>=4.70.1,<5',
+    'cffi>=2.1,<3',
+    'tzlocal>=5.4.4,<6',
+    'websockets>=17.1,<18',
+    'lz4>=4.4.5,<5',
+    'flask>=3.1.3,<4',
+    'waitress>=3.0.2,<4',
+    'requests>=2.34.2,<3',
+    'flasgger==0.9.7.1',
+    'pymongo>=4.18.2,<5',
+    'setuptools>=84',
+    'filelock>=4.0.9,<5',
+    'gunicorn>=26.2,<27',
+    'aiohttp>=3.14.3,<4',
+]
+
+# The S3, Kafka and Redis clients: installed with the package, none is optional.
+client_requires = [
+    'boto3>=1.43,<2',
+    'confluent-kafka>=2.15.1,<3',
+    'aiokafka>=0.14,<1',
+    'redis>=8.1,<9',
+]
+install_requires = install_requires + client_requires
+
+# JavaScript WebUI backend.
+dashboard_requires = [
+    'Authlib>=1.8,<2',
+    'PyJWT>=2.8,<3',
+    'argon2-cffi>=25.1,<26',
+    'flask-jwt-extended>=4.6,<5',
+    'cryptography>=50.0.2,<51',  # two-factor secrets at rest
+    'segno>=1.6,<2',     # two-factor QR codes (pure Python)
+    'psutil>=5.9,<8',
+]
+# Ship the web/dashboard layer by default (no optional extra).
+install_requires = install_requires + dashboard_requires
+
+# The MCP server (src/SharedData/MCP, the shareddata-mcp console script): reads the API over HTTP.
+mcp_requires = [
+    'mcp>=1.27',        # 1.27 is the oldest version the smoke test ran on; 2.x is included
+    'httpx>=0.27',
+]
+install_requires = install_requires + mcp_requires
+
+test_requires = [
+    'pytest>=8.3.4,<10',
+    'pytest-asyncio>=0.24,<2',
+    'playwright>=1.63,<2',
+]
+
+setup(
+    name='shareddata',
+    version='6.85.0',
+    description='Memory Mapped / Shared Memory Database with S3 repository',
+    author='shareddata',
+    author_email='admin@shareddata.net',
+    url='https://shareddata.net',
+    project_urls={
+        'Homepage': 'https://shareddata.net',
+    },
+    license='MIT',
+    long_description=open('README.md').read(),
+    long_description_content_type='text/markdown',
+    classifiers=[
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
+        'License :: OSI Approved :: MIT License',
+        'Operating System :: OS Independent',
+    ],
+    packages=find_packages(where='src'),
+    package_dir={'': 'src'},
+    python_requires='>=3.11, <3.13',
+    install_requires=install_requires,
+    # s3, kafka, redis and all are kept so that an install that names them still works: they add nothing.
+    extras_require={'test': test_requires, 's3': [], 'kafka': [], 'redis': [], 'all': []},
+    include_package_data=True,
+    package_data={
+        'SharedData.API': ['apidocs.yml', 'apidocs-dark.css', 'install/*'],
+        'SharedData.Web': [
+            'templates/*', 'templates/**/*',
+            'static/*', 'static/**/*',
+            'assets/*',
+            'ADDING_A_PAGE.md',
+        ],
+    },
+    entry_points={
+        'console_scripts': [
+            'shdata-web=SharedData.Web.server:main',
+            'shdata-server=SharedData.server:main',
+            'shareddata-mcp=SharedData.MCP.server:main',
+        ],
+    },
+)

{shareddata-6.83.22 → shareddata-6.85.0}/src/SharedData/API/ServerGunicorn.py

@@ -10,13 +10,14 @@ Usage:
 from __future__ import annotations
 
 import argparse
+import logging
 import os
 import sys
 import threading
 import time
 from gunicorn.app.base import BaseApplication
 
-from SharedData.Logger import Logger
+from SharedData.Logger import Logger, install_redaction
 Logger.connect('SharedData.API.ServerGunicorn')
 
 from SharedData.API import create_app
@@ -91,6 +92,10 @@ def parse_cli():
                    help="Number of request threads *per* worker (gthread)")
     p.add_argument("--timeout", type=int, default=120,
                    help="Hard kill after N seconds")
+    p.add_argument("--table-waiters", type=int, default=None,
+                   help="Request threads per worker that may wait for one locked table; "
+                        "further requests for it get 503. Default: a quarter of --nthreads "
+                        "(or SHAREDDATA_API_TABLE_WAITERS); 0 turns it off")
     p.add_argument("--log-level", default="warning")
     return p.parse_args()
 
@@ -131,6 +136,13 @@ def main():
     # Launch in-process helper threads
     start_background_threads()
 
+    # One locked table must not take every request thread: per worker, at most
+    # this many may wait for the same table's lock; the rest get 503 at once.
+    waiters = args.table_waiters
+    if waiters is None:
+        waiters = int(os.environ.get('SHAREDDATA_API_TABLE_WAITERS') or max(1, args.nthreads // 4))
+    flask_app.config['TABLE_WAITERS'] = max(0, waiters)
+
     # Assemble Gunicorn config
     gunicorn_opts = {
         "bind":            f"{args.host}:{args.port}",
@@ -148,13 +160,18 @@ def main():
         "limit_request_line": 8190,       # 8190 is HTTP/1.1 spec max        
     }
 
+    # Gunicorn's error log names the request it failed on, query string (token) and all.
+    install_redaction(logging.getLogger('gunicorn.error'))
+
     # Block here; Gunicorn handles signals and will exit cleanly
     GunicornEmbedded(flask_app, gunicorn_opts).run()
 
 
 if __name__ == "__main__":
     # Basic env-var sanity checks
-    required_env = ("SHAREDDATA_SECRET_KEY", "SHAREDDATA_TOKEN")
+    # SHAREDDATA_TOKEN is the ClientAPI default, not a server setting: the API
+    # checks hashed tokens in the users store.
+    required_env = ("SHAREDDATA_SECRET_KEY",)
     missing = [v for v in required_env if v not in os.environ]
     if missing:
         sys.exit(f"Missing environment variables: {', '.join(missing)}")

{shareddata-6.83.22 → shareddata-6.85.0}/src/SharedData/API/ServerWaitress.py

@@ -31,7 +31,9 @@ def parse_args():
 
 def validate_environment():
     """Validate required environment variables are set."""
-    required_env = ('SHAREDDATA_SECRET_KEY', 'SHAREDDATA_TOKEN')
+    # SHAREDDATA_TOKEN is the ClientAPI default, not a server setting: the API
+    # checks hashed tokens in the users store.
+    required_env = ('SHAREDDATA_SECRET_KEY',)
     missing = [v for v in required_env if v not in os.environ]
     if missing:
         sys.exit(f"Missing environment variables: {', '.join(missing)}")

{shareddata-6.83.22 → shareddata-6.85.0}/src/SharedData/API/__init__.py

@@ -7,11 +7,12 @@ This package provides a modular, blueprint-based Flask API for SharedData operat
 import os
 from pathlib import Path
 
-from flask import Flask
+from flask import Flask, jsonify, request
 from flasgger import Swagger
 from werkzeug.middleware.proxy_fix import ProxyFix
 
 from SharedData.SharedData import SharedData
+from SharedData.Redaction import install_flask_redaction
 from SharedData.Logger import Logger
 
 from SharedData.API.auth import authenticate
@@ -32,7 +33,7 @@ from SharedData.API.routes.system import (
 )
 
 
-def create_app(shdata=None, quiet=True):
+def create_app(shdata=None, quiet=True, app=None):
     """
     Create and configure the Flask application with all blueprints.
     
@@ -44,7 +45,8 @@ def create_app(shdata=None, quiet=True):
         Configured Flask application instance.
     """
     # Create Flask app
-    app = Flask(__name__)
+    app = app or Flask(__name__)
+    install_flask_redaction(app)
     app.config['APP_NAME'] = 'SharedData API'
     app.config['FLASK_ENV'] = 'production'
     app.config['FLASK_DEBUG'] = '0'
@@ -53,12 +55,21 @@ def create_app(shdata=None, quiet=True):
     if 'SHAREDDATA_SECRET_KEY' not in os.environ:
         raise Exception('SHAREDDATA_SECRET_KEY environment variable not set')
 
-    app.config['SECRET_KEY'] = os.environ['SHAREDDATA_SECRET_KEY']
+    if not app.config.get('SECRET_KEY'):
+        app.config['SECRET_KEY'] = os.environ['SHAREDDATA_SECRET_KEY']
+    dark_css = Path(__file__).parent / 'apidocs-dark.css'
     app.config['SWAGGER'] = {
         'title': 'SharedData API',
         'uiversion': 3,
+        'openapi': '3.0.3',  # apidocs.yml is OpenAPI 3
         'hide_top_bar': True,
-        'doc_expansion': 'list'
+        'doc_expansion': 'none',  # tags collapsed: the overview first
+        # flasgger prints this straight into the page's script: without it the page gets
+        # `let auth_config = None;` and a script error (the docs still rendered).
+        'auth': {},
+        # Dark theme matching the WebUI.
+        'head_text': ('<meta name="viewport" content="width=device-width, initial-scale=1">'
+                      + ('<style>' + dark_css.read_text() + '</style>' if dark_css.exists() else '')),
     }
 
     # Setup Swagger documentation
@@ -80,24 +91,37 @@ def create_app(shdata=None, quiet=True):
         return authenticate(request, sd)
 
     # Initialize and register blueprints
-    init_tables_routes(shdata, authenticate_fn)
-    init_collections_routes(shdata, authenticate_fn)
-    init_timeseries_routes(shdata, authenticate_fn)
-    init_workers_routes(shdata, authenticate_fn)
-    init_metadata_routes(shdata, authenticate_fn)
-    init_cache_routes(shdata, authenticate_fn)
-    downloads_bp = init_downloads_routes(shdata, authenticate_fn)
-    init_system_routes(shdata, authenticate_fn)
-
-    # Register blueprints with /api prefix
-    app.register_blueprint(tables_bp, url_prefix='/api')
-    app.register_blueprint(collections_bp, url_prefix='/api')
-    app.register_blueprint(timeseries_bp, url_prefix='/api')
-    app.register_blueprint(workers_bp, url_prefix='/api')
-    app.register_blueprint(metadata_bp, url_prefix='/api')
-    app.register_blueprint(cache_bp, url_prefix='/api')
-    app.register_blueprint(downloads_bp, url_prefix='/api')
-    app.register_blueprint(system_bp, url_prefix='/api')
+    for factory in (init_tables_routes, init_collections_routes, init_timeseries_routes,
+                    init_workers_routes, init_metadata_routes, init_cache_routes,
+                    init_downloads_routes, init_system_routes):
+        app.register_blueprint(factory(shdata, authenticate_fn), url_prefix='/api')
+    from SharedData.API.routes.sqlite import init_sqlite_routes
+    app.register_blueprint(init_sqlite_routes(shdata, authenticate_fn), url_prefix='/api')
+    from SharedData.API.routes.web_routes import init_routes_routes
+    app.register_blueprint(init_routes_routes(shdata, authenticate_fn), url_prefix='/api')
+    from SharedData.API.routes.schedules import init_schedules_routes
+    app.register_blueprint(init_schedules_routes(shdata, authenticate_fn), url_prefix='/api')
+    from SharedData.API.routes.operations import init_operations_routes
+    app.register_blueprint(init_operations_routes(shdata, authenticate_fn), url_prefix='/api')
+    from SharedData.API.routes.data import init_data_routes
+    app.register_blueprint(init_data_routes(shdata, authenticate_fn), url_prefix='/api')
+
+    from SharedData.API.utils import init_table_waiters
+    init_table_waiters(app)
+
+    @app.before_request
+    def auth_availability():
+        from SharedData.Auth.store import StoreUnavailable
+        # Worker install scripts and enrollment take a one-time install code instead of a token.
+        public = request.path in ('/api/health', '/api/heartbeat', '/api/workers/enroll') or \
+            request.path.startswith(('/api/auth/', '/api/web/', '/api/workers/install/'))
+        from SharedData.API.auth import local_log_token_ok
+        if request.path.startswith('/api/') and not public and not local_log_token_ok(request):
+            try:
+                if not authenticate_fn(request, shdata):
+                    return jsonify(error='unauthorized'), 401
+            except StoreUnavailable:
+                return jsonify(error='auth_store_unavailable'), 503
 
     # Register request lifecycle hooks
     register_request_hooks(app)