keepup-modules 0.5.0__py3-none-any.whl

keepup_modules/catalogue.py

@@ -0,0 +1,812 @@
+"""The catalogue of panel sections.
+
+A section is a piece of the front end -- a script, a stylesheet, an init
+function -- and a role is granted the sections it may see. The catalogue lives
+in the database; the file the application ships is a seed and a fallback, and
+new sections are added from it without touching what an administrator has
+already switched off or taken away from a role.
+"""
+
+import asyncio
+import json
+from pathlib import Path
+import logging
+import time
+from typing import Any, Dict, List, Optional
+
+import keepup
+from fastapi import Depends, HTTPException
+from pydantic import BaseModel
+
+from keepup_auth.dependencies import get_current_admin, get_current_user
+from keepup_auth import user_roles
+from keepup import cache
+from keepup_audit import trail as admin_trail
+from keepup.logging_setup import for_log
+
+#: What an application may import from this module. Everything else is
+#: internal and may change without notice -- see doc/keepup.md.
+__all__ = [
+    "create_or_update_module",
+    "delete_module",
+    "get_all_modules_from_db",
+    "get_module_by_id",
+    "get_modules_for_role",
+    "get_modules_for_roles",
+    "import_modules_from_json",
+    "register_module_routes",
+    "sync_framework_sections",
+    "sync_new_modules_from_json",
+    "update_role_modules",
+]
+
+logger = logging.getLogger(__name__)
+
+MODULES_CONFIG_PATH = 'config/modules.json'
+
+
+def configure(path: str = None) -> None:
+    """Where this deployment keeps its catalogue of sections and plugins.
+
+    The application names the file in KeepupSettings.plugins_config_path and
+    create_app() hands it here. Until this existed only initialize_plugins()
+    was given that path: the catalogue was seeded from the default file, so a
+    deployment that named another one came up with the framework's own sections
+    and none of its own, and nothing said why (keepup-98).
+    """
+    global MODULES_CONFIG_PATH
+    if path:
+        MODULES_CONFIG_PATH = path
+
+
+class ModuleCreate(BaseModel):
+    """A panel section being declared."""
+
+    module_id: str
+    name: str
+    description: Optional[str] = None
+    js_path: Optional[str] = None
+    css_path: Optional[str] = None
+    init_function: Optional[str] = None
+    version: str = "1.0.0"
+    config: Optional[Dict[str, Any]] = None
+
+
+class ModuleUpdate(BaseModel):
+    """The parts of a declared section that may be changed."""
+
+    name: Optional[str] = None
+    description: Optional[str] = None
+    js_path: Optional[str] = None
+    css_path: Optional[str] = None
+    init_function: Optional[str] = None
+    version: Optional[str] = None
+    config: Optional[Dict[str, Any]] = None
+    is_active: Optional[bool] = None
+
+
+class RoleModulesUpdate(BaseModel):
+    """Which sections a role is granted."""
+
+    role_name: str
+    module_ids: List[str]
+
+
+def get_all_modules_from_db():
+    """Return every module from the database."""
+    from keepup_db import DatabaseManagerV2
+    return DatabaseManagerV2.execute('''
+    SELECT * FROM frontend_modules 
+    WHERE is_active = TRUE 
+    ORDER BY name
+    ''')
+
+
+def get_module_by_id(module_id: str):
+    """Return a module by id."""
+    from keepup_db import DatabaseManagerV2
+    return DatabaseManagerV2.execute_one('''
+    SELECT * FROM frontend_modules 
+    WHERE module_id = :module_id AND is_active = TRUE
+    ''', {"module_id": module_id})
+
+
+#: The replica cache of each role's sections (keepup/cache.py): every page of
+#: the panel asks for its role's list, and the catalogue changes only when an
+#: administrator changes it.
+SECTIONS_CACHE = "panel_sections"
+
+#: Which plugins have declared a section, and which of them are running. A
+#: section whose plugin is not running is not in the panel -- and its row and
+#: the grants an administrator made are kept, so it comes back with its plugin
+#: (keepup-113, doc/plugin_constructor.md 9.1).
+_declaring_plugins: set = set()
+_running_plugins: set = set()
+
+
+def set_declared_plugins(plugin_ids) -> None:
+    """Remember which plugins declared a section, whoever asked when."""
+    global _declaring_plugins
+
+    _declaring_plugins = set(plugin_ids)
+    _sections_changed()
+
+
+def set_running_plugins(plugin_ids) -> None:
+    """Remember which plugins came up; the panel reads the sections of those."""
+    global _running_plugins
+
+    _running_plugins = set(plugin_ids)
+    _sections_changed()
+
+
+def hidden_plugins() -> tuple:
+    """The plugins that declared a section and are not running now."""
+    return tuple(sorted(_declaring_plugins - _running_plugins))
+
+
+def get_modules_for_role(role_name: str):
+    """Return the modules granted to a role.
+
+    Kept in this replica for the cache's lifetime and dropped whenever the
+    catalogue or a grant changes; each caller gets its own copies.
+    """
+    rows = cache.get_cache(SECTIONS_CACHE).get(
+        role_name, lambda: _read_modules_for_role(role_name))
+    return [dict(row) for row in rows]
+
+
+def get_modules_for_roles(role_names):
+    """Return the modules granted to any of these roles, each of them once.
+
+    A user holds a set of roles (keepup/auth/user_roles.py) and sees the sections
+    of all of them glued together. A section granted to two of the roles is in
+    the answer once: the loader mounts a section by its id, and a second copy
+    would initialise the page twice.
+
+    Each role's list is still read and kept on its own (the cache is keyed by
+    role name), so gluing costs nothing beyond the roles the person actually
+    holds. The order is by name and does not depend on the order of the roles --
+    the panel's menu must not rearrange itself because a role was granted later.
+
+    Args:
+        role_names: the roles to glue.
+
+    Returns:
+        The modules, ordered by name.
+    """
+    glued = {}
+    for role_name in role_names or []:
+        for module in get_modules_for_role(role_name):
+            glued.setdefault(module["module_id"], module)
+    def place(module):
+        order = module.get("nav_order")
+        return (order is None, order or 0, module.get("name") or "", module["module_id"])
+
+    return sorted(glued.values(), key=place)
+
+
+def _sections_changed() -> None:
+    cache.invalidate_everywhere(SECTIONS_CACHE)
+
+
+def _read_modules_for_role(role_name: str):
+    """The sections this role may see, from plugins that are running.
+
+    The filter is what makes a capability switched off by an administrator
+    disappear from the panel with its routes; the row stays, so switching it
+    back on brings the section and its grants back (keepup-113).
+    """
+    from keepup_db import DatabaseManagerV2
+
+    hidden = hidden_plugins()
+    statement = ("SELECT fm.* FROM frontend_modules fm"
+                 " JOIN role_modules rm ON fm.module_id = rm.module_id"
+                 " WHERE rm.role_name = :role_name"
+                 " AND rm.is_active = TRUE AND fm.is_active = TRUE")
+    params: Dict[str, Any] = {"role_name": role_name}
+    if hidden:
+        placeholders = ", ".join(f":hidden_{index}" for index in range(len(hidden)))
+        statement += f" AND (fm.declared_by IS NULL OR fm.declared_by NOT IN ({placeholders}))"
+        params.update({f"hidden_{index}": name for index, name in enumerate(hidden)})
+    statement += " ORDER BY fm.name"
+    return DatabaseManagerV2.execute(statement, params)
+
+
+def create_or_update_module(module_data: Dict[str, Any]):
+    """Create or update a module."""
+    from keepup_db import DatabaseManagerV2, db_config
+    try:
+        module_id = module_data['id']
+        logger.info(f"=== Processing module: {module_id} ===")
+
+        name = module_data['name']
+        description = module_data.get('description')
+        js_path = module_data.get('js')
+        css_path = module_data.get('css')
+        init_function = module_data.get('initFunction')
+        version = module_data.get('version', '1.0.0')
+        config = json.dumps(module_data.get('config', {}))
+
+        logger.info(f"DB type: {db_config.db_type}")
+        logger.info(f"Name: {name}")
+        logger.info(f"JS path: {js_path}")
+        logger.info(f"CSS path: {css_path}")
+        logger.info(f"Init function: {init_function}")
+
+        existing = get_module_by_id(module_id)
+        logger.info(f"Module exists: {existing is not None}")
+
+        params = {"module_id": module_id, "name": name, "description": description,
+                  "js_path": js_path, "css_path": css_path, "init_function": init_function,
+                  "version": version, "config": config}
+        if existing:
+            logger.info(f"UPDATE path for module: {module_id}")
+            query = '''
+            UPDATE frontend_modules 
+            SET name = :name, description = :description, js_path = :js_path,
+                css_path = :css_path, init_function = :init_function, version = :version,
+                config = :config, updated_at = CURRENT_TIMESTAMP
+            WHERE module_id = :module_id
+            '''
+        else:
+            logger.info(f"INSERT path for module: {module_id}")
+            query = '''
+            INSERT INTO frontend_modules 
+            (module_id, name, description, js_path, css_path, init_function, version, config)
+            VALUES (:module_id, :name, :description, :js_path, :css_path, :init_function,
+                    :version, :config)
+            '''
+
+        start_time = time.time()
+        result = DatabaseManagerV2.execute_commit(query, params)
+        _sections_changed()
+
+        execution_time = time.time() - start_time
+        logger.info(f"Query executed in {execution_time:.2f} seconds")
+
+        if result:
+            logger.info(f"✓ Successfully {'updated' if existing else 'created'} module: {module_id}")
+        else:
+            logger.error(f"✗ Failed to {'update' if existing else 'create'} module: {module_id}")
+
+    except Exception as e:
+        logger.error(f"🚨 Error in create_or_update_module for {module_data.get('id', 'unknown')}: {str(e)}")
+        import traceback
+        logger.error(f"Stack trace: {traceback.format_exc()}")
+        raise
+
+
+def _catalogue_entry(module_id: str, fields: Dict[str, Any]) -> Dict[str, Any]:
+    """A section as the routes describe it, in the shape the catalogue file has.
+
+    create_or_update_module() reads the file's names (`id`, `js`, `css`,
+    `initFunction`), and the routes handed it their own (`module_id`, `js_path`,
+    ...): every create and every change through the API answered 400 (keepup-80).
+    """
+    config = fields.get("config") or {}
+    if isinstance(config, str):
+        config = json.loads(config or "{}")
+    return {"id": module_id, "name": fields["name"],
+            "description": fields.get("description"),
+            "js": fields.get("js_path"), "css": fields.get("css_path"),
+            "initFunction": fields.get("init_function"),
+            "version": fields.get("version") or "1.0.0", "config": config}
+
+
+#: The fields of a stored section a change through the API may carry.
+_EDITABLE = ("name", "description", "js_path", "css_path", "init_function", "version", "config")
+
+
+def _changed_entry(existing: Dict[str, Any], changes: Dict[str, Any]) -> Dict[str, Any]:
+    """The stored section with the given fields changed and the rest kept."""
+    merged = {field: existing.get(field) for field in _EDITABLE}
+    merged.update({field: value for field, value in changes.items() if field in _EDITABLE})
+    return _catalogue_entry(existing["module_id"], merged)
+
+
+def update_role_modules(role_name: str, module_ids: List[str]):
+    """Update the modules granted to a role."""
+    from keepup_db import DatabaseManagerV2, db_config
+    DatabaseManagerV2.execute_commit('''
+    UPDATE role_modules 
+    SET is_active = FALSE 
+    WHERE role_name = :role_name
+    ''', {"role_name": role_name})
+
+    for module_id in module_ids:
+        if db_config.is_postgres():
+            DatabaseManagerV2.execute_commit('''
+            INSERT INTO role_modules (role_name, module_id, is_active)
+            VALUES (:role_name, :module_id, TRUE)
+            ON CONFLICT (role_name, module_id) 
+            DO UPDATE SET is_active = TRUE
+            ''', {"role_name": role_name, "module_id": module_id})
+        else:
+            DatabaseManagerV2.execute_commit('''
+            INSERT OR REPLACE INTO role_modules (role_name, module_id, is_active)
+            VALUES (:role_name, :module_id, TRUE)
+            ''', {"role_name": role_name, "module_id": module_id})
+    _sections_changed()
+
+
+def delete_module(module_id: str):
+    """Delete a module (soft delete)."""
+    from keepup_db import DatabaseManagerV2
+    DatabaseManagerV2.execute_commit('''
+    UPDATE frontend_modules 
+    SET is_active = FALSE, updated_at = CURRENT_TIMESTAMP
+    WHERE module_id = :module_id
+    ''', {"module_id": module_id})
+    _sections_changed()
+
+
+def import_modules_from_json():
+    """Import the modules from the JSON file into the database."""
+    try:
+        with open(MODULES_CONFIG_PATH, 'r', encoding='utf-8') as f:
+            config = json.load(f)
+
+        for module_config in config.get('modules', []):
+            create_or_update_module(module_config)
+
+        for role_config in config.get('roles', []):
+            role_name = role_config.get('name')
+            module_ids = role_config.get('modules', [])
+            update_role_modules(role_name, module_ids)
+
+        logger.info("Modules imported successfully from JSON")
+        return True
+    except Exception as e:
+        logger.error(f"Error importing modules from JSON: {str(e)}")
+        return False
+
+
+#: The catalogue the framework ships. Resolved from this file's location, like
+#: the panel shell: it is data of the package, and the process may have been
+#: started anywhere.
+#: The framework's own declaration of its sections ships with the package the
+#: capability is built on, and there is one such file, not two (keepup-113).
+FRAMEWORK_SECTIONS = str(Path(keepup.__file__).resolve().parent / "sections.json")
+
+
+_INSERT_SECTION_IF_NEW = (
+    "INSERT INTO frontend_modules (module_id, name, description, js_path, css_path, "
+    "init_function, icon, handler_function, route_path, section_id, admin_only, "
+    "nav_order, declared_by, version, config) "
+    "VALUES (:id, :name, :description, :js, :css, :init, :icon, :handler, :route, "
+    ":section_id, :admin_only, :nav_order, :declared_by, :version, :config) "
+    "ON CONFLICT (module_id) DO NOTHING")
+
+_INSERT_GRANT_IF_NEW = (
+    "INSERT INTO role_modules (role_name, module_id, is_active) "
+    "VALUES (:role, :module_id, TRUE) ON CONFLICT (role_name, module_id) DO NOTHING")
+
+
+#: The fields the declaring plugin owns: they are code, and the code knows
+#: where its files are. An administrator's name, description, is_active and
+#: grants are never in this list (doc/plugin_constructor.md, 9.1).
+_CODE_OWNED = ("js_path", "css_path", "init_function", "icon", "handler_function",
+               "route_path", "section_id", "admin_only", "nav_order")
+
+
+def _repoint_statement() -> str:
+    """The update that gives the declaring plugin its own fields back.
+
+    Nothing an administrator owns is touched: not `name`, not `description`,
+    not `is_active`, not a grant. A deployment that has been running since
+    before a move keeps `/static/modules/js/users.js` otherwise, the file is
+    not there any more, and the section breaks silently.
+    """
+    from keepup_db import db_config
+
+    changed = " OR ".join(f"{field} IS DISTINCT FROM :{field}" for field in _CODE_OWNED)
+    if db_config.is_postgres():
+        return (f"UPDATE frontend_modules SET {_assignment()} WHERE module_id = :id "
+                f"AND ({changed})")
+    not_is = " OR ".join(f"{field} IS NOT :{field}" for field in _CODE_OWNED)
+    return f"UPDATE frontend_modules SET {_assignment()} WHERE module_id = :id AND ({not_is})"
+
+
+def _assignment() -> str:
+    """`field = :field` for every field the declaring plugin owns."""
+    return ", ".join(f"{field} = :{field}" for field in _CODE_OWNED)
+
+
+def _repoint_params(section: Dict[str, Any]) -> Dict[str, Any]:
+    """The values of those fields, as the catalogue entry declares them."""
+    row = _section_row(section)
+    return {"id": section["id"], "js_path": row["js"], "css_path": row["css"],
+            "init_function": row["init"],
+            "icon": row["icon"], "handler_function": row["handler"],
+            "route_path": row["route"], "section_id": row["section_id"],
+            "admin_only": row["admin_only"], "nav_order": row["nav_order"]}
+
+
+def _section_row(section: Dict[str, Any], declared_by: str = "sections.json") -> Dict[str, Any]:
+    """A catalogue entry as the parameters of _INSERT_SECTION_IF_NEW."""
+    return {"id": section["id"], "name": section["name"],
+            "description": section.get("description"), "js": section.get("js"),
+            "css": section.get("css"), "init": section.get("initFunction"),
+            "icon": section.get("icon"),
+            "handler": section.get("handlerFunction"),
+            "route": section.get("routePath"),
+            "section_id": section.get("sectionId"),
+            "admin_only": bool(section.get("adminOnly", False)),
+            "nav_order": section.get("order"),
+            "declared_by": declared_by,
+            "version": section.get("version", "1.0.0"),
+            "config": json.dumps(section.get("config", {}))}
+
+
+def sync_framework_sections(path: str = FRAMEWORK_SECTIONS) -> Dict[str, int]:
+    """Bring the framework's own panel sections into the catalogue.
+
+    The application does not declare them: eight sections -- users, the section
+    catalogue itself, the cluster, metrics, themes, events, background tasks
+    and the integration log -- belong to the framework, and an application that
+    had to copy eight entries into its own file would get them wrong one at a
+    time (task keepup-4).
+
+    One rule here differs from sync_new_modules_from_json, and it is worth
+    saying plainly:
+
+        the framework owns *where the files of its sections are*;
+        the administrator owns *whether they are shown and to whom*.
+
+    So `js_path`, `css_path` and `init_function` are updated on rows that
+    already exist, while `is_active` and the grants are only ever inserted when
+    missing. Without the update, a deployment that has been running since
+    before the move keeps `/static/modules/js/users.js` in its database, the
+    file is not there any more, and the section breaks silently -- the panel
+    fetches a 404 and shows nothing.
+
+    `name` and `description` are left alone on an existing row: they are what a
+    person reads in the menu, and an administrator may have changed them on
+    purpose.
+
+    Args:
+        path: the catalogue to read; the package's own by default.
+
+    Returns:
+        How many rows were added and how many had their paths corrected.
+    """
+    from keepup_db import DatabaseManagerV2
+    counted = {"modules": 0, "repointed": 0, "grants": 0}
+    try:
+        with open(path, "r", encoding="utf-8") as catalogue:
+            declared = json.load(catalogue)
+    except Exception as error:
+        logger.error("Framework panel sections not synchronised: cannot read %s: %s",
+                     path, error)
+        return counted
+
+    for section in declared.get("modules", []):
+        try:
+            counted["modules"] += max(0, DatabaseManagerV2.execute_commit(
+                _INSERT_SECTION_IF_NEW, _section_row(section)))
+
+            counted["repointed"] += max(0, DatabaseManagerV2.execute_commit(
+                _repoint_statement(), _repoint_params(section)))
+        except Exception as error:
+            logger.warning("Framework section %s not synchronised: %s",
+                           section.get("id"), error)
+
+    for role in declared.get("roles", []):
+        for module_id in role.get("modules", []):
+            try:
+                counted["grants"] += max(0, DatabaseManagerV2.execute_commit(
+                    _INSERT_GRANT_IF_NEW, {"role": role.get("name"), "module_id": module_id}))
+            except Exception as error:
+                logger.warning("Framework grant %s/%s not added: %s",
+                               role.get("name"), module_id, error)
+
+    if any(counted.values()):
+        _sections_changed()
+    if counted["modules"] or counted["repointed"]:
+        logger.info("Framework panel sections: %s added, %s repointed",
+                    counted["modules"], counted["repointed"])
+    return counted
+
+
+def import_declared_sections(declared: Dict[str, List[Dict[str, Any]]],
+                             declared_by: str) -> Dict[str, int]:
+    """Import what one declarer says, with the same ownership rule as the file.
+
+    A plugin declares its sections through the `sections` contribution
+    (`get_panel_sections()`); they are imported here exactly as the framework's
+    own file is, and `declared_by` is what lets the read leave out the sections
+    of a plugin that is not running (keepup-113).
+    """
+    from keepup_db import DatabaseManagerV2
+
+    counted = {"modules": 0, "repointed": 0}
+    for section in declared.get("modules", []):
+        try:
+            row = _section_row(section, declared_by)
+            counted["modules"] += max(0, DatabaseManagerV2.execute_commit(
+                _INSERT_SECTION_IF_NEW, row))
+            counted["repointed"] += max(0, DatabaseManagerV2.execute_commit(
+                _repoint_statement(), _repoint_params(section)))
+        except Exception as error:
+            logger.warning("Section %s of %s not imported: %s",
+                           section.get("id"), declared_by, error)
+    if counted["modules"] or counted["repointed"]:
+        set_declared_plugins(_declaring_plugins | {declared_by})
+    return counted
+
+
+def sync_new_modules_from_json(path: str = None) -> Dict[str, int]:
+    """Add what the file declares and the database has never seen; change nothing else.
+
+    The catalogue of panel sections lives in the database, and the file is only its
+    seed. Until this ran at start-up, a section added to the file stayed invisible
+    after deployment until somebody remembered to call the import by hand. The full
+    import cannot run on its own: it rewrites every role's grants from the file and
+    so undoes what an administrator changed in the panel.
+
+    So only rows that do not exist at all are written: a module whose id has no row,
+    a grant whose (role, module) pair has no row. A section an administrator switched
+    off or took away from a role keeps its row, and stays the way he left it.
+
+    Args:
+        path: the catalogue to read; the deployment's own by default, which is
+            the path the application named (configure()).
+    """
+    from keepup_db import DatabaseManagerV2
+    path = path or MODULES_CONFIG_PATH
+    added = {"modules": 0, "grants": 0}
+    try:
+        with open(path, 'r', encoding='utf-8') as f:
+            config = json.load(f)
+    except Exception as e:
+        logger.warning(f"Panel sections not synchronised: cannot read {path}: {e}")
+        return added
+
+    for module in config.get('modules', []):
+        try:
+            added["modules"] += max(0, DatabaseManagerV2.execute_commit(
+                _INSERT_SECTION_IF_NEW,
+                _section_row(module, path or MODULES_CONFIG_PATH)))
+        except Exception as e:
+            logger.warning(f"Panel section {module.get('id')} not added: {e}")
+
+    for role in config.get('roles', []):
+        for module_id in role.get('modules', []):
+            try:
+                added["grants"] += max(0, DatabaseManagerV2.execute_commit(
+                    _INSERT_GRANT_IF_NEW, {"role": role.get('name'), "module_id": module_id}))
+            except Exception as e:
+                logger.warning(f"Panel section {module_id} not granted to {role.get('name')}: {e}")
+
+    if added["modules"] or added["grants"]:
+        _sections_changed()
+        logger.info(f"Panel sections from {path}: added {added['modules']} section(s) and "
+                    f"{added['grants']} role grant(s) the database did not have")
+    return added
+
+
+async def get_modules_from_json_fallback(current_user: dict):
+    """Load the modules from the JSON file when the database read fails."""
+    try:
+        with open(MODULES_CONFIG_PATH, 'r', encoding='utf-8') as f:
+            config = json.load(f)
+
+        held = set(user_roles.held_by(current_user))
+        allowed_module_names = set()
+        for role in config.get('roles', []):
+            if role.get('name') in held:
+                allowed_module_names.update(role.get('modules', []))
+
+        if not allowed_module_names:
+            return {"modules": []}
+
+        # The file's own order, and each module once: the same answer the
+        # database path gives, with the roles glued rather than one of them
+        # chosen (keepup-51).
+        available_modules = [
+            module for module in config.get('modules', [])
+            if module.get('id') in allowed_module_names
+        ]
+
+        return {"modules": available_modules}
+
+    except Exception as e:
+        logger.error(f"Error getting modules from JSON fallback: {str(e)}")
+        return {"modules": []}
+
+
+def register_module_routes(app):
+    """Register the panel-section endpoints on the application."""
+
+    @app.get("/api/admin/modules")
+    def get_all_modules_admin(admin: dict = Depends(get_current_admin)):
+        """Return every module (administrators only)."""
+        modules = get_all_modules_from_db()
+        return {"modules": modules}
+    @app.get("/api/admin/modules/{module_id}")
+    def get_module_admin(module_id: str, admin: dict = Depends(get_current_admin)):
+        """Return a module by id (administrators only)."""
+        module = get_module_by_id(module_id)
+        if not module:
+            raise HTTPException(status_code=404, detail="Module not found")
+        return module
+    @app.post("/api/admin/modules")
+    async def create_module(
+            module_data: ModuleCreate,
+            admin: dict = Depends(get_current_admin)
+    ):
+        """Create a module."""
+        try:
+            await asyncio.to_thread(create_or_update_module,
+                                    _catalogue_entry(module_data.module_id,
+                                                     module_data.model_dump()))
+            await admin_trail.record(admin_trail.SECTIONS_CHANGED, admin,
+                                     f"Section {module_data.module_id} created",
+                                     action="create", module_id=module_data.module_id)
+            return {"success": True, "message": "Module created successfully"}
+        except Exception as e:
+            logger.error("Could not save panel sections: %s", for_log(e))
+            raise HTTPException(status_code=400, detail="Could not save the panel sections; "
+                                                        "see the application log")
+    @app.put("/api/admin/modules/{module_id}")
+    async def update_module(
+            module_id: str,
+            module_data: ModuleUpdate,
+            admin: dict = Depends(get_current_admin)
+    ):
+        """Update a module."""
+        existing = await asyncio.to_thread(get_module_by_id, module_id)
+        if not existing:
+            raise HTTPException(status_code=404, detail="Module not found")
+
+        try:
+            update_data = module_data.model_dump(exclude_unset=True)
+            # Switching a section off is taking it away: the catalogue keeps the
+            # row and reads only active ones, which is what delete does.
+            if update_data.pop("is_active", None) is False:
+                await asyncio.to_thread(delete_module, module_id)
+            if update_data:
+                await asyncio.to_thread(create_or_update_module,
+                                        _changed_entry(existing, update_data))
+            await admin_trail.record(admin_trail.SECTIONS_CHANGED, admin,
+                                     f"Section {module_id} updated", action="update",
+                                     module_id=module_id,
+                                     fields=sorted(k for k in update_data if k != "module_id"))
+            return {"success": True, "message": "Module updated successfully"}
+        except Exception as e:
+            logger.error("Could not save panel sections: %s", for_log(e))
+            raise HTTPException(status_code=400, detail="Could not save the panel sections; "
+                                                        "see the application log")
+    @app.delete("/api/admin/modules/{module_id}")
+    async def delete_module_endpoint(
+            module_id: str,
+            admin: dict = Depends(get_current_admin)
+    ):
+        """Delete a module."""
+        existing = await asyncio.to_thread(get_module_by_id, module_id)
+        if not existing:
+            raise HTTPException(status_code=404, detail="Module not found")
+
+        await asyncio.to_thread(delete_module, module_id)
+        await admin_trail.record(admin_trail.SECTIONS_CHANGED, admin,
+                                 f"Section {module_id} deleted", action="delete",
+                                 module_id=module_id)
+        return {"success": True, "message": "Module deleted successfully"}
+    @app.get("/api/admin/role-modules")
+    def get_role_modules(admin: dict = Depends(get_current_admin)):
+        """Return the modules granted to each role."""
+        from keepup_db import DatabaseManagerV2
+        roles_modules = {}
+        roles = DatabaseManagerV2.execute('''
+        SELECT DISTINCT role_name FROM role_modules WHERE is_active = TRUE
+        ''')
+
+        for role in roles:
+            modules = get_modules_for_role(role['role_name'])
+            roles_modules[role['role_name']] = modules
+
+        return roles_modules
+    @app.post("/api/admin/role-modules")
+    async def update_role_modules_endpoint(
+            role_data: RoleModulesUpdate,
+            admin: dict = Depends(get_current_admin)
+    ):
+        """Update the modules granted to a role."""
+        try:
+            await asyncio.to_thread(update_role_modules, role_data.role_name,
+                                    role_data.module_ids)
+            await admin_trail.record(admin_trail.SECTIONS_CHANGED, admin,
+                                     f"Sections of role {role_data.role_name} set",
+                                     action="grant", role_name=role_data.role_name,
+                                     module_ids=list(role_data.module_ids))
+            return {"success": True, "message": "Role modules updated successfully"}
+        except Exception as e:
+            logger.error("Could not save panel sections: %s", for_log(e))
+            raise HTTPException(status_code=400, detail="Could not save the panel sections; "
+                                                        "see the application log")
+    @app.post("/api/admin/modules/import-from-json")
+    async def import_modules_from_json_endpoint(admin: dict = Depends(get_current_admin)):
+        """Import the modules from the JSON file."""
+        success = await asyncio.to_thread(import_modules_from_json)
+        if success:
+            await admin_trail.record(admin_trail.SECTIONS_CHANGED, admin,
+                                     "Sections and grants reset from the file",
+                                     action="import")
+            return {"success": True, "message": "Modules imported successfully"}
+        else:
+            raise HTTPException(status_code=500, detail="Error importing modules")
+    @app.post("/api/admin/modules/export-to-json")
+    def export_modules_to_json_endpoint(admin: dict = Depends(get_current_admin)):
+        """Export the modules to the JSON file."""
+        from keepup_db import DatabaseManagerV2
+        try:
+            modules = get_all_modules_from_db()
+            roles_modules = {}
+            roles = DatabaseManagerV2.execute('SELECT DISTINCT role_name FROM role_modules WHERE is_active = TRUE')
+
+            for role in roles:
+                role_modules = get_modules_for_role(role['role_name'])
+                roles_modules[role['role_name']] = [module['module_id'] for module in role_modules]
+
+            export_data = {
+                "roles": [
+                    {
+                        "name": role_name,
+                        "modules": module_ids
+                    }
+                    for role_name, module_ids in roles_modules.items()
+                ],
+                "modules": [
+                    {
+                        "id": module['module_id'],
+                        "name": module['name'],
+                        "description": module['description'],
+                        "js": module['js_path'],
+                        "css": module['css_path'],
+                        "initFunction": module['init_function'],
+                        "version": module['version'],
+                        "config": json.loads(module['config']) if module['config'] else {}
+                    }
+                    for module in modules
+                ]
+            }
+
+            with open(MODULES_CONFIG_PATH, 'w', encoding='utf-8') as f:
+                json.dump(export_data, f, ensure_ascii=False, indent=2)
+
+            return {"success": True, "message": "Modules exported successfully"}
+        except Exception as e:
+            logger.error(f"Error exporting modules to JSON: {str(e)}")
+            raise HTTPException(status_code=500, detail="Error exporting modules")
+    @app.get("/api/modules")
+    async def get_modules(current_user: dict = Depends(get_current_user)):
+        """Return the modules available to the current user, from the database."""
+        try:
+            # Every page of the panel asks for this; the read leaves the loop.
+            modules = await asyncio.to_thread(get_modules_for_roles, user_roles.held_by(current_user))
+
+            if modules:
+                formatted_modules = []
+                for module in modules:
+                    formatted_module = {
+                        "id": module['module_id'],
+                        "name": module['name'],
+                        "description": module['description'],
+                        "js": module['js_path'],
+                        "css": module['css_path'],
+                        "initFunction": module['init_function'],
+                        "version": module['version']
+                    }
+                    if module['config']:
+                        formatted_module["config"] = json.loads(module['config'])
+                    formatted_modules.append(formatted_module)
+
+                return {"modules": formatted_modules}
+            else:
+                logger.warning("No modules found in DB, falling back to JSON file")
+                return await get_modules_from_json_fallback(current_user)
+
+        except Exception as e:
+            logger.error(f"Error getting modules from DB: {str(e)}")
+            return await get_modules_from_json_fallback(current_user)