queryapigate 0.5.0__py3-none-any.whl

queryapigate/store.py

@@ -0,0 +1,433 @@
+"""Persistence: the connection registry and the saved-query files (plain JSON on disk).
+
+Everything that touches those files goes through here so that access stays confined to the
+configured home folder and read-modify-write cycles are serialised.
+"""
+import json
+import os
+import re
+import threading
+import uuid
+from datetime import datetime
+
+from . import config
+from .errors import ApiError
+
+# Serialises read-modify-write cycles on the JSON files this app manages.
+lock = threading.RLock()
+
+_ENV_REF_RE = re.compile(r'\$\{(\w+)\}')
+_FILENAME_RE = re.compile(r'^[\w .\-]{1,100}$')
+_ENC_PREFIX = 'enc:'  # marks a connection password as Fernet-encrypted - see _encrypt_password()
+
+
+def write_json_atomic(path, data):
+    tmp_path = f'{path}.{uuid.uuid4().hex}.tmp'
+    with open(tmp_path, 'w') as f:
+        json.dump(data, f, indent=4)
+    os.replace(tmp_path, path)
+
+
+def now():
+    return datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+
+
+# --------------------------------------------------------------------------------------
+# Audit log: a durable record of administrative changes (API keys, connections, saved
+# queries) - distinct from execution_history below (query *runs*) and from the ephemeral,
+# stdout-only access log (app.decorate()). See app.py's route handlers for what calls this
+# and with what `changes` shape; this module only persists whatever it's given.
+# --------------------------------------------------------------------------------------
+
+def record_audit(actor, action, target, changes=None):
+    """Append one entry (newest last, capped at config.audit_log_limit() - same shape as execution_history
+    below) and, when QUERYAPIGATE_AUDIT_LOG_EXPORT_FILE is set, also append it there - a second, never-capped
+    copy for retention beyond audit_log.json's own rolling window. Never raises: an audit-log write failing
+    should not block the action it's recording, the same trade-off record_execution() already makes for
+    query-run history; the two writes are independently fault-tolerant so a problem with one (e.g. the
+    export path's directory missing) never suppresses the other."""
+    entry = {'timestamp': now(), 'actor': actor, 'action': action, 'target': target, 'changes': changes}
+    try:
+        with lock:
+            path = config.audit_log_file()
+            try:
+                with open(path, 'r') as f:
+                    entries = json.load(f).get('entries', [])
+            except (FileNotFoundError, json.JSONDecodeError):
+                entries = []
+            entries.append(entry)
+            del entries[:-config.audit_log_limit()]
+            config.home().mkdir(parents=True, exist_ok=True)
+            write_json_atomic(path, {'entries': entries})
+    except OSError:
+        pass
+    export_path = config.audit_log_export_file()
+    if export_path is not None:
+        try:
+            with lock, open(export_path, 'a') as f:
+                f.write(json.dumps(entry, default=str) + '\n')
+        except OSError:
+            pass
+
+
+def read_audit_log():
+    try:
+        with open(config.audit_log_file(), 'r') as f:
+            return json.load(f).get('entries', [])
+    except (FileNotFoundError, json.JSONDecodeError):
+        return []
+
+
+# --------------------------------------------------------------------------------------
+# Connections
+# --------------------------------------------------------------------------------------
+
+def read_connections():
+    try:
+        with open(config.connections_file(), 'r') as f:
+            data = json.load(f)
+    except FileNotFoundError:
+        return {}
+    return data.get('connections', {})
+
+
+def _expand_env(value):
+    """Replace ${VAR} references in a connection value with the environment variable's content."""
+    if not isinstance(value, str):
+        return value
+
+    def replace(match):
+        name = match.group(1)
+        if name not in os.environ:
+            raise ApiError(f"Environment variable '{name}' referenced by the connection is not set", 500)
+        return os.environ[name]
+
+    return _ENV_REF_RE.sub(replace, value)
+
+
+def _is_env_ref(value):
+    return isinstance(value, str) and bool(_ENV_REF_RE.fullmatch(value))
+
+
+def _is_encrypted(value):
+    return isinstance(value, str) and value.startswith(_ENC_PREFIX)
+
+
+def _get_fernet():
+    """The configured Fernet instance for connection-password encryption at rest (QUERYAPIGATE_SECRET_KEY), or
+    None when unset - encryption at rest is opt-in. The key's format is already validated at startup
+    (config.check_settings()), so a Fernet() call here should not itself fail."""
+    key = config.secret_key()
+    if not key:
+        return None
+    from cryptography.fernet import Fernet
+    return Fernet(key.encode('utf-8'))
+
+
+def _encrypt_password(value):
+    """Encrypt a literal connection password for storage, if QUERYAPIGATE_SECRET_KEY is set - a ${VAR} reference
+    or an already-encrypted value passes through unchanged (never double-encrypted, and a ${VAR} reference
+    is not a secret stored in this file at all). Returns the value unchanged when no key is configured -
+    today's behaviour, unaffected without opting in."""
+    if not isinstance(value, str) or not value or _is_env_ref(value) or _is_encrypted(value):
+        return value
+    fernet = _get_fernet()
+    if fernet is None:
+        return value
+    return _ENC_PREFIX + fernet.encrypt(value.encode('utf-8')).decode('ascii')
+
+
+def _decrypt_password(value):
+    """Decrypt a stored connection password for actual use, at the moment a connection is opened
+    (get_connection()) - never held decrypted anywhere else. Fails clearly rather than silently: a missing
+    or rotated QUERYAPIGATE_SECRET_KEY must not pass ciphertext to the driver, or silently fall back to treating
+    it as a literal, the same "fail closed, say why" precedent is_expired() already sets for a malformed
+    stored value that could otherwise fail dangerously quiet."""
+    if not _is_encrypted(value):
+        return value
+    fernet = _get_fernet()
+    if fernet is None:
+        raise ApiError("This connection's password is encrypted but QUERYAPIGATE_SECRET_KEY is not set - it "
+                       'cannot be decrypted', 500)
+    from cryptography.fernet import InvalidToken
+    try:
+        return fernet.decrypt(value[len(_ENC_PREFIX):].encode('ascii')).decode('utf-8')
+    except InvalidToken:
+        raise ApiError("This connection's password could not be decrypted - QUERYAPIGATE_SECRET_KEY may have "
+                       'been rotated since it was encrypted', 500) from None
+
+
+def get_connection(connection_name):
+    """Return the usable (active, env-expanded, password-decrypted) details of a named connection."""
+    if not isinstance(connection_name, str) or not connection_name:
+        raise ApiError('Connection name is missing')
+    details = read_connections().get(connection_name)
+    if not details:
+        raise ApiError(f"Connection '{connection_name}' not found", 404)
+    if not details.get('active', False):
+        raise ApiError(f"The connection '{connection_name}' is currently not active. "
+                       "To use it, it must be set to active.", 403)
+    if details.get('db') not in config.SUPPORTED_DB_TYPES:
+        raise ApiError('Unsupported database type')
+    resolved = {key: _expand_env(value) for key, value in details.items()}
+    if 'password' in resolved:
+        resolved['password'] = _decrypt_password(resolved['password'])
+    return resolved
+
+
+def _is_plaintext_password(password):
+    """A literal, unprotected password - not a ${VAR} reference and not already encrypted at rest. Used
+    only for the startup warning and the auto-migration on start (plaintext_password_connections(),
+    encrypt_plaintext_passwords_in_place()) - see mask_passwords() below for the broader "is this a real
+    secret value" check used for API responses, which masks an encrypted value too."""
+    return bool(password) and not _is_env_ref(password) and not _is_encrypted(password)
+
+
+def mask_passwords(connections):
+    """Hide any real secret value in API responses - a literal password and an encrypted one both mask to
+    the same PASSWORD_MASK; only a ${VAR} reference (never a secret itself, just a pointer to one) is shown
+    as written."""
+    def mask(conn):
+        password = conn.get('password')
+        if password and not _is_env_ref(password):
+            return {**conn, 'password': config.PASSWORD_MASK}
+        return conn
+
+    return {name: mask(conn) for name, conn in connections.items()}
+
+
+def plaintext_password_connections():
+    """Names of connections whose password is a literal string on disk rather than a ${VAR} reference to
+    an environment variable *or* already encrypted at rest - used only for the startup warning in
+    app.create_app(). An empty password (common for local/dev databases with none) is not flagged."""
+    return sorted(name for name, conn in read_connections().items() if _is_plaintext_password(conn.get('password')))
+
+
+def encrypt_plaintext_passwords_in_place():
+    """Called once at startup when QUERYAPIGATE_SECRET_KEY is set: encrypts any connection password that's still
+    a literal, so a connection saved before the key existed benefits immediately rather than waiting for its
+    next PATCH /connections - "don't require a one-time manual migration step" from the start."""
+    with lock:
+        connections = read_connections()
+        changed = False
+        for details in connections.values():
+            password = details.get('password')
+            if _is_plaintext_password(password):
+                details['password'] = _encrypt_password(password)
+                changed = True
+        if changed:
+            write_json_atomic(config.connections_file(), {'connections': connections})
+
+
+def encrypted_password_connections():
+    """Names of connections whose password is encrypted at rest - used only for the startup warning that
+    fires when QUERYAPIGATE_SECRET_KEY is missing but encrypted passwords already exist on disk (a rotated-out
+    or removed key, most likely): those connections cannot be used until the key is restored."""
+    return sorted(name for name, conn in read_connections().items() if _is_encrypted(conn.get('password')))
+
+
+def update_connections(connections):
+    for name, details in connections.items():
+        if not isinstance(details, dict) or details.get('db') not in config.SUPPORTED_DB_TYPES:
+            raise ApiError(f"Connection '{name}' must be an object whose 'db' is one of: "
+                           f"{', '.join(config.SUPPORTED_DB_TYPES)}")
+    with lock:
+        existing = read_connections()
+        for name, details in connections.items():
+            # GET masks passwords, so a client echoing the mask back means "keep the current one" - reused
+            # exactly as stored (already encrypted, if it was), never re-encrypted.
+            if details.get('password') == config.PASSWORD_MASK and name in existing:
+                details = {**details, 'password': existing[name].get('password', '')}
+            elif 'password' in details:
+                # A genuinely new literal password (or ${VAR} reference, which _encrypt_password() passes
+                # through unchanged) - encrypted here if QUERYAPIGATE_SECRET_KEY is set, stored as given otherwise.
+                details = {**details, 'password': _encrypt_password(details['password'])}
+            existing[name] = details
+        config.home().mkdir(parents=True, exist_ok=True)
+        write_json_atomic(config.connections_file(), {'connections': existing})
+
+
+def delete_connection(name):
+    with lock:
+        existing = read_connections()
+        if name not in existing:
+            raise ApiError(f"Connection '{name}' not found", 404)
+        del existing[name]
+        write_json_atomic(config.connections_file(), {'connections': existing})
+
+
+# --------------------------------------------------------------------------------------
+# Saved queries (all access is confined to the saved_sql folder)
+# --------------------------------------------------------------------------------------
+
+def saved_path_for_name(filename):
+    """Path of the saved-query file for a client-supplied name (without extension)."""
+    if not isinstance(filename, str) or not _FILENAME_RE.match(filename) or filename.startswith('.'):
+        raise ApiError("Filename may only contain letters, digits, spaces, '.', '_' and '-'")
+    return os.path.join(config.saved_sql_dir(), f'{filename}.json')
+
+
+def resolve_saved_file(ref):
+    """Resolve a client-supplied file reference to an existing .json file inside the saved_sql folder.
+
+    Accepts a bare name ("cht" or "cht.json"), a path relative to the home folder
+    ("saved_sql/cht.json") or an absolute path - as long as it ends up in the saved_sql folder.
+    """
+    if not isinstance(ref, str) or not ref.strip():
+        raise ApiError('Filename is missing')
+    saved_dir = str(config.saved_sql_dir())
+    if os.path.isabs(ref):
+        candidate = ref
+    elif os.sep in ref or '/' in ref:
+        candidate = os.path.join(config.home(), ref)
+    else:
+        candidate = os.path.join(saved_dir, ref if ref.endswith('.json') else f'{ref}.json')
+    candidate = os.path.realpath(candidate)
+    if os.path.dirname(candidate) != os.path.realpath(saved_dir) or not candidate.endswith('.json'):
+        raise ApiError('Only .json files inside the saved_sql folder can be accessed', 403)
+    if not os.path.isfile(candidate):
+        raise ApiError('File not found', 404)
+    return candidate
+
+
+def query_name(path):
+    """The canonical saved-query name for a path resolve_saved_file() returned - the same string an admin
+    types into an API key's ``queries`` grant (see apikeys.py), regardless of which of resolve_saved_file()'s
+    accepted reference forms (bare name, relative path, absolute path) the caller originally used."""
+    return os.path.splitext(os.path.basename(path))[0]
+
+
+def load_versions(path):
+    try:
+        with open(path, 'r') as f:
+            content = json.load(f)
+    except json.JSONDecodeError:
+        raise ApiError('Saved query file is not valid JSON', 500) from None
+    if not isinstance(content, dict):
+        raise ApiError('Saved query file has an unexpected structure', 500)
+    return content
+
+
+def version_numbers(content):
+    return [int(v) for v in content if v.isdigit()]
+
+
+def select_version(content, version=None):
+    """Return (number, data) for the requested version, or the latest one when none is given."""
+    if version is None:
+        numbers = version_numbers(content)
+        if not numbers:
+            raise ApiError('No valid versions found in the file')
+        version = max(numbers)
+    data = content.get(str(version))
+    if not isinstance(data, dict):
+        raise ApiError(f'Version {version} not found', 404)
+    return int(version), data
+
+
+def save_version(filename, fields):
+    """Store ``fields`` as the next version of a saved query; returns (uuid, version number)."""
+    path = saved_path_for_name(filename)
+    query_uuid = str(uuid.uuid4())
+    timestamp = now()
+    with lock:
+        os.makedirs(config.saved_sql_dir(), exist_ok=True)
+        versions = load_versions(path) if os.path.exists(path) else {}
+        number = max(version_numbers(versions), default=0) + 1
+        versions[str(number)] = {
+            'uuid': query_uuid,
+            **fields,
+            'created_at': timestamp,
+            'last_modified_at': timestamp,
+            'status': 'active',
+            'version': number,
+            'execution_history': [],
+        }
+        write_json_atomic(path, versions)
+    return query_uuid, number
+
+
+def delete_saved(ref, version=None):
+    """Delete a saved query file, or a single version of it (the file goes with its last version)."""
+    path = resolve_saved_file(ref)
+    with lock:
+        if version is None:
+            os.remove(path)
+            return
+        content = load_versions(path)
+        if str(version) not in content:
+            raise ApiError(f'Version {version} not found', 404)
+        del content[str(version)]
+        if version_numbers(content):
+            write_json_atomic(path, content)
+        else:
+            os.remove(path)
+
+
+def record_execution(path, version, entry):
+    """Append to a saved version's execution_history (newest last, capped). Never raises."""
+    try:
+        with lock:
+            content = load_versions(path)
+            data = content.get(str(version))
+            if not isinstance(data, dict):
+                return
+            history = data.setdefault('execution_history', [])
+            history.append(entry)
+            del history[:-config.HISTORY_LIMIT]
+            write_json_atomic(path, content)
+    except (OSError, ApiError):
+        pass
+
+
+def latest_versions():
+    """(name, version number, data) for the newest version of every saved query; unreadable files are skipped."""
+    saved_dir = str(config.saved_sql_dir())
+    if not os.path.isdir(saved_dir):
+        return []
+    found = []
+    for filename in sorted(os.listdir(saved_dir)):
+        path = os.path.join(saved_dir, filename)
+        if not (filename.endswith('.json') and os.path.isfile(path)):
+            continue
+        try:
+            number, data = select_version(load_versions(path))
+        except ApiError:
+            continue
+        found.append((filename[:-5], number, data))
+    return found
+
+
+def list_saved():
+    """Return every saved query with its versions' metadata (SQL text is not included).
+
+    A saved_sql folder that does not exist yet (nothing has ever been saved) is an empty list, not an
+    error - consistent with latest_versions() above and with how a list endpoint should behave.
+    """
+    saved_dir = str(config.saved_sql_dir())
+    if not os.path.isdir(saved_dir):
+        return []
+    files = []
+    for filename in os.listdir(saved_dir):
+        file_path = os.path.join(saved_dir, filename)
+        if not (os.path.isfile(file_path) and filename.endswith('.json')):
+            continue
+        try:
+            content = load_versions(file_path)
+        except ApiError:
+            continue
+        versions = [{
+            'version': int(version),
+            'author': data.get('author'),
+            'description': data.get('description'),
+            'tags': data.get('tags', []),
+            'query_parameters': data.get('query_parameters', {}),
+            'connection_name': data.get('connection_name'),
+            'created_at': data.get('created_at'),
+            'last_modified_at': data.get('last_modified_at'),
+            'status': data.get('status'),
+            'execution_history': data.get('execution_history', []),
+        } for version, data in content.items() if version.isdigit() and isinstance(data, dict)]
+        versions.sort(key=lambda v: v['version'])
+        files.append({'filename': filename[:-5], 'versions': versions})
+    return files