psdi-data-conversion 0.1.7__py3-none-any.whl → 0.2.1__py3-none-any.whl

psdi_data_conversion/gui/accessibility.py

@@ -0,0 +1,51 @@
+"""
+# post.py
+
+This module defines the various web addresses which do something (the "POST" methods) provided by the website,
+connecting them to relevant functions.
+"""
+
+
+import json
+
+from flask import Flask, make_response, render_template, request
+
+from psdi_data_conversion import constants as const
+from psdi_data_conversion.gui.env import get_env_kwargs
+
+
+def accessibility():
+    """Return the accessibility page
+    """
+    return render_template("accessibility.htm",
+                           **get_env_kwargs())
+
+
+def save_accessibility():
+    """Save the user's accessibility settings in a cookie
+    """
+
+    resp = make_response("Cookie saved successfully")
+
+    d_settings: dict[str, str] = json.loads(request.form['data'])
+
+    for key, val in d_settings.items():
+        resp.set_cookie(key, val, max_age=const.YEAR)
+
+    return resp
+
+
+def load_accessibility():
+    """Load the user's accessibility settings from the cookie
+    """
+    return json.dumps(request.cookies)
+
+
+def init_accessibility(app: Flask):
+    """Connect the provided Flask app to each of the post methods
+    """
+
+    app.route('/accessibility.htm')(accessibility)
+
+    app.route('/save_accessibility/', methods=["POST"])(save_accessibility)
+    app.route('/load_accessibility/', methods=["GET"])(load_accessibility)

psdi_data_conversion/gui/env.py

@@ -0,0 +1,239 @@
+"""
+# env.py
+
+This module handles setting up and storing the state of the environment for the website, e.g. environmental variables
+"""
+
+import os
+import sys
+from argparse import Namespace
+from datetime import datetime
+from hashlib import md5
+from subprocess import run
+from traceback import format_exc
+from typing import TypeVar
+
+from psdi_data_conversion import constants as const
+from psdi_data_conversion import log_utility
+
+# Env var for the tag and SHA of the latest commit
+TAG_EV = "TAG"
+TAG_SHA_EV = "TAG_SHA"
+SHA_EV = "SHA"
+
+# Env var for whether this is a production release or development
+PRODUCTION_EV = "PRODUCTION_MODE"
+
+# Env var for whether this is a production release or development
+DEBUG_EV = "DEBUG_MODE"
+
+
+class SiteEnv:
+    def __init__(self, args: Namespace | None = None):
+
+        self._args = args
+        """The parsed arguments provided to the script to start the server"""
+
+        self.log_mode: str = self._determine_log_mode()
+        """The logging mode"""
+
+        self.log_level: str = self._determine_log_level()
+        """The logging level"""
+
+        self.max_file_size = self._determine_value(ev=const.MAX_FILESIZE_EV,
+                                                   arg="max_file_size",
+                                                   default=const.DEFAULT_MAX_FILE_SIZE /
+                                                   const.MEGABYTE)*const.MEGABYTE
+        """The maximum file size for converters other than Open Babel"""
+
+        self.max_file_size_ob = self._determine_value(ev=const.MAX_FILESIZE_OB_EV,
+                                                      arg="max_file_size_ob",
+                                                      default=const.DEFAULT_MAX_FILE_SIZE_OB /
+                                                      const.MEGABYTE)*const.MEGABYTE
+        """The maximum file size for the Open Babel converter"""
+
+        self.service_mode = self._determine_value(ev=const.SERVICE_MODE_EV,
+                                                  arg="service_mode",
+                                                  value_type=bool,
+                                                  default=False)
+        """True if the app is running in service mode, False if it's running in local mode"""
+
+        self.production_mode = self._determine_value(ev=PRODUCTION_EV,
+                                                     arg="!dev_mode",
+                                                     value_type=bool,
+                                                     default=False)
+        """True if the app is running in production mode, False if it's running in developmennt mode"""
+
+        self.debug_mode = self._determine_value(ev=DEBUG_EV,
+                                                arg="debug",
+                                                value_type=bool,
+                                                default=False)
+        """True if the app is running in debug mode, False if not"""
+
+        tag, sha = self._determine_tag_and_sha()
+
+        self.tag: str = tag
+        """The latest tag in the repo"""
+
+        self.sha: str = sha
+        """The SHA of the latest commit, if the latest commit isn't tagged, otherwise an empty string"""
+
+        dt = str(datetime.now())
+        self.token = md5(dt.encode('utf8')).hexdigest()
+        """A token for this session, created by hashing the the current date and time"""
+
+        self._kwargs: dict[str, str] | None = None
+        """Cached value for dict containing all env values"""
+
+    @property
+    def kwargs(self) -> dict[str, str]:
+        """Get a dict which can be used to provide kwargs for rendering a template"""
+        if not self._kwargs:
+            self._kwargs = {}
+            for key, val in self.__dict__.items():
+                if not key.startswith("_"):
+                    self._kwargs[key] = val
+        return self._kwargs
+
+    def _determine_log_mode(self) -> str:
+        """Determine the log mode from args and environmental variables, preferring the former"""
+        if self._args:
+            return self._args.log_mode
+
+        ev_log_mode = os.environ.get(const.LOG_MODE_EV)
+        if ev_log_mode is None:
+            return const.LOG_MODE_DEFAULT
+
+        ev_log_mode = ev_log_mode.lower()
+        if ev_log_mode not in const.L_ALLOWED_LOG_MODES:
+            raise ValueError(f"ERROR: Unrecognised logging option: {ev_log_mode}. "
+                             f"Allowed options are: {const.L_ALLOWED_LOG_MODES}")
+
+        return ev_log_mode
+
+    def _determine_log_level(self) -> str | None:
+        """Determine the log level from args and environmental variables, preferring the former"""
+        if self._args:
+            return self._args.log_level
+
+        ev_log_level = os.environ.get(const.LOG_LEVEL_EV)
+        if ev_log_level is None:
+            return None
+
+        return log_utility.get_log_level_from_str(ev_log_level)
+
+    T = TypeVar('T')
+
+    def _determine_value(self,
+                         ev: str,
+                         arg: str,
+                         value_type: type[T] = float,
+                         default: T = None) -> T | None:
+        """Determine a value using input arguments (preferred if present) and environmental variables"""
+        if self._args and arg:
+            # Special handling for bool, which allows flipping the value of an arg
+            if value_type is bool and arg.startswith("!"):
+                return not getattr(self._args, arg[1:])
+            return getattr(self._args, arg)
+
+        ev_value = os.environ.get(ev)
+        if not ev_value:
+            return default
+
+        # Special handling for bool, to properly parse strings into bools
+        if value_type is bool:
+            return ev_value.lower().startswith("t")
+
+        return value_type(ev_value)
+
+    def _determine_tag_and_sha(self) -> tuple[str, str]:
+        """Get latest tag and SHA of latest commit, if the latest commit differs from the latest tagged commit
+        """
+
+        # Get the tag of the latest commit
+        ev_tag = os.environ.get(TAG_EV)
+        if ev_tag:
+            tag = ev_tag
+        else:
+            try:
+                # This bash command calls `git tag` to get a sorted list of tags, with the most recent at the top, then
+                # uses `head` to trim it to one line
+                cmd = "git tag --sort -version:refname | head -n 1"
+
+                out_bytes = run(cmd, shell=True, capture_output=True).stdout
+                tag = str(out_bytes.decode()).strip()
+
+            except Exception:
+                # Failsafe exception block, since this is reasonably likely to occur (e.g. due to a shallow fetch of the
+                # repo, and we don't want to crash the whole app because of it)
+                print("ERROR: Could not determine most recent tag. Error was:\n" + format_exc(),
+                      file=sys.stderr)
+                tag = ""
+
+        # Get the SHA associated with this tag
+        ev_tag_sha = os.environ.get(TAG_SHA_EV)
+        if ev_tag_sha:
+            tag_sha: str | None = ev_tag_sha
+        else:
+            try:
+                cmd = f"git show {tag}" + " | head -n 1 | gawk '{print($2)}'"
+
+                out_bytes = run(cmd, shell=True, capture_output=True).stdout
+                tag_sha = str(out_bytes.decode()).strip()
+
+            except Exception:
+                # Another failsafe block, same reason as before
+                print("ERROR: Could not determine SHA for most recent tag. Error was:\n" + format_exc(),
+                      file=sys.stderr)
+                tag_sha = None
+
+        # First check if the SHA is provided through an environmental variable
+        ev_sha = os.environ.get(SHA_EV)
+        if ev_sha:
+            sha = ev_sha
+        else:
+            try:
+                # This bash command calls `git log` to get info on the last commit, uses `head` to trim it to one line,
+                # then uses `gawk` to get just the second word of this line, which is the SHA of this commit
+                cmd = "git log -n 1 | head -n 1 | gawk '{print($2)}'"
+
+                out_bytes = run(cmd, shell=True, capture_output=True).stdout
+                sha = str(out_bytes.decode()).strip()
+
+            except Exception:
+                # Another failsafe block, same reason as before
+                print("ERROR: Could not determine SHA of most recent commit. Error was:\n" + format_exc(),
+                      file=sys.stderr)
+                sha = ""
+
+        # If the SHA of the tag is the same as the current SHA, we indicate this by returning a blank SHA
+        if tag_sha == sha:
+            sha = ""
+
+        return (tag, sha)
+
+
+_env: SiteEnv | None = None
+
+
+def get_env():
+    """Get a reference to the global `SiteEnv` object, creating it if necessary.
+    """
+    global _env
+    if not _env:
+        _env = SiteEnv()
+    return _env
+
+
+def update_env(args: Namespace | None = None):
+    """Update the global `SiteEnv` object, optionally using arguments passed at the command-line to override values
+    passed through environmental variables.
+    """
+    global _env
+    _env = SiteEnv(args)
+
+
+def get_env_kwargs():
+    """Get a dict of common kwargs for the environment
+    """
+    return get_env().kwargs

psdi_data_conversion/gui/get.py

@@ -0,0 +1,53 @@
+"""
+# get.py
+
+This module defines the various webpages (the "GET" methods) provided by the website, connecting them to relevant
+functions to return rendered templates.
+"""
+
+
+from flask import Flask, render_template
+
+from psdi_data_conversion.database import get_database_path
+from psdi_data_conversion.gui.env import get_env_kwargs
+
+
+def index():
+    """Return the web page along with relevant data
+    """
+    return render_template("index.htm",
+                           **get_env_kwargs())
+
+
+def documentation():
+    """Return the documentation page
+    """
+    return render_template("documentation.htm",
+                           **get_env_kwargs())
+
+
+def database():
+    """Return the raw database JSON file
+    """
+    return open(get_database_path(), "r").read()
+
+
+def report():
+    """Return the report page
+    """
+    return render_template("report.htm",
+                           **get_env_kwargs())
+
+
+def init_get(app: Flask):
+    """Connect the provided Flask app to each of the pages on the site
+    """
+
+    app.route('/')(index)
+    app.route('/index.htm')(index)
+
+    app.route('/documentation.htm')(documentation)
+
+    app.route('/report.htm')(report)
+
+    app.route('/database/')(database)

psdi_data_conversion/gui/post.py

@@ -0,0 +1,176 @@
+"""
+# post.py
+
+This module defines the various web addresses which do something (the "POST" methods) provided by the website,
+connecting them to relevant functions.
+"""
+
+
+import json
+import os
+import sys
+from multiprocessing import Lock
+from traceback import format_exc
+
+from flask import Flask, Response, abort, request
+from werkzeug.utils import secure_filename
+
+from psdi_data_conversion import constants as const
+from psdi_data_conversion import log_utility
+from psdi_data_conversion.converter import run_converter
+from psdi_data_conversion.database import get_format_info
+from psdi_data_conversion.file_io import split_archive_ext
+from psdi_data_conversion.gui.env import get_env
+
+# Key for the label given to the file uploaded in the web interface
+FILE_TO_UPLOAD_KEY = 'fileToUpload'
+
+# A lock to prevent multiple threads from logging at the same time
+logLock = Lock()
+
+
+def convert():
+    """Convert file to a different format and save to folder 'downloads'. Delete original file. Note that downloading is
+    achieved in format.js
+    """
+
+    env = get_env()
+
+    # Make sure the upload directory exists
+    os.makedirs(const.DEFAULT_INPUT_DIR, exist_ok=True)
+
+    file = request.files[FILE_TO_UPLOAD_KEY]
+    filename = secure_filename(file.filename)
+
+    qualified_filename = os.path.join(const.DEFAULT_INPUT_DIR, filename)
+    file.save(qualified_filename)
+    qualified_output_log = os.path.join(const.DEFAULT_OUTPUT_DIR,
+                                        split_archive_ext(filename)[0] + const.OUTPUT_LOG_EXT)
+
+    # Determine the input and output formats
+    d_formats = {}
+    for format_label in "to", "from":
+        name = request.form[format_label]
+        full_note = request.form[format_label+"_full"]
+
+        l_possible_formats = get_format_info(name, which="all")
+
+        # If there's only one possible format, use that
+        if len(l_possible_formats) == 1:
+            d_formats[format_label] = l_possible_formats[0]
+            continue
+
+        # Otherwise, find the format with the matching note
+        for possible_format in l_possible_formats:
+            if possible_format.note in full_note:
+                d_formats[format_label] = possible_format
+                break
+        else:
+            print(f"Format '{name}' with full description '{full_note}' could not be found in database.",
+                  file=sys.stderr)
+            abort(const.STATUS_CODE_GENERAL)
+
+    if (not env.service_mode) or (request.form['token'] == env.token and env.token != ''):
+        try:
+            conversion_output = run_converter(name=request.form['converter'],
+                                              filename=qualified_filename,
+                                              data=request.form,
+                                              to_format=d_formats["to"],
+                                              from_format=d_formats["from"],
+                                              strict=(request.form['check_ext'] != "false"),
+                                              log_mode=env.log_mode,
+                                              log_level=env.log_level,
+                                              delete_input=True,
+                                              abort_callback=abort)
+        except Exception as e:
+
+            # Check for anticipated exceptions, and write a simpler message for them
+            for err_message in (const.ERR_CONVERSION_FAILED, const.ERR_CONVERTER_NOT_RECOGNISED,
+                                const.ERR_EMPTY_ARCHIVE, const.ERR_WRONG_EXTENSIONS):
+                if log_utility.string_with_placeholders_matches(err_message, str(e)):
+                    with open(qualified_output_log, "w") as fo:
+                        fo.write(str(e))
+                    abort(const.STATUS_CODE_GENERAL)
+
+            # If the exception provides a status code, get it
+            status_code: int
+            if hasattr(e, "status_code"):
+                status_code = e.status_code
+            else:
+                status_code = const.STATUS_CODE_GENERAL
+
+            # If the exception provides a message, report it
+            if hasattr(e, "message"):
+                msg = f"An unexpected exception was raised by the converter, with error message:\n{e.message}\n"
+            else:
+                # Failsafe exception message
+                msg = ("The following unexpected exception was raised by the converter:\n" +
+                       format_exc()+"\n")
+            with open(qualified_output_log, "w") as fo:
+                fo.write(msg)
+            abort(status_code)
+
+        return repr(conversion_output)
+    else:
+        # return http status code 405
+        abort(405)
+
+
+def feedback():
+    """Take feedback data from the web app and log it
+    """
+
+    try:
+
+        entry = {
+            "datetime": log_utility.get_date_time(),
+        }
+
+        report = json.loads(request.form['data'])
+
+        for key in ["type", "missing", "reason", "from", "to"]:
+            if key in report:
+                entry[key] = str(report[key])
+
+        # Write data in JSON format and send to stdout
+        logLock.acquire()
+        sys.stdout.write(f"{json.dumps(entry) + '\n'}")
+        logLock.release()
+
+        return Response(status=201)
+
+    except Exception:
+
+        return Response(status=400)
+
+
+def delete():
+    """Delete files in folder 'downloads'
+    """
+
+    realbase = os.path.realpath(const.DEFAULT_OUTPUT_DIR)
+
+    realfilename = os.path.realpath(os.path.join(const.DEFAULT_OUTPUT_DIR, request.form['filename']))
+    reallogname = os.path.realpath(os.path.join(const.DEFAULT_OUTPUT_DIR, request.form['logname']))
+
+    if realfilename.startswith(realbase + os.sep) and reallogname.startswith(realbase + os.sep):
+
+        os.remove(realfilename)
+        os.remove(reallogname)
+
+        return 'okay'
+
+    else:
+
+        return Response(status=400)
+
+
+def init_post(app: Flask):
+    """Connect the provided Flask app to each of the post methods
+    """
+
+    app.route('/convert/', methods=["POST"])(convert)
+
+    app.route('/feedback/', methods=["POST"])(feedback)
+
+    app.route('/delete/', methods=["POST"])(delete)

psdi_data_conversion/gui/setup.py

@@ -0,0 +1,102 @@
+"""
+# setup.py
+
+This module handles setting up the Flask app
+"""
+
+
+import os
+from collections.abc import Callable
+from functools import wraps
+from typing import Any
+
+import werkzeug
+from flask import Flask, cli
+
+import psdi_data_conversion
+from psdi_data_conversion import constants as const
+from psdi_data_conversion.gui.accessibility import init_accessibility
+from psdi_data_conversion.gui.env import get_env
+from psdi_data_conversion.gui.get import init_get
+from psdi_data_conversion.gui.post import init_post
+
+_app: Flask | None = None
+
+
+def _patch_flask_warning():
+    """Monkey-patch Flask to disable the warnings that would otherwise appear for this so they don't confuse the user
+    """
+
+    def suppress_warning(func: Callable[..., Any]) -> Callable[..., Any]:
+        @wraps(func)
+        def wrapper(*args, **kwargs) -> Any:
+            if args and isinstance(args[0], str) and args[0].startswith('WARNING: This is a development server.'):
+                return ''
+            return func(*args, **kwargs)
+        return wrapper
+
+    werkzeug.serving._ansi_style = suppress_warning(werkzeug.serving._ansi_style)
+    cli.show_server_banner = lambda *_: None
+
+
+def _init_app():
+    """Create and return the Flask app with appropriate settings"""
+
+    # Suppress Flask's warning, since we're using the dev server as a GUI
+    _patch_flask_warning()
+
+    app = Flask(const.APP_NAME)
+
+    # Set the file upload limit based on env vars
+    limit_upload_size(app)
+
+    # Connect the app to the various pages and methods of the website
+    init_get(app)
+    init_post(app)
+    init_accessibility(app)
+
+    return app
+
+
+def limit_upload_size(app: Flask | None = None):
+    """Impose a limit on the maximum file that can be uploaded before Flask will raise an error"""
+
+    if app is None:
+        app = get_app()
+
+    env = get_env()
+
+    # Determine the largest possible file size that can be uploaded, keeping in mind that 0 indicates unlimited
+    larger_max_file_size = env.max_file_size
+    if (env.max_file_size > 0) and (env.max_file_size_ob > env.max_file_size):
+        larger_max_file_size = env.max_file_size_ob
+
+    if larger_max_file_size > 0:
+        app.config['MAX_CONTENT_LENGTH'] = larger_max_file_size
+    else:
+        app.config['MAX_CONTENT_LENGTH'] = None
+
+
+def get_app() -> Flask:
+    """Get a reference to the global `Flask` app, creating it if necessary.
+    """
+    global _app
+    if not _app:
+        _app = _init_app()
+    return _app
+
+
+def start_app():
+    """Start the Flask app - this requires being run from the base directory of the project, so this changes the
+    current directory to there. Anything else which changes it while the app is running may interfere with its proper
+    execution.
+    """
+
+    old_cwd = os.getcwd()
+
+    try:
+        os.chdir(os.path.join(psdi_data_conversion.__path__[0], ".."))
+        get_app().run(debug=get_env().debug_mode)
+    finally:
+        # Return to the previous directory after running the app
+        os.chdir(old_cwd)