psdi-data-conversion 0.0.38__py3-none-any.whl → 0.1.0__py3-none-any.whl

psdi_data_conversion/app.py

@@ -5,44 +5,57 @@ Version 1.0, 8th November 2024
 This script acts as a server for the PSDI Data Conversion Service website.
 """
 
-from argparse import ArgumentParser
-import hashlib
-import os
 import json
+from multiprocessing import Lock
+import os
+import sys
+from argparse import ArgumentParser
+from collections.abc import Callable
 from datetime import datetime
+from functools import wraps
+from hashlib import md5
 from subprocess import run
-import sys
-import traceback
-from flask import Flask, request, render_template, abort, Response
+from traceback import format_exc
+from typing import Any
+
+import werkzeug.serving
+from flask import Flask, Response, abort, cli, render_template, request
+from werkzeug.utils import secure_filename
 
 import psdi_data_conversion
-from psdi_data_conversion import log_utility
 from psdi_data_conversion import constants as const
+from psdi_data_conversion import log_utility
 from psdi_data_conversion.converter import run_converter
-from psdi_data_conversion.database import FormatInfo, get_format_info
+from psdi_data_conversion.database import get_format_info
 from psdi_data_conversion.file_io import split_archive_ext
+from psdi_data_conversion.main import print_wrap
 
 # Env var for the SHA of the latest commit
 SHA_EV = "SHA"
 
-# Env var for whether this is running in service mode or locally
-SERVICE_MODE_EV = "SERVICE_MODE"
-
 # Env var for whether this is a production release or development
 PRODUCTION_EV = "PRODUCTION_MODE"
 
+# Env var for whether this is a production release or development
+DEBUG_EV = "DEBUG_MODE"
+
 # Key for the label given to the file uploaded in the web interface
 FILE_TO_UPLOAD_KEY = 'fileToUpload'
 
+# A lock to prevent multiple threads from logging at the same time
+logLock = Lock()
+
 # Create a token by hashing the current date and time.
 dt = str(datetime.now())
-token = hashlib.md5(dt.encode('utf8')).hexdigest()
+token = md5(dt.encode('utf8')).hexdigest()
 
-# Get the service and production modes from their envvars
-service_mode_ev = os.environ.get(SERVICE_MODE_EV)
+# Get the debug, service and production modes from their envvars
+service_mode_ev = os.environ.get(const.SERVICE_MODE_EV)
 service_mode = (service_mode_ev is not None) and (service_mode_ev.lower() == "true")
 production_mode_ev = os.environ.get(PRODUCTION_EV)
 production_mode = (production_mode_ev is not None) and (production_mode_ev.lower() == "true")
+debug_mode_ev = os.environ.get(DEBUG_EV)
+debug_mode = (debug_mode_ev is not None) and (debug_mode_ev.lower() == "true")
 
 # Get the logging mode and level from their envvars
 ev_log_mode = os.environ.get(const.LOG_MODE_EV)
@@ -80,9 +93,43 @@ if ev_max_file_size_ob is not None:
 else:
     max_file_size_ob = const.DEFAULT_MAX_FILE_SIZE_OB
 
+
+def suppress_warning(func: Callable[..., Any]) -> Callable[..., Any]:
+    """Since we're using the development server as the user GUI, we monkey-patch Flask to disable the warnings that
+    would otherwise appear for this so they don't confuse the user
+    """
+    @wraps(func)
+    def wrapper(*args, **kwargs) -> Any:
+        if args and isinstance(args[0], str) and args[0].startswith('WARNING: This is a development server.'):
+            return ''
+        return func(*args, **kwargs)
+    return wrapper
+
+
+werkzeug.serving._ansi_style = suppress_warning(werkzeug.serving._ansi_style)
+cli.show_server_banner = lambda *_: None
+
 app = Flask(__name__)
 
 
+def limit_upload_size():
+    """Impose a limit on the maximum file that can be uploaded before Flask will raise an error"""
+
+    # Determine the largest possible file size that can be uploaded, keeping in mind that 0 indicates unlimited
+    larger_max_file_size = max_file_size
+    if (max_file_size > 0) and (max_file_size_ob > max_file_size):
+        larger_max_file_size = max_file_size_ob
+
+    if larger_max_file_size > 0:
+        app.config['MAX_CONTENT_LENGTH'] = larger_max_file_size
+    else:
+        app.config['MAX_CONTENT_LENGTH'] = None
+
+
+# Set the upload limit based on env vars to start with
+limit_upload_size()
+
+
 def get_last_sha() -> str:
     """Get the SHA of the last commit
     """
@@ -101,7 +148,7 @@ def get_last_sha() -> str:
         out_str = str(out_bytes.decode()).strip()
 
     except Exception:
-        print("ERROR: Could not determine SHA of most recent commit. Error was:\n" + traceback.format_exc(),
+        print("ERROR: Could not determine SHA of most recent commit. Error was:\n" + format_exc(),
               file=sys.stderr)
         out_str = "N/A"
 
@@ -112,14 +159,13 @@ def get_last_sha() -> str:
 def website():
     """Return the web page along with the token
     """
-
-    data = [{'token': token,
-             'max_file_size': max_file_size,
-             'max_file_size_ob': max_file_size_ob,
-             'service_mode': service_mode,
-             'production_mode': production_mode,
-             'sha': get_last_sha()}]
-    return render_template("index.htm", data=data)
+    return render_template("index.htm",
+                           token=token,
+                           max_file_size=max_file_size,
+                           max_file_size_ob=max_file_size_ob,
+                           service_mode=service_mode,
+                           production_mode=production_mode,
+                           sha=get_last_sha())
 
 
 @app.route('/convert/', methods=['POST'])
@@ -131,9 +177,8 @@ def convert():
     # Make sure the upload directory exists
     os.makedirs(const.DEFAULT_UPLOAD_DIR, exist_ok=True)
 
-    # Save the file in the upload directory
     file = request.files[FILE_TO_UPLOAD_KEY]
-    filename = filename = file.filename
+    filename = secure_filename(file.filename)
 
     qualified_filename = os.path.join(const.DEFAULT_UPLOAD_DIR, filename)
     file.save(qualified_filename)
@@ -146,7 +191,7 @@ def convert():
         name = request.form[format_label]
         full_note = request.form[format_label+"_full"]
 
-        l_possible_formats: list[FormatInfo] = get_format_info(name, which="all")
+        l_possible_formats = get_format_info(name, which="all")
 
         # If there's only one possible format, use that
         if len(l_possible_formats) == 1:
@@ -154,13 +199,11 @@ def convert():
             continue
 
         # Otherwise, find the format with the matching note
-        found = False
         for possible_format in l_possible_formats:
             if possible_format.note in full_note:
                 d_formats[format_label] = possible_format
-                found = True
                 break
-        if not found:
+        else:
             print(f"Format '{name}' with full description '{full_note}' could not be found in database.",
                   file=sys.stderr)
             abort(const.STATUS_CODE_GENERAL)
@@ -200,7 +243,7 @@ def convert():
             else:
                 # Failsafe exception message
                 msg = ("The following unexpected exception was raised by the converter:\n" +
-                       traceback.format_exc()+"\n")
+                       format_exc()+"\n")
             with open(qualified_output_log, "w") as fo:
                 fo.write(msg)
             abort(status_code)
@@ -228,7 +271,10 @@ def feedback():
             if key in report:
                 entry[key] = str(report[key])
 
-        log_utility.append_to_log_file("feedback", entry)
+        # Write data in JSON format and send to stdout
+        logLock.acquire()
+        sys.stdout.write(f"{json.dumps(entry) +  '\n'}")
+        logLock.release()
 
         return Response(status=201)
 
@@ -298,7 +344,7 @@ def start_app():
     """
 
     os.chdir(os.path.join(psdi_data_conversion.__path__[0], ".."))
-    app.run()
+    app.run(debug=debug_mode)
 
 
 def main():
@@ -323,7 +369,11 @@ def main():
                         help="If set, will run as if deploying a service rather than the local GUI")
 
     parser.add_argument("--dev-mode", action="store_true",
-                        help="If set, will expose development elements")
+                        help="If set, will expose development elements, such as the SHA of the latest commit")
+
+    parser.add_argument("--debug", action="store_true",
+                        help="If set, will run the Flask server in debug mode, which will cause it to automatically "
+                        "reload if code changes and show an interactive debugger in the case of errors")
 
     parser.add_argument("--log-mode", type=str, default=const.LOG_FULL,
                         help="How logs should be stored. Allowed values are: \n"
@@ -351,6 +401,9 @@ def main():
         global service_mode
         service_mode = args.service_mode
 
+        global debug_mode
+        debug_mode = args.debug
+
         global production_mode
         production_mode = not args.dev_mode
 
@@ -360,6 +413,13 @@ def main():
         global log_level
         log_level = args.log_level
 
+    # Set the upload limit based on provided arguments now
+    limit_upload_size()
+
+    print_wrap("Starting the PSDI Data Conversion GUI. This GUI is run as a webpage, which you can open by "
+               "right-clicking the link below to open it in your default browser, or by copy-and-pasting it into your "
+               "browser of choice.")
+
     start_app()
 
 

psdi_data_conversion/constants.py

@@ -41,6 +41,7 @@ LOG_MODE_EV = "LOG_MODE"
 LOG_LEVEL_EV = "LOG_LEVEL"
 MAX_FILESIZE_EV = "MAX_FILESIZE"
 MAX_FILESIZE_OB_EV = "MAX_FILESIZE_OB"
+SERVICE_MODE_EV = "SERVICE_MODE"
 
 # Files and Folders
 # -----------------

psdi_data_conversion/converter.py

@@ -6,20 +6,29 @@ Class and functions to perform file conversion
 """
 
 from dataclasses import dataclass, field
-import os
+import json
+import glob
 import importlib
+import os
 import sys
-from tempfile import TemporaryDirectory
 import traceback
 from typing import Any, Callable, NamedTuple
+from multiprocessing import Lock
+from psdi_data_conversion import log_utility
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from tempfile import TemporaryDirectory
+from typing import Any, NamedTuple
+
 from psdi_data_conversion import constants as const
 from psdi_data_conversion.converters import base
-
-import glob
-
 from psdi_data_conversion.converters.openbabel import CONVERTER_OB
 from psdi_data_conversion.file_io import (is_archive, is_supported_archive, pack_zip_or_tar, split_archive_ext,
                                           unpack_zip_or_tar)
+from psdi_data_conversion.utils import regularize_name
+
+# A lock to prevent multiple threads from logging at the same time
+logLock = Lock()
 
 # Find all modules for specific converters
 l_converter_modules = glob.glob(os.path.dirname(base.__file__) + "/*.py")
@@ -49,7 +58,8 @@ try:
 
         converter_class = module.converter
 
-        name = converter_class.name
+        # To make querying case/space-insensitive, we store all names in lowercase with spaces stripped
+        name = converter_class.name.lower().replace(" ", "")
 
         return NameAndClass(name, converter_class)
 
@@ -91,6 +101,66 @@ except Exception:
     D_CONVERTER_ARGS = {}
 
 
+def get_supported_converter_class(name: str):
+    """Get the appropriate converter class matching the provided name from the dict of supported converters
+
+    Parameters
+    ----------
+    name : str
+        Converter name (case- and space-insensitive)
+
+    Returns
+    -------
+    type[base.FileConverter]
+    """
+    return D_SUPPORTED_CONVERTERS[regularize_name(name)]
+
+
+def get_registered_converter_class(name: str):
+    """Get the appropriate converter class matching the provided name from the dict of supported converters
+
+    Parameters
+    ----------
+    name : str
+        Converter name (case- and space-insensitive)
+
+    Returns
+    -------
+    type[base.FileConverter]
+    """
+    return D_REGISTERED_CONVERTERS[regularize_name(name)]
+
+
+def converter_is_supported(name: str):
+    """Checks if a converter is supported in principle by this project
+
+    Parameters
+    ----------
+    name : str
+        Converter name (case- and space-insensitive)
+
+    Returns
+    -------
+    bool
+    """
+    return regularize_name(name) in L_SUPPORTED_CONVERTERS
+
+
+def converter_is_registered(name: str):
+    """Checks if a converter is registered (usable)
+
+    Parameters
+    ----------
+    name : str
+        Converter name (case- and space-insensitive)
+
+    Returns
+    -------
+    bool
+    """
+    return regularize_name(name) in L_REGISTERED_CONVERTERS
+
+
 def get_converter(*args, name=const.CONVERTER_DEFAULT, **converter_kwargs) -> base.FileConverter:
     """Get a FileConverter of the proper subclass for the requested converter type
 
@@ -129,7 +199,7 @@ def get_converter(*args, name=const.CONVERTER_DEFAULT, **converter_kwargs) -> ba
         If provided, all logging will go to a single file or stream. Otherwise, logs will be split up among multiple
         files for server-style logging.
     log_mode : str
-        How logs should be stores. Allowed values are:
+        How logs should be stored. Allowed values are:
         - 'full' - Multi-file logging, only recommended when running as a public web app
         - 'simple' - Logs saved to one file
         - 'stdout' - Output logs and errors only to stdout
@@ -155,10 +225,11 @@ def get_converter(*args, name=const.CONVERTER_DEFAULT, **converter_kwargs) -> ba
     FileConverterInputException
         If the converter isn't recognized or there's some other issue with the input
     """
+    name = regularize_name(name)
     if name not in L_REGISTERED_CONVERTERS:
         raise base.FileConverterInputException(const.ERR_CONVERTER_NOT_RECOGNISED.format(name) +
                                                f"{L_REGISTERED_CONVERTERS}")
-    converter_class = D_REGISTERED_CONVERTERS[name]
+    converter_class = get_registered_converter_class(name)
 
     return converter_class(*args, **converter_kwargs)
 
@@ -302,7 +373,7 @@ def run_converter(filename: str,
         If provided, all logging will go to a single file or stream. Otherwise, logs will be split up among multiple
         files for server-style logging.
     log_mode : str
-        How logs should be stores. Allowed values are:
+        How logs should be stored. Allowed values are:
         - 'full' - Multi-file logging, only recommended when running as a public web app
         - 'simple' - Logs saved to one file
         - 'stdout' - Output logs and errors only to stdout
@@ -355,14 +426,14 @@ def run_converter(filename: str,
         if from_format is not None:
             check_from_format(filename, from_format, strict=strict)
         l_run_output.append(get_converter(filename,
-                                          to_format,
-                                          *args,
-                                          from_format=from_format,
-                                          download_dir=download_dir,
-                                          max_file_size=max_file_size,
-                                          log_file=log_file,
-                                          log_mode=log_mode,
-                                          **converter_kwargs).run())
+                            to_format,
+                            *args,
+                            from_format=from_format,
+                            download_dir=download_dir,
+                            max_file_size=max_file_size,
+                            log_file=log_file,
+                            log_mode=log_mode,
+                            **converter_kwargs).run())
 
     elif not is_supported_archive(filename):
         raise base.FileConverterInputException(f"{filename} is an unsupported archive type. Supported types are: "
@@ -473,4 +544,61 @@ def run_converter(filename: str,
                 exception_class = base.FileConverterAbortException
             raise exception_class(status_code, msg)
 
+    # Log conversion information if in service mode
+    service_mode_ev = os.environ.get(const.SERVICE_MODE_EV)
+    service_mode = (service_mode_ev is not None) and (service_mode_ev.lower() == "true")
+    if service_mode:
+        try:
+            l_index = filename.rfind('/') + 1
+            r_index = len(filename)
+            in_filename = filename[l_index:r_index]
+
+            l_index = run_output.output_filename.rfind('/') + 1
+            r_index = len(run_output.output_filename)
+
+            input_size = set_size_units(run_output.in_size)
+            output_size = set_size_units(run_output.out_size)
+
+            if status_code:
+                outcome = "failed"
+                fail_reason = l_error_lines
+            else:
+                outcome = "succeeded"
+                fail_reason = ""
+
+            entry = {
+                "datetime": log_utility.get_date_time(),
+                "input_format": converter_kwargs['data']['from_full'],
+                "output_format": converter_kwargs['data']['to_full'],
+                "input_filename": in_filename,
+                "output_filename": run_output.output_filename[l_index:r_index],
+                "input_size": input_size,
+                "output_size": output_size }
+
+            for key in [ "converter", "coordinates", "coordOption", "from_flags",
+                "to_flags", "from_arg_flags", "to_arg_flags" ]:
+                if key in converter_kwargs['data'] and converter_kwargs['data'][key] != "" and not \
+                    ((key == "coordinates" or key == "coordOption") and converter_kwargs['data']['coordinates'] == "neither") :
+                    entry[key] = converter_kwargs['data'][key]
+
+            entry["outcome"] = outcome
+
+            if fail_reason != "":
+                entry["fail_reason"] = fail_reason
+
+            logLock.acquire()
+            sys.__stdout__.write(f"{json.dumps(entry) + '\n'}")
+            logLock.release()
+        except Exception:
+            sys.__stdout__.write({"datetime": log_utility.get_date_time(),
+                                  "logging_error": "An error occurred during logging of conversion information."})
+
     return run_output
+
+def set_size_units(size):
+    if size >= 1024:
+        return str('%.3f' % (size / 1024)) + ' kB'
+    elif size >= const.MEGABYTE:
+        return str(size / const.MEGABYTE) + ' MB'
+    else:
+        return str(size) + ' B'

psdi_data_conversion/converters/base.py

@@ -6,19 +6,19 @@ Base class and information for file format converters
 """
 
 
-from copy import deepcopy
-from dataclasses import dataclass
+import abc
 import logging
-from collections.abc import Callable
 import os
 import subprocess
-import abc
-
 import sys
 import traceback
+from collections.abc import Callable
+from copy import deepcopy
+from dataclasses import dataclass
 from typing import Any
 
-from psdi_data_conversion import constants as const, log_utility
+from psdi_data_conversion import constants as const
+from psdi_data_conversion import log_utility
 from psdi_data_conversion.dist import bin_exists, get_bin_path, get_dist
 from psdi_data_conversion.security import SAFE_STRING_RE, string_is_safe
 
@@ -735,24 +735,11 @@ class ScriptFileConverter(FileConverter):
 
         self.logger.debug(f"Performing conversion with ScriptFileConverter using script '{self.script}'")
 
-        from_flags = self.data.get("from_flags", "")
-        to_flags = self.data.get("from_flags", "")
-        from_options = self.data.get("from_options", "")
-        to_options = self.data.get("from_options", "")
-
-        # Check that all user-provided input passes security checks
-        for user_args in [from_flags, to_flags, from_options, to_options]:
-            if not string_is_safe(user_args):
-                raise FileConverterInputException(f"Provided argument '{user_args}' does not pass security check - it "
-                                                  f"must match the regex {SAFE_STRING_RE.pattern}.", help=True)
-
         env = {"DIST": get_dist()}
         if self.required_bin is not None:
             env["BIN_PATH"] = get_bin_path(self.required_bin)
 
-        process = subprocess.run(['sh', f'psdi_data_conversion/scripts/{self.script}',
-                                  '--' + self.to_format_info.name, self.in_filename, self.out_filename, from_flags,
-                                  to_flags, from_options, to_options],
+        process = subprocess.run(['sh', f'psdi_data_conversion/scripts/{self.script}', *self._get_script_args()],
                                  env=env, capture_output=True, text=True)
 
         self.out = process.stdout
@@ -763,3 +750,20 @@ class ScriptFileConverter(FileConverter):
             self._abort_from_err()
         else:
             self.logger.debug("Conversion process completed successfully")
+
+    def _get_script_args(self):
+        """Get the list of arguments which will be passed to the script"""
+
+        from_flags = self.data.get("from_flags", "")
+        to_flags = self.data.get("from_flags", "")
+        from_options = self.data.get("from_options", "")
+        to_options = self.data.get("from_options", "")
+
+        # Check that all user-provided input passes security checks
+        for user_args in [from_flags, to_flags, from_options, to_options]:
+            if not string_is_safe(user_args):
+                raise FileConverterInputException(f"Provided argument '{user_args}' does not pass security check - it "
+                                                  f"must match the regex {SAFE_STRING_RE.pattern}.", help=True)
+
+        return ['--' + self.to_format_info.name, self.in_filename, self.out_filename, from_flags, to_flags,
+                from_options, to_options]

psdi_data_conversion/converters/c2x.py

@@ -29,6 +29,19 @@ class C2xFileConverter(ScriptFileConverter):
             "may be downloaded from https://www.c2x.org.uk/downloads/")
     supports_ambiguous_extensions = True
 
+    def _get_script_args(self):
+        """Override the standard script arguments so we can set the different format names expected by c2x
+        """
+        l_script_args = super()._get_script_args()
+
+        # Update the output format to c2x style
+        l_script_args[0] = "--" + self.to_format_info.c2x_format
+
+        # TODO - check if the input file has an extension which will be accepted by c2x for its format, and handle if
+        # not
+
+        return l_script_args
+
 
 # Assign this converter to the `converter` variable - this lets the psdi_data_conversion.converter module detect and
 # register it, making it available for use by the command-line script, python library, and web app

psdi_data_conversion/converters/openbabel.py

@@ -6,8 +6,9 @@ Open Babel FileConverter
 """
 
 from copy import deepcopy
-from openbabel import openbabel
+
 import py
+from openbabel import openbabel
 
 from psdi_data_conversion.converters.base import FileConverter, FileConverterInputException
 from psdi_data_conversion.security import SAFE_STRING_RE, string_is_safe