psdi-data-conversion 0.0.39__tar.gz → 0.1.0__tar.gz

{psdi_data_conversion-0.0.39 → psdi_data_conversion-0.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: psdi_data_conversion
-Version: 0.0.39
+Version: 0.1.0
 Summary: Chemistry file format conversion service, provided by PSDI
 Project-URL: Homepage, https://data-conversion.psdi.ac.uk/
 Project-URL: Documentation, https://psdi-uk.github.io/psdi-data-conversion/
@@ -243,7 +243,7 @@ Description-Content-Type: text/markdown
 
 # PSDI Data Conversion
 
-Version: Pre-release 2024-04-14
+Release date: 2024-04-29
 
 This is the repository for the PSDI PF2 Chemistry File Format Conversion project. The goal of this project is to provide utilities to assist in converting files between the many different file formats used in chemistry, providing information on what converters are available for a given conversion and the expected quality of it, and providing multiple interfaces to perform these conversions. These interfaces are:
 

{psdi_data_conversion-0.0.39 → psdi_data_conversion-0.1.0}/README.md

@@ -1,6 +1,6 @@
 # PSDI Data Conversion
 
-Version: Pre-release 2024-04-14
+Release date: 2024-04-29
 
 This is the repository for the PSDI PF2 Chemistry File Format Conversion project. The goal of this project is to provide utilities to assist in converting files between the many different file formats used in chemistry, providing information on what converters are available for a given conversion and the expected quality of it, and providing multiple interfaces to perform these conversions. These interfaces are:
 

{psdi_data_conversion-0.0.39 → psdi_data_conversion-0.1.0}/psdi_data_conversion/app.py

@@ -6,6 +6,7 @@ This script acts as a server for the PSDI Data Conversion Service website.
 """
 
 import json
+from multiprocessing import Lock
 import os
 import sys
 from argparse import ArgumentParser
@@ -19,6 +20,7 @@ from typing import Any
 
 import werkzeug.serving
 from flask import Flask, Response, abort, cli, render_template, request
+from werkzeug.utils import secure_filename
 
 import psdi_data_conversion
 from psdi_data_conversion import constants as const
@@ -31,24 +33,29 @@ from psdi_data_conversion.main import print_wrap
 # Env var for the SHA of the latest commit
 SHA_EV = "SHA"
 
-# Env var for whether this is running in service mode or locally
-SERVICE_MODE_EV = "SERVICE_MODE"
-
 # Env var for whether this is a production release or development
 PRODUCTION_EV = "PRODUCTION_MODE"
 
+# Env var for whether this is a production release or development
+DEBUG_EV = "DEBUG_MODE"
+
 # Key for the label given to the file uploaded in the web interface
 FILE_TO_UPLOAD_KEY = 'fileToUpload'
 
+# A lock to prevent multiple threads from logging at the same time
+logLock = Lock()
+
 # Create a token by hashing the current date and time.
 dt = str(datetime.now())
 token = md5(dt.encode('utf8')).hexdigest()
 
-# Get the service and production modes from their envvars
-service_mode_ev = os.environ.get(SERVICE_MODE_EV)
+# Get the debug, service and production modes from their envvars
+service_mode_ev = os.environ.get(const.SERVICE_MODE_EV)
 service_mode = (service_mode_ev is not None) and (service_mode_ev.lower() == "true")
 production_mode_ev = os.environ.get(PRODUCTION_EV)
 production_mode = (production_mode_ev is not None) and (production_mode_ev.lower() == "true")
+debug_mode_ev = os.environ.get(DEBUG_EV)
+debug_mode = (debug_mode_ev is not None) and (debug_mode_ev.lower() == "true")
 
 # Get the logging mode and level from their envvars
 ev_log_mode = os.environ.get(const.LOG_MODE_EV)
@@ -86,11 +93,11 @@ if ev_max_file_size_ob is not None:
 else:
     max_file_size_ob = const.DEFAULT_MAX_FILE_SIZE_OB
 
-# Since we're using the development server as the user GUI, we monkey-patch Flask to disable the warnings that would
-# otherwise appear for this so they don't confuse the user
-
 
 def suppress_warning(func: Callable[..., Any]) -> Callable[..., Any]:
+    """Since we're using the development server as the user GUI, we monkey-patch Flask to disable the warnings that
+    would otherwise appear for this so they don't confuse the user
+    """
     @wraps(func)
     def wrapper(*args, **kwargs) -> Any:
         if args and isinstance(args[0], str) and args[0].startswith('WARNING: This is a development server.'):
@@ -105,6 +112,24 @@ cli.show_server_banner = lambda *_: None
 app = Flask(__name__)
 
 
+def limit_upload_size():
+    """Impose a limit on the maximum file that can be uploaded before Flask will raise an error"""
+
+    # Determine the largest possible file size that can be uploaded, keeping in mind that 0 indicates unlimited
+    larger_max_file_size = max_file_size
+    if (max_file_size > 0) and (max_file_size_ob > max_file_size):
+        larger_max_file_size = max_file_size_ob
+
+    if larger_max_file_size > 0:
+        app.config['MAX_CONTENT_LENGTH'] = larger_max_file_size
+    else:
+        app.config['MAX_CONTENT_LENGTH'] = None
+
+
+# Set the upload limit based on env vars to start with
+limit_upload_size()
+
+
 def get_last_sha() -> str:
     """Get the SHA of the last commit
     """
@@ -134,14 +159,13 @@ def get_last_sha() -> str:
 def website():
     """Return the web page along with the token
     """
-
-    data = [{'token': token,
-             'max_file_size': max_file_size,
-             'max_file_size_ob': max_file_size_ob,
-             'service_mode': service_mode,
-             'production_mode': production_mode,
-             'sha': get_last_sha()}]
-    return render_template("index.htm", data=data)
+    return render_template("index.htm",
+                           token=token,
+                           max_file_size=max_file_size,
+                           max_file_size_ob=max_file_size_ob,
+                           service_mode=service_mode,
+                           production_mode=production_mode,
+                           sha=get_last_sha())
 
 
 @app.route('/convert/', methods=['POST'])
@@ -153,9 +177,8 @@ def convert():
     # Make sure the upload directory exists
     os.makedirs(const.DEFAULT_UPLOAD_DIR, exist_ok=True)
 
-    # Save the file in the upload directory
     file = request.files[FILE_TO_UPLOAD_KEY]
-    filename = filename = file.filename
+    filename = secure_filename(file.filename)
 
     qualified_filename = os.path.join(const.DEFAULT_UPLOAD_DIR, filename)
     file.save(qualified_filename)
@@ -248,7 +271,10 @@ def feedback():
             if key in report:
                 entry[key] = str(report[key])
 
-        log_utility.append_to_log_file("feedback", entry)
+        # Write data in JSON format and send to stdout
+        logLock.acquire()
+        sys.stdout.write(f"{json.dumps(entry) +  '\n'}")
+        logLock.release()
 
         return Response(status=201)
 
@@ -318,7 +344,7 @@ def start_app():
     """
 
     os.chdir(os.path.join(psdi_data_conversion.__path__[0], ".."))
-    app.run()
+    app.run(debug=debug_mode)
 
 
 def main():
@@ -343,7 +369,11 @@ def main():
                         help="If set, will run as if deploying a service rather than the local GUI")
 
     parser.add_argument("--dev-mode", action="store_true",
-                        help="If set, will expose development elements")
+                        help="If set, will expose development elements, such as the SHA of the latest commit")
+
+    parser.add_argument("--debug", action="store_true",
+                        help="If set, will run the Flask server in debug mode, which will cause it to automatically "
+                        "reload if code changes and show an interactive debugger in the case of errors")
 
     parser.add_argument("--log-mode", type=str, default=const.LOG_FULL,
                         help="How logs should be stored. Allowed values are: \n"
@@ -371,6 +401,9 @@ def main():
         global service_mode
         service_mode = args.service_mode
 
+        global debug_mode
+        debug_mode = args.debug
+
         global production_mode
         production_mode = not args.dev_mode
 
@@ -380,6 +413,9 @@ def main():
         global log_level
         log_level = args.log_level
 
+    # Set the upload limit based on provided arguments now
+    limit_upload_size()
+
     print_wrap("Starting the PSDI Data Conversion GUI. This GUI is run as a webpage, which you can open by "
                "right-clicking the link below to open it in your default browser, or by copy-and-pasting it into your "
                "browser of choice.")

{psdi_data_conversion-0.0.39 → psdi_data_conversion-0.1.0}/psdi_data_conversion/constants.py

@@ -41,6 +41,7 @@ LOG_MODE_EV = "LOG_MODE"
 LOG_LEVEL_EV = "LOG_LEVEL"
 MAX_FILESIZE_EV = "MAX_FILESIZE"
 MAX_FILESIZE_OB_EV = "MAX_FILESIZE_OB"
+SERVICE_MODE_EV = "SERVICE_MODE"
 
 # Files and Folders
 # -----------------

{psdi_data_conversion-0.0.39 → psdi_data_conversion-0.1.0}/psdi_data_conversion/converter.py

@@ -5,11 +5,16 @@ Created 2024-12-10 by Bryan Gillis.
 Class and functions to perform file conversion
 """
 
+from dataclasses import dataclass, field
+import json
 import glob
 import importlib
 import os
 import sys
 import traceback
+from typing import Any, Callable, NamedTuple
+from multiprocessing import Lock
+from psdi_data_conversion import log_utility
 from collections.abc import Callable
 from dataclasses import dataclass, field
 from tempfile import TemporaryDirectory
@@ -20,6 +25,10 @@ from psdi_data_conversion.converters import base
 from psdi_data_conversion.converters.openbabel import CONVERTER_OB
 from psdi_data_conversion.file_io import (is_archive, is_supported_archive, pack_zip_or_tar, split_archive_ext,
                                           unpack_zip_or_tar)
+from psdi_data_conversion.utils import regularize_name
+
+# A lock to prevent multiple threads from logging at the same time
+logLock = Lock()
 
 # Find all modules for specific converters
 l_converter_modules = glob.glob(os.path.dirname(base.__file__) + "/*.py")
@@ -49,7 +58,8 @@ try:
 
         converter_class = module.converter
 
-        name = converter_class.name
+        # To make querying case/space-insensitive, we store all names in lowercase with spaces stripped
+        name = converter_class.name.lower().replace(" ", "")
 
         return NameAndClass(name, converter_class)
 
@@ -91,6 +101,66 @@ except Exception:
     D_CONVERTER_ARGS = {}
 
 
+def get_supported_converter_class(name: str):
+    """Get the appropriate converter class matching the provided name from the dict of supported converters
+
+    Parameters
+    ----------
+    name : str
+        Converter name (case- and space-insensitive)
+
+    Returns
+    -------
+    type[base.FileConverter]
+    """
+    return D_SUPPORTED_CONVERTERS[regularize_name(name)]
+
+
+def get_registered_converter_class(name: str):
+    """Get the appropriate converter class matching the provided name from the dict of supported converters
+
+    Parameters
+    ----------
+    name : str
+        Converter name (case- and space-insensitive)
+
+    Returns
+    -------
+    type[base.FileConverter]
+    """
+    return D_REGISTERED_CONVERTERS[regularize_name(name)]
+
+
+def converter_is_supported(name: str):
+    """Checks if a converter is supported in principle by this project
+
+    Parameters
+    ----------
+    name : str
+        Converter name (case- and space-insensitive)
+
+    Returns
+    -------
+    bool
+    """
+    return regularize_name(name) in L_SUPPORTED_CONVERTERS
+
+
+def converter_is_registered(name: str):
+    """Checks if a converter is registered (usable)
+
+    Parameters
+    ----------
+    name : str
+        Converter name (case- and space-insensitive)
+
+    Returns
+    -------
+    bool
+    """
+    return regularize_name(name) in L_REGISTERED_CONVERTERS
+
+
 def get_converter(*args, name=const.CONVERTER_DEFAULT, **converter_kwargs) -> base.FileConverter:
     """Get a FileConverter of the proper subclass for the requested converter type
 
@@ -129,7 +199,7 @@ def get_converter(*args, name=const.CONVERTER_DEFAULT, **converter_kwargs) -> ba
         If provided, all logging will go to a single file or stream. Otherwise, logs will be split up among multiple
         files for server-style logging.
     log_mode : str
-        How logs should be stores. Allowed values are:
+        How logs should be stored. Allowed values are:
         - 'full' - Multi-file logging, only recommended when running as a public web app
         - 'simple' - Logs saved to one file
         - 'stdout' - Output logs and errors only to stdout
@@ -155,10 +225,11 @@ def get_converter(*args, name=const.CONVERTER_DEFAULT, **converter_kwargs) -> ba
     FileConverterInputException
         If the converter isn't recognized or there's some other issue with the input
     """
+    name = regularize_name(name)
     if name not in L_REGISTERED_CONVERTERS:
         raise base.FileConverterInputException(const.ERR_CONVERTER_NOT_RECOGNISED.format(name) +
                                                f"{L_REGISTERED_CONVERTERS}")
-    converter_class = D_REGISTERED_CONVERTERS[name]
+    converter_class = get_registered_converter_class(name)
 
     return converter_class(*args, **converter_kwargs)
 
@@ -302,7 +373,7 @@ def run_converter(filename: str,
         If provided, all logging will go to a single file or stream. Otherwise, logs will be split up among multiple
         files for server-style logging.
     log_mode : str
-        How logs should be stores. Allowed values are:
+        How logs should be stored. Allowed values are:
         - 'full' - Multi-file logging, only recommended when running as a public web app
         - 'simple' - Logs saved to one file
         - 'stdout' - Output logs and errors only to stdout
@@ -355,14 +426,14 @@ def run_converter(filename: str,
         if from_format is not None:
             check_from_format(filename, from_format, strict=strict)
         l_run_output.append(get_converter(filename,
-                                          to_format,
-                                          *args,
-                                          from_format=from_format,
-                                          download_dir=download_dir,
-                                          max_file_size=max_file_size,
-                                          log_file=log_file,
-                                          log_mode=log_mode,
-                                          **converter_kwargs).run())
+                            to_format,
+                            *args,
+                            from_format=from_format,
+                            download_dir=download_dir,
+                            max_file_size=max_file_size,
+                            log_file=log_file,
+                            log_mode=log_mode,
+                            **converter_kwargs).run())
 
     elif not is_supported_archive(filename):
         raise base.FileConverterInputException(f"{filename} is an unsupported archive type. Supported types are: "
@@ -473,4 +544,61 @@ def run_converter(filename: str,
                 exception_class = base.FileConverterAbortException
             raise exception_class(status_code, msg)
 
+    # Log conversion information if in service mode
+    service_mode_ev = os.environ.get(const.SERVICE_MODE_EV)
+    service_mode = (service_mode_ev is not None) and (service_mode_ev.lower() == "true")
+    if service_mode:
+        try:
+            l_index = filename.rfind('/') + 1
+            r_index = len(filename)
+            in_filename = filename[l_index:r_index]
+
+            l_index = run_output.output_filename.rfind('/') + 1
+            r_index = len(run_output.output_filename)
+
+            input_size = set_size_units(run_output.in_size)
+            output_size = set_size_units(run_output.out_size)
+
+            if status_code:
+                outcome = "failed"
+                fail_reason = l_error_lines
+            else:
+                outcome = "succeeded"
+                fail_reason = ""
+
+            entry = {
+                "datetime": log_utility.get_date_time(),
+                "input_format": converter_kwargs['data']['from_full'],
+                "output_format": converter_kwargs['data']['to_full'],
+                "input_filename": in_filename,
+                "output_filename": run_output.output_filename[l_index:r_index],
+                "input_size": input_size,
+                "output_size": output_size }
+
+            for key in [ "converter", "coordinates", "coordOption", "from_flags",
+                "to_flags", "from_arg_flags", "to_arg_flags" ]:
+                if key in converter_kwargs['data'] and converter_kwargs['data'][key] != "" and not \
+                    ((key == "coordinates" or key == "coordOption") and converter_kwargs['data']['coordinates'] == "neither") :
+                    entry[key] = converter_kwargs['data'][key]
+
+            entry["outcome"] = outcome
+
+            if fail_reason != "":
+                entry["fail_reason"] = fail_reason
+
+            logLock.acquire()
+            sys.__stdout__.write(f"{json.dumps(entry) + '\n'}")
+            logLock.release()
+        except Exception:
+            sys.__stdout__.write({"datetime": log_utility.get_date_time(),
+                                  "logging_error": "An error occurred during logging of conversion information."})
+
     return run_output
+
+def set_size_units(size):
+    if size >= 1024:
+        return str('%.3f' % (size / 1024)) + ' kB'
+    elif size >= const.MEGABYTE:
+        return str(size / const.MEGABYTE) + ' MB'
+    else:
+        return str(size) + ' B'

{psdi_data_conversion-0.0.39 → psdi_data_conversion-0.1.0}/psdi_data_conversion/database.py

@@ -15,8 +15,9 @@ from logging import getLogger
 from typing import Any, Literal, overload
 
 from psdi_data_conversion import constants as const
-from psdi_data_conversion.converter import D_REGISTERED_CONVERTERS, D_SUPPORTED_CONVERTERS
+from psdi_data_conversion.converter import D_SUPPORTED_CONVERTERS, get_registered_converter_class
 from psdi_data_conversion.converters.base import FileConverterException
+from psdi_data_conversion.utils import regularize_name
 
 # Keys for top-level and general items in the database
 DB_FORMATS_KEY = "formats"
@@ -117,14 +118,14 @@ class ConverterInfo:
         Parameters
         ----------
         name : str
-            The name of the converter
+            The regularized name of the converter
         parent : DataConversionDatabase
             The database which this belongs to
         d_data : dict[str, Any]
             The loaded database dict
         """
 
-        self.name = name
+        self.name = regularize_name(name)
         self.parent = parent
 
         # Get info about the converter from the database
@@ -134,7 +135,7 @@ class ConverterInfo:
 
         # Get necessary info about the converter from the class
         try:
-            self._key_prefix = D_REGISTERED_CONVERTERS[name].database_key_prefix
+            self._key_prefix = get_registered_converter_class(name).database_key_prefix
         except KeyError:
             # We'll get a KeyError for converters in the database that don't yet have their own class, which we can
             # safely ignore
@@ -529,6 +530,10 @@ class ConversionQualityInfo:
     input and output file formats and a note on the implications
     """
 
+    def __post_init__(self):
+        """Regularize the converter name"""
+        self.converter_name = regularize_name(self.converter_name)
+
 
 class ConversionsTable:
     """Class providing information on available file format conversions.
@@ -623,7 +628,7 @@ class ConversionsTable:
 
         # Check if this converter deals with ambiguous formats, so we know if we need to be strict about getting format
         # info
-        if D_REGISTERED_CONVERTERS[converter_name].supports_ambiguous_extensions:
+        if get_registered_converter_class(converter_name).supports_ambiguous_extensions:
             which_format = None
         else:
             which_format = 0
@@ -805,7 +810,7 @@ class DataConversionDatabase:
         if self._d_converter_info is None:
             self._d_converter_info: dict[str, ConverterInfo] = {}
             for d_single_converter_info in self.converters:
-                name: str = d_single_converter_info[DB_NAME_KEY]
+                name: str = regularize_name(d_single_converter_info[DB_NAME_KEY])
                 if name in self._d_converter_info:
                     logger.warning(f"Converter '{name}' appears more than once in the database. Only the first instance"
                                    " will be used.")
@@ -1095,7 +1100,7 @@ def get_converter_info(name: str) -> ConverterInfo:
     ConverterInfo
     """
 
-    return get_database().d_converter_info[name]
+    return get_database().d_converter_info[regularize_name(name)]
 
 
 @overload
@@ -1155,7 +1160,7 @@ def get_conversion_quality(converter_name: str,
         `ConversionQualityInfo` object with info on the conversion
     """
 
-    return get_database().conversions_table.get_conversion_quality(converter_name=converter_name,
+    return get_database().conversions_table.get_conversion_quality(converter_name=regularize_name(converter_name),
                                                                    in_format=in_format,
                                                                    out_format=out_format)
 
@@ -1210,6 +1215,9 @@ def disambiguate_formats(converter_name: str,
         If more than one format combination is possible for this conversion, or no conversion is possible
     """
 
+    # Regularize the converter name so we don't worry about case/spacing mismatches
+    converter_name = regularize_name(converter_name)
+
     # Get all possible conversions, and see if we only have one for this converter
     l_possible_conversions = [x for x in get_possible_conversions(in_format, out_format)
                               if x[0] == converter_name]
@@ -1243,7 +1251,7 @@ def get_possible_formats(converter_name: str) -> tuple[list[FormatInfo], list[Fo
     tuple[list[FormatInfo], list[FormatInfo]]
         A tuple of a list of the supported input formats and a list of the supported output formats
     """
-    return get_database().conversions_table.get_possible_formats(converter_name=converter_name)
+    return get_database().conversions_table.get_possible_formats(converter_name=regularize_name(converter_name))
 
 
 def _find_arg(tl_args: tuple[list[FlagInfo], list[OptionInfo]],