psdi-data-conversion 0.1.7__py3-none-any.whl → 0.2.0__py3-none-any.whl

psdi_data_conversion/app.py

@@ -5,398 +5,14 @@ Version 1.0, 8th November 2024
 This script acts as a server for the PSDI Data Conversion Service website.
 """
 
-import json
-import os
-import sys
 from argparse import ArgumentParser
-from collections.abc import Callable
-from datetime import datetime
-from functools import wraps
-from hashlib import md5
-from multiprocessing import Lock
-from subprocess import run
-from traceback import format_exc
-from typing import Any
 
-import werkzeug.serving
-from flask import Flask, Response, abort, cli, render_template, request
-from werkzeug.utils import secure_filename
-
-import psdi_data_conversion
 from psdi_data_conversion import constants as const
-from psdi_data_conversion import log_utility
-from psdi_data_conversion.converter import run_converter
-from psdi_data_conversion.database import get_format_info
-from psdi_data_conversion.file_io import split_archive_ext
+from psdi_data_conversion.gui.env import update_env
+from psdi_data_conversion.gui.setup import get_app, limit_upload_size, start_app
 from psdi_data_conversion.main import print_wrap
 
-# Env var for the tag and SHA of the latest commit
-TAG_EV = "TAG"
-TAG_SHA_EV = "TAG_SHA"
-SHA_EV = "SHA"
-
-# Env var for whether this is a production release or development
-PRODUCTION_EV = "PRODUCTION_MODE"
-
-# Env var for whether this is a production release or development
-DEBUG_EV = "DEBUG_MODE"
-
-# Key for the label given to the file uploaded in the web interface
-FILE_TO_UPLOAD_KEY = 'fileToUpload'
-
-# A lock to prevent multiple threads from logging at the same time
-logLock = Lock()
-
-# Create a token by hashing the current date and time.
-dt = str(datetime.now())
-token = md5(dt.encode('utf8')).hexdigest()
-
-# Get the debug, service and production modes from their envvars
-service_mode_ev = os.environ.get(const.SERVICE_MODE_EV)
-service_mode = (service_mode_ev is not None) and (service_mode_ev.lower() == "true")
-production_mode_ev = os.environ.get(PRODUCTION_EV)
-production_mode = (production_mode_ev is not None) and (production_mode_ev.lower() == "true")
-debug_mode_ev = os.environ.get(DEBUG_EV)
-debug_mode = (debug_mode_ev is not None) and (debug_mode_ev.lower() == "true")
-
-# Get the logging mode and level from their envvars
-ev_log_mode = os.environ.get(const.LOG_MODE_EV)
-if ev_log_mode is None:
-    log_mode = const.LOG_MODE_DEFAULT
-else:
-    ev_log_mode = ev_log_mode.lower()
-    if ev_log_mode not in const.L_ALLOWED_LOG_MODES:
-        print(f"ERROR: Unrecognised logging option: {ev_log_mode}. Allowed options are: {const.L_ALLOWED_LOG_MODES}",
-              file=sys.stderr)
-        exit(1)
-    log_mode = ev_log_mode
-
-ev_log_level = os.environ.get(const.LOG_LEVEL_EV)
-if ev_log_level is None:
-    log_level = None
-else:
-    try:
-        log_level = log_utility.get_log_level_from_str(ev_log_level)
-    except ValueError as e:
-        print(f"ERROR: {str(e)}")
-        exit(1)
-
-# Get the maximum allowed size from the envvar for it
-ev_max_file_size = os.environ.get(const.MAX_FILESIZE_EV)
-if ev_max_file_size is not None:
-    max_file_size = float(ev_max_file_size)*const.MEGABYTE
-else:
-    max_file_size = const.DEFAULT_MAX_FILE_SIZE
-
-# And same for the Open Babel maximum file size
-ev_max_file_size_ob = os.environ.get(const.MAX_FILESIZE_OB_EV)
-if ev_max_file_size_ob is not None:
-    max_file_size_ob = float(ev_max_file_size_ob)*const.MEGABYTE
-else:
-    max_file_size_ob = const.DEFAULT_MAX_FILE_SIZE_OB
-
-
-def suppress_warning(func: Callable[..., Any]) -> Callable[..., Any]:
-    """Since we're using the development server as the user GUI, we monkey-patch Flask to disable the warnings that
-    would otherwise appear for this so they don't confuse the user
-    """
-    @wraps(func)
-    def wrapper(*args, **kwargs) -> Any:
-        if args and isinstance(args[0], str) and args[0].startswith('WARNING: This is a development server.'):
-            return ''
-        return func(*args, **kwargs)
-    return wrapper
-
-
-werkzeug.serving._ansi_style = suppress_warning(werkzeug.serving._ansi_style)
-cli.show_server_banner = lambda *_: None
-
-app = Flask(__name__)
-
-
-def limit_upload_size():
-    """Impose a limit on the maximum file that can be uploaded before Flask will raise an error"""
-
-    # Determine the largest possible file size that can be uploaded, keeping in mind that 0 indicates unlimited
-    larger_max_file_size = max_file_size
-    if (max_file_size > 0) and (max_file_size_ob > max_file_size):
-        larger_max_file_size = max_file_size_ob
-
-    if larger_max_file_size > 0:
-        app.config['MAX_CONTENT_LENGTH'] = larger_max_file_size
-    else:
-        app.config['MAX_CONTENT_LENGTH'] = None
-
-
-# Set the upload limit based on env vars to start with
-limit_upload_size()
-
-
-def get_tag_and_sha() -> str:
-    """Get the SHA of the last commit
-    """
-
-    # Get the tag of the latest commit
-    ev_tag = os.environ.get(TAG_EV)
-    if ev_tag:
-        tag = ev_tag
-    else:
-        try:
-            # This bash command calls `git tag` to get a sorted list of tags, with the most recent at the top, then uses
-            # `head` to trim it to one line
-            cmd = "git tag --sort -version:refname | head -n 1"
-
-            out_bytes = run(cmd, shell=True, capture_output=True).stdout
-            tag = str(out_bytes.decode()).strip()
-
-        except Exception:
-            print("ERROR: Could not determine most recent tag. Error was:\n" + format_exc(),
-                  file=sys.stderr)
-            tag = ""
-
-    # Get the SHA associated with this tag
-    ev_tag_sha = os.environ.get(TAG_SHA_EV)
-    if ev_tag_sha:
-        tag_sha: str | None = ev_tag_sha
-    else:
-        try:
-            cmd = f"git show {tag}" + " | head -n 1 | gawk '{print($2)}'"
-
-            out_bytes = run(cmd, shell=True, capture_output=True).stdout
-            tag_sha = str(out_bytes.decode()).strip()
-
-        except Exception:
-            print("ERROR: Could not determine SHA for most recent tag. Error was:\n" + format_exc(),
-                  file=sys.stderr)
-            tag_sha = None
-
-    # First check if the SHA is provided through an environmental variable
-    ev_sha = os.environ.get(SHA_EV)
-    if ev_sha:
-        sha = ev_sha
-    else:
-        try:
-            # This bash command calls `git log` to get info on the last commit, uses `head` to trim it to one line, then
-            # uses `gawk` to get just the second word of this line, which is the SHA of this commit
-            cmd = "git log -n 1 | head -n 1 | gawk '{print($2)}'"
-
-            out_bytes = run(cmd, shell=True, capture_output=True).stdout
-            sha = str(out_bytes.decode()).strip()
-
-        except Exception:
-            print("ERROR: Could not determine SHA of most recent commit. Error was:\n" + format_exc(),
-                  file=sys.stderr)
-            sha = ""
-
-    # If the SHA of the tag is the same as the current SHA, we indicate this by returning a blank SHA
-    if tag_sha == sha:
-        sha = ""
-
-    return (tag, sha)
-
-
-@app.route('/')
-def website():
-    """Return the web page along with relevant data
-    """
-    tag, sha = get_tag_and_sha()
-    return render_template("index.htm",
-                           token=token,
-                           max_file_size=max_file_size,
-                           max_file_size_ob=max_file_size_ob,
-                           service_mode=service_mode,
-                           production_mode=production_mode,
-                           tag=tag,
-                           sha=sha)
-
-
-@app.route('/documentation.htm')
-def documentation():
-    """Return the documentation page
-    """
-    tag, sha = get_tag_and_sha()
-    return render_template("documentation.htm",
-                           tag=tag,
-                           sha=sha)
-
-
-@app.route('/convert/', methods=['POST'])
-def convert():
-    """Convert file to a different format and save to folder 'downloads'. Delete original file. Note that downloading is
-    achieved in format.js
-    """
-
-    # Make sure the upload directory exists
-    os.makedirs(const.DEFAULT_UPLOAD_DIR, exist_ok=True)
-
-    file = request.files[FILE_TO_UPLOAD_KEY]
-    filename = secure_filename(file.filename)
-
-    qualified_filename = os.path.join(const.DEFAULT_UPLOAD_DIR, filename)
-    file.save(qualified_filename)
-    qualified_output_log = os.path.join(const.DEFAULT_DOWNLOAD_DIR,
-                                        split_archive_ext(filename)[0] + const.OUTPUT_LOG_EXT)
-
-    # Determine the input and output formats
-    d_formats = {}
-    for format_label in "to", "from":
-        name = request.form[format_label]
-        full_note = request.form[format_label+"_full"]
-
-        l_possible_formats = get_format_info(name, which="all")
-
-        # If there's only one possible format, use that
-        if len(l_possible_formats) == 1:
-            d_formats[format_label] = l_possible_formats[0]
-            continue
-
-        # Otherwise, find the format with the matching note
-        for possible_format in l_possible_formats:
-            if possible_format.note in full_note:
-                d_formats[format_label] = possible_format
-                break
-        else:
-            print(f"Format '{name}' with full description '{full_note}' could not be found in database.",
-                  file=sys.stderr)
-            abort(const.STATUS_CODE_GENERAL)
-
-    if (not service_mode) or (request.form['token'] == token and token != ''):
-        try:
-            conversion_output = run_converter(name=request.form['converter'],
-                                              filename=qualified_filename,
-                                              data=request.form,
-                                              to_format=d_formats["to"],
-                                              from_format=d_formats["from"],
-                                              strict=(request.form['check_ext'] != "false"),
-                                              log_mode=log_mode,
-                                              log_level=log_level,
-                                              delete_input=True,
-                                              abort_callback=abort)
-        except Exception as e:
-
-            # Check for anticipated exceptions, and write a simpler message for them
-            for err_message in (const.ERR_CONVERSION_FAILED, const.ERR_CONVERTER_NOT_RECOGNISED,
-                                const.ERR_EMPTY_ARCHIVE, const.ERR_WRONG_EXTENSIONS):
-                if log_utility.string_with_placeholders_matches(err_message, str(e)):
-                    with open(qualified_output_log, "w") as fo:
-                        fo.write(str(e))
-                    abort(const.STATUS_CODE_GENERAL)
-
-            # If the exception provides a status code, get it
-            status_code: int
-            if hasattr(e, "status_code"):
-                status_code = e.status_code
-            else:
-                status_code = const.STATUS_CODE_GENERAL
-
-            # If the exception provides a message, report it
-            if hasattr(e, "message"):
-                msg = f"An unexpected exception was raised by the converter, with error message:\n{e.message}\n"
-            else:
-                # Failsafe exception message
-                msg = ("The following unexpected exception was raised by the converter:\n" +
-                       format_exc()+"\n")
-            with open(qualified_output_log, "w") as fo:
-                fo.write(msg)
-            abort(status_code)
-
-        return repr(conversion_output)
-    else:
-        # return http status code 405
-        abort(405)
-
-
-@app.route('/feedback/', methods=['POST'])
-def feedback():
-    """Take feedback data from the web app and log it
-    """
-
-    try:
-
-        entry = {
-            "datetime": log_utility.get_date_time(),
-        }
-
-        report = json.loads(request.form['data'])
-
-        for key in ["type", "missing", "reason", "from", "to"]:
-            if key in report:
-                entry[key] = str(report[key])
-
-        # Write data in JSON format and send to stdout
-        logLock.acquire()
-        sys.stdout.write(f"{json.dumps(entry) + '\n'}")
-        logLock.release()
-
-        return Response(status=201)
-
-    except Exception:
-
-        return Response(status=400)
-
-
-@app.route('/delete/', methods=['POST'])
-def delete():
-    """Delete files in folder 'downloads'
-    """
-
-    realbase = os.path.realpath(const.DEFAULT_DOWNLOAD_DIR)
-
-    realfilename = os.path.realpath(os.path.join(const.DEFAULT_DOWNLOAD_DIR, request.form['filename']))
-    reallogname = os.path.realpath(os.path.join(const.DEFAULT_DOWNLOAD_DIR, request.form['logname']))
-
-    if realfilename.startswith(realbase + os.sep) and reallogname.startswith(realbase + os.sep):
-
-        os.remove(realfilename)
-        os.remove(reallogname)
-
-        return 'okay'
-
-    else:
-
-        return Response(status=400)
-
-
-@app.route('/del/', methods=['POST'])
-def delete_file():
-    """Delete file (cURL)
-    """
-    os.remove(request.form['filepath'])
-    return 'Server-side file ' + request.form['filepath'] + ' deleted\n'
-
-
-@app.route('/data/', methods=['GET'])
-def data():
-    """Check that the incoming token matches the one sent to the user (should mostly prevent spambots). Write date- and
-    time-stamped user input to server-side file 'user_responses'.
-
-    $$$$$$$$$$ Retained in case direct logging is required in the future. $$$$$$$$$$
-
-    Returns
-    -------
-    str
-        Output status - 'okay' if exited successfuly
-    """
-    if service_mode and request.args['token'] == token and token != '':
-        message = '[' + log_utility.get_date_time() + '] ' + request.args['data'] + '\n'
-
-        with open("user_responses", "a") as f:
-            f.write(message)
-
-        return 'okay'
-    else:
-        # return http status code 405
-        abort(405)
-
-
-def start_app():
-    """Start the Flask app - this requires being run from the base directory of the project, so this changes the
-    current directory to there. Anything else which changes it while the app is running may interfere with its proper
-    execution.
-    """
-
-    os.chdir(os.path.join(psdi_data_conversion.__path__[0], ".."))
-    app.run(debug=debug_mode)
+app = get_app()
 
 
 def main():
@@ -443,27 +59,8 @@ def main():
     args = parser.parse_args()
 
     if not args.use_env_vars:
-
-        global max_file_size
-        max_file_size = args.max_file_size*const.MEGABYTE
-
-        global max_file_size_ob
-        max_file_size_ob = args.max_file_size_ob*const.MEGABYTE
-
-        global service_mode
-        service_mode = args.service_mode
-
-        global debug_mode
-        debug_mode = args.debug
-
-        global production_mode
-        production_mode = not args.dev_mode
-
-        global log_mode
-        log_mode = args.log_mode
-
-        global log_level
-        log_level = args.log_level
+        # Overwrite the values from environmental variables with the values from the command-line arguments
+        update_env(args)
 
     # Set the upload limit based on provided arguments now
     limit_upload_size()

psdi_data_conversion/constants.py

@@ -36,6 +36,9 @@ import shutil
 # The name of the command-line script
 CL_SCRIPT_NAME = "psdi-data-convert"
 
+# The name of the Flask app
+APP_NAME = "psdi_data_conversion"
+
 # Environmental variables
 LOG_MODE_EV = "LOG_MODE"
 LOG_LEVEL_EV = "LOG_LEVEL"
@@ -52,8 +55,8 @@ MEGABYTE = 1024*1024
 DEFAULT_MAX_FILE_SIZE = 0 * MEGABYTE
 DEFAULT_MAX_FILE_SIZE_OB = 1 * MEGABYTE
 
-DEFAULT_UPLOAD_DIR = './psdi_data_conversion/static/uploads'
-DEFAULT_DOWNLOAD_DIR = './psdi_data_conversion/static/downloads'
+DEFAULT_INPUT_DIR = './psdi_data_conversion/static/uploads'
+DEFAULT_OUTPUT_DIR = './psdi_data_conversion/static/downloads'
 
 # Filename of the database, relative to the base of the python package
 DATABASE_FILENAME = "static/data/data.json"
@@ -150,11 +153,6 @@ QUAL_2D_LABEL = "2D atomic coordinates are"
 QUAL_3D_KEY = "three_dim"
 QUAL_3D_LABEL = "2D atomic coordinates are"
 
-D_QUAL_LABELS = {QUAL_COMP_KEY: QUAL_COMP_LABEL,
-                 QUAL_CONN_KEY: QUAL_CONN_LABEL,
-                 QUAL_2D_KEY: QUAL_2D_LABEL,
-                 QUAL_3D_KEY: QUAL_3D_LABEL}
-
 # Notes for conversion quality
 QUAL_NOTE_IN_UNKNOWN = ("Potential data extrapolation: {} represented in the output format but its representation in "
                         "the input format is unknown")
@@ -187,3 +185,9 @@ ERR_WRONG_EXTENSIONS = "Input file '{file}' does not have expected extension '{e
 ERR_EMPTY_ARCHIVE = "No files to convert were contained in archive"
 ERR_CONVERSION_FAILED = ("File conversion failed for one or more files. Lines from the output log "
                          "{} which indicate possible sources of error: ")
+
+# Misc
+# ----
+
+# Year in seconds
+YEAR = 365*24*60*60

psdi_data_conversion/converter.py

@@ -5,22 +5,20 @@ Created 2024-12-10 by Bryan Gillis.
 Class and functions to perform file conversion
 """
 
-from dataclasses import dataclass, field
-import json
 import glob
 import importlib
+import json
 import os
 import sys
 import traceback
-from typing import Any, Callable, NamedTuple
-from multiprocessing import Lock
-from psdi_data_conversion import log_utility
 from collections.abc import Callable
 from dataclasses import dataclass, field
+from multiprocessing import Lock
 from tempfile import TemporaryDirectory
 from typing import Any, NamedTuple
 
 from psdi_data_conversion import constants as const
+from psdi_data_conversion import log_utility
 from psdi_data_conversion.converters import base
 from psdi_data_conversion.converters.openbabel import CONVERTER_OB
 from psdi_data_conversion.file_io import (is_archive, is_supported_archive, pack_zip_or_tar, split_archive_ext,
@@ -184,9 +182,9 @@ def get_converter(*args, name=const.CONVERTER_DEFAULT, **converter_kwargs) -> ba
     use_envvars : bool
         If set to True, environment variables will be checked for any that set options for this class and used,
         default False
-    upload_dir : str
+    input_dir : str
         The location of input files relative to the current directory
-    download_dir : str
+    output_dir : str
         The location of output files relative to the current directory
     max_file_size : float
         The maximum allowed file size for input/output files, in MB, default 1 MB for Open Babel, unlimited for other
@@ -315,11 +313,18 @@ def check_from_format(filename: str,
     return False
 
 
+def _run_single_file_conversion(*args, **kwargs):
+    """Run a conversion on a single file, after all arguments have been checked
+    """
+    return get_converter(*args, **kwargs).run()
+
+
 def run_converter(filename: str,
                   to_format: str,
                   *args,
                   from_format: str | None = None,
-                  download_dir=const.DEFAULT_DOWNLOAD_DIR,
+                  input_dir=const.DEFAULT_INPUT_DIR,
+                  output_dir=const.DEFAULT_OUTPUT_DIR,
                   max_file_size=None,
                   log_file: str | None = None,
                   log_mode=const.LOG_SIMPLE,
@@ -351,9 +356,9 @@ def run_converter(filename: str,
     use_envvars : bool
         If set to True, environment variables will be checked for any that set options for this class and used,
         default False
-    upload_dir : str
+    input_dir : str
         The location of input files relative to the current directory
-    download_dir : str
+    output_dir : str
         The location of output files relative to the current directory
     strict : bool
         If True and `from_format` is not None, will fail if any input file has the wrong extension (including files
@@ -410,7 +415,12 @@ def run_converter(filename: str,
     # converter class, so it needs to be set up here to match what will be set up there
     if log_file is None:
         base_filename = os.path.basename(split_archive_ext(filename)[0])
-        log_file = os.path.join(download_dir, base_filename + const.OUTPUT_LOG_EXT)
+        log_file = os.path.join(output_dir, base_filename + const.OUTPUT_LOG_EXT)
+
+    if os.path.exists(filename):
+        qualified_filename = filename
+    else:
+        qualified_filename = os.path.join(input_dir, filename)
 
     # Check if the filename is for an archive file, and handle appropriately
 
@@ -425,15 +435,16 @@ def run_converter(filename: str,
         # Not an archive, so just get and run the converter straightforwardly
         if from_format is not None:
             check_from_format(filename, from_format, strict=strict)
-        l_run_output.append(get_converter(filename,
+        l_run_output.append(_run_single_file_conversion(filename,
                             to_format,
                             *args,
                             from_format=from_format,
-                            download_dir=download_dir,
+                            input_dir=input_dir,
+                            output_dir=output_dir,
                             max_file_size=max_file_size,
                             log_file=log_file,
                             log_mode=log_mode,
-                            **converter_kwargs).run())
+                            **converter_kwargs))
 
     elif not is_supported_archive(filename):
         raise base.FileConverterInputException(f"{filename} is an unsupported archive type. Supported types are: "
@@ -443,7 +454,7 @@ def run_converter(filename: str,
         # The filename is of a supported archive type. Make a temporary directory to extract its contents
         # to, then run the converter on each file extracted
         with TemporaryDirectory() as extract_dir:
-            l_filenames = unpack_zip_or_tar(filename, extract_dir=extract_dir)
+            l_filenames = unpack_zip_or_tar(qualified_filename, extract_dir=extract_dir)
 
             # Check for no files in archive
             if len(l_filenames) == 0:
@@ -468,15 +479,15 @@ def run_converter(filename: str,
                 individual_log_mode = log_mode if log_mode != const.LOG_FULL else const.LOG_FULL_FORCE
 
                 try:
-                    individual_run_output = get_converter(extracted_filename,
-                                                          to_format,
-                                                          *args,
-                                                          from_format=from_format,
-                                                          download_dir=download_dir,
-                                                          log_file=individual_log_file,
-                                                          log_mode=individual_log_mode,
-                                                          max_file_size=remaining_file_size,
-                                                          **converter_kwargs).run()
+                    individual_run_output = _run_single_file_conversion(extracted_filename,
+                                                                        to_format,
+                                                                        *args,
+                                                                        from_format=from_format,
+                                                                        output_dir=output_dir,
+                                                                        log_file=individual_log_file,
+                                                                        log_mode=individual_log_mode,
+                                                                        max_file_size=remaining_file_size,
+                                                                        **converter_kwargs)
                 except base.FileConverterAbortException as e:
                     # If the run fails, create a run output object to indicate that
                     individual_run_output = base.FileConversionResult(log_filename=individual_log_file,
@@ -573,12 +584,13 @@ def run_converter(filename: str,
                 "input_filename": in_filename,
                 "output_filename": run_output.output_filename[l_index:r_index],
                 "input_size": input_size,
-                "output_size": output_size }
+                "output_size": output_size}
 
-            for key in [ "converter", "coordinates", "coordOption", "from_flags",
-                "to_flags", "from_arg_flags", "to_arg_flags" ]:
+            for key in ["converter", "coordinates", "coordOption", "from_flags",
+                        "to_flags", "from_arg_flags", "to_arg_flags"]:
                 if key in converter_kwargs['data'] and converter_kwargs['data'][key] != "" and not \
-                    ((key == "coordinates" or key == "coordOption") and converter_kwargs['data']['coordinates'] == "neither") :
+                        ((key == "coordinates" or key == "coordOption") and
+                         converter_kwargs['data']['coordinates'] == "neither"):
                     entry[key] = converter_kwargs['data'][key]
 
             entry["outcome"] = outcome
@@ -595,6 +607,7 @@ def run_converter(filename: str,
 
     return run_output
 
+
 def set_size_units(size):
     if size >= 1024:
         return str('%.3f' % (size / 1024)) + ' kB'