psdi-data-conversion 0.0.23__py3-none-any.whl

psdi_data_conversion/__init__.py

@@ -0,0 +1,11 @@
+"""
+# PSDI Data Conversion package
+
+This is the primary package for the PSDI Data Conversion python package, including all library code and executable
+scripts used for it.
+
+The README for this project can be found either packaged alongside it or online at
+https://github.com/PSDI-UK/psdi-data-conversion/blob/main/README.md
+
+The documentation for this package is still WIP as we prepare for release.
+"""

psdi_data_conversion/app.py

@@ -0,0 +1,242 @@
+"""app.py
+
+Version 1.0, 8th November 2024
+
+This script acts as a server for the PSDI Data Conversion Service website.
+"""
+
+import hashlib
+import os
+import json
+from datetime import datetime
+from subprocess import run
+import sys
+import traceback
+from flask import Flask, request, render_template, abort, Response
+
+from psdi_data_conversion import log_utility
+from psdi_data_conversion import constants as const
+from psdi_data_conversion.converter import run_converter
+from psdi_data_conversion.file_io import split_archive_ext
+
+# Env var for the SHA of the latest commit
+SHA_EV = "SHA"
+
+# Env var for whether this is running in service mode or locally
+SERVICE_MODE_EV = "SERVICE_MODE"
+
+# Env var for whether this is a production release or development
+PRODUCTION_EV = "PRODUCTION_MODE"
+
+# Key for the label given to the file uploaded in the web interface
+FILE_TO_UPLOAD_KEY = 'fileToUpload'
+
+# Create a token by hashing the current date and time.
+dt = str(datetime.now())
+token = hashlib.md5(dt.encode('utf8')).hexdigest()
+
+# Get the service and production modes from their envvars
+service_mode_ev = os.environ.get(SERVICE_MODE_EV)
+service_mode = (service_mode_ev is not None) and (service_mode_ev.lower() == "true")
+production_mode_ev = os.environ.get(PRODUCTION_EV)
+production_mode = (production_mode_ev is not None) and (production_mode_ev.lower() == "true")
+
+# Get the logging mode and level from their envvars
+ev_log_mode = os.environ.get(const.LOG_MODE_EV)
+if ev_log_mode is None:
+    log_mode = const.LOG_MODE_DEFAULT
+else:
+    ev_log_mode = ev_log_mode.lower()
+    if ev_log_mode not in const.L_ALLOWED_LOG_MODES:
+        print(f"ERROR: Unrecognised logging option: {ev_log_mode}. Allowed options are: {const.L_ALLOWED_LOG_MODES}",
+              file=sys.stderr)
+        exit(1)
+    log_mode = ev_log_mode
+
+ev_log_level = os.environ.get(const.LOG_LEVEL_EV)
+if ev_log_level is None:
+    log_level = None
+else:
+    try:
+        log_level = log_utility.get_log_level_from_str(ev_log_level)
+    except ValueError as e:
+        print(f"ERROR: {str(e)}")
+        exit(1)
+
+app = Flask(__name__)
+
+
+def get_last_sha() -> str:
+    """Get the SHA of the last commit
+    """
+
+    # First check if the SHA is provided through an environmental variable
+    ev_sha = os.environ.get(SHA_EV)
+    if ev_sha:
+        return ev_sha
+
+    try:
+        # This bash command calls `git log` to get info on the last commit, uses `head` to trim it to one line, then
+        # uses `gawk` to get just the second word of this line, which is the SHA of this commit
+        cmd = "git log -n 1 | head -n 1 | gawk '{print($2)}'"
+
+        out_bytes = run(cmd, shell=True, capture_output=True).stdout
+        out_str = str(out_bytes.decode()).strip()
+
+    except Exception:
+        print("ERROR: Could not determine SHA of most recent commit. Error was:\n" + traceback.format_exc(),
+              file=sys.stderr)
+        out_str = "N/A"
+
+    return out_str
+
+
+@app.route('/')
+def website():
+    """Return the web page along with the token
+    """
+    # Get the maximum allowed size from the envvar for it
+    ev_max_file_size = os.environ.get(const.MAX_FILESIZE_EV)
+    if ev_max_file_size is not None:
+        max_file_size = float(ev_max_file_size)*const.MEGABYTE
+    else:
+        max_file_size = const.DEFAULT_MAX_FILE_SIZE
+
+    data = [{'token': token,
+             'max_file_size': max_file_size,
+             'service_mode': service_mode,
+             'production_mode': production_mode,
+             'sha': get_last_sha()}]
+    return render_template("index.htm", data=data)
+
+
+@app.route('/convert/', methods=['POST'])
+def convert():
+    """Convert file to a different format and save to folder 'downloads'. Delete original file. Note that downloading is
+    achieved in format.js
+    """
+
+    # Make sure the upload directory exists
+    os.makedirs(const.DEFAULT_UPLOAD_DIR, exist_ok=True)
+
+    # Save the file in the upload directory
+    file = request.files[FILE_TO_UPLOAD_KEY]
+    filename = filename = file.filename
+
+    qualified_filename = os.path.join(const.DEFAULT_UPLOAD_DIR, filename)
+    file.save(qualified_filename)
+    qualified_output_log = os.path.join(const.DEFAULT_DOWNLOAD_DIR,
+                                        split_archive_ext(filename)[0] + const.OUTPUT_LOG_EXT)
+
+    if (not service_mode) or (request.form['token'] == token and token != ''):
+        try:
+            conversion_output = run_converter(name=request.form['converter'],
+                                              filename=qualified_filename,
+                                              data=request.form,
+                                              to_format=request.form['to'],
+                                              from_format=request.form['from'],
+                                              strict=request.form['check_ext'],
+                                              log_mode=log_mode,
+                                              log_level=log_level,
+                                              delete_input=True,
+                                              abort_callback=abort)
+        except Exception as e:
+
+            # Check for anticipated exceptions, and write a simpler message for them
+            for err_message in (const.ERR_CONVERSION_FAILED, const.ERR_CONVERTER_NOT_RECOGNISED,
+                                const.ERR_EMPTY_ARCHIVE, const.ERR_WRONG_EXTENSIONS):
+                if log_utility.string_with_placeholders_matches(err_message, str(e)):
+                    with open(qualified_output_log, "w") as fo:
+                        fo.write(str(e))
+                    abort(const.STATUS_CODE_GENERAL)
+
+            # Failsafe exception message
+            msg = "The following unexpected exception was raised by the converter:\n" + traceback.format_exc()+"\n"
+            with open(qualified_output_log, "w") as fo:
+                fo.write(msg)
+            abort(const.STATUS_CODE_GENERAL)
+
+        return repr(conversion_output)
+    else:
+        # return http status code 405
+        abort(405)
+
+
+@app.route('/feedback/', methods=['POST'])
+def feedback():
+    """Take feedback data from the web app and log it
+    """
+
+    try:
+
+        entry = {
+            "datetime": log_utility.get_date_time(),
+        }
+
+        report = json.loads(request.form['data'])
+
+        for key in ["type", "missing", "reason", "from", "to"]:
+            if key in report:
+                entry[key] = str(report[key])
+
+        log_utility.append_to_log_file("feedback", entry)
+
+        return Response(status=201)
+
+    except Exception:
+
+        return Response(status=400)
+
+
+@app.route('/delete/', methods=['POST'])
+def delete():
+    """Delete files in folder 'downloads'
+    """
+
+    realbase = os.path.realpath(const.DEFAULT_DOWNLOAD_DIR)
+
+    realfilename = os.path.realpath(os.path.join(const.DEFAULT_DOWNLOAD_DIR, request.form['filename']))
+    reallogname = os.path.realpath(os.path.join(const.DEFAULT_DOWNLOAD_DIR, request.form['logname']))
+
+    if realfilename.startswith(realbase + os.sep) and reallogname.startswith(realbase + os.sep):
+
+        os.remove(realfilename)
+        os.remove(reallogname)
+
+        return 'okay'
+
+    else:
+
+        return Response(status=400)
+
+
+@app.route('/del/', methods=['POST'])
+def delete_file():
+    """Delete file (cURL)
+    """
+    os.remove(request.form['filepath'])
+    return 'Server-side file ' + request.form['filepath'] + ' deleted\n'
+
+
+@app.route('/data/', methods=['GET'])
+def data():
+    """Check that the incoming token matches the one sent to the user (should mostly prevent spambots). Write date- and
+    time-stamped user input to server-side file 'user_responses'.
+
+    $$$$$$$$$$ Retained in case direct logging is required in the future. $$$$$$$$$$
+
+    Returns
+    -------
+    str
+        Output status - 'okay' if exited successfuly
+    """
+    if service_mode and request.args['token'] == token and token != '':
+        message = '[' + log_utility.get_date_time() + '] ' + request.args['data'] + '\n'
+
+        with open("user_responses", "a") as f:
+            f.write(message)
+
+        return 'okay'
+    else:
+        # return http status code 405
+        abort(405)

psdi_data_conversion/constants.py

@@ -0,0 +1,185 @@
+"""@file psdi_data_conversion/constants.py
+
+Created 2025-01-23 by Bryan Gillis.
+
+Miscellaneous constant values used within this project.
+
+These values are stored as constants rather than hardcoded literals for various reasons, including:
+- Better assurance of consistency that the same value is used every time
+- If a value needs to be changed, this only needs to be done at one location
+- Compatibility with IDE features - a constant can be checked for validity by an IDE, while e.g. a string key for a dict
+  can't, allowing more errors to be caught and fixed by the IDE rather than at runtime
+- The use of a constant may improve readability - e.g. `MEGABYTE = 1024*1024; max_file_size = 1*MEGABYTE` is more
+  readable than `max_file_size = 1*1024*1024`, and so doesn't require a comment like the latter would
+
+There are some known drawbacks to this approach which need to be considered though:
+- Constants may obscure readability - it may be quite relevant to the reader exactly what a constant represents, which
+  is obscured until they (at minimum) mouse over it
+- More code is necessary to use a constant than a literal (at minimum it needs an extra line to define it, and when
+  stored here, it also needs a line to import it or this module)
+
+With these drawbacks in mind, we make the following recommendations for constant use:
+- Messages for the user (print/logging messages, exceptions) should by default not be stored as constants. They should
+  be made constants if it's necessary to reference their exact text elsewhere (either in the executable code or unit
+  tests). In this case, the name of the constant should be descriptive, even if this means a rather long name
+- If a value is only used in one file and only likely to ever be used in that file, it can be defined as a constant
+  there (or if used only two or three times in quick succession, left as a literal)
+- Of course, deviations from this should be made when necessary, such as to avoid circular imports
+"""
+
+import logging
+import shutil
+
+# Interface
+# ---------
+
+# The name of the command-line script
+CL_SCRIPT_NAME = "psdi-data-convert"
+
+# Environmental variables
+LOG_MODE_EV = "LOG_MODE"
+LOG_LEVEL_EV = "LOG_LEVEL"
+MAX_FILESIZE_EV = "MAX_FILESIZE"
+
+# Files and Folders
+# -----------------
+
+# Maximum output file size in bytes
+MEGABYTE = 1024*1024
+DEFAULT_MAX_FILE_SIZE = 0*MEGABYTE
+
+DEFAULT_UPLOAD_DIR = './psdi_data_conversion/static/uploads'
+DEFAULT_DOWNLOAD_DIR = './psdi_data_conversion/static/downloads'
+
+# Filename of the database, relative to the base of the python package
+DATABASE_FILENAME = "static/data/data.json"
+
+# Archive extensions and formats ('format' here meaning the value expected by shutil's archive functions)
+
+ZIP_EXTENSION = ".zip"
+ZIP_FORMAT = "zip"
+
+D_ZIP_FORMATS = {ZIP_EXTENSION: ZIP_FORMAT}
+
+TAR_EXTENSION = ".tar"
+TAR_FORMAT = "tar"
+GZTAR_EXTENSION = ".tar.gz"
+GZTAR_FORMAT = "gztar"
+BZTAR_EXTENSION = ".tar.bz"
+BZTAR_FORMAT = "bztar"
+XZTAR_EXTENSION = ".tar.xz"
+XZTAR_FORMAT = "xztar"
+
+D_TAR_FORMATS = {TAR_EXTENSION: TAR_FORMAT,
+                 GZTAR_EXTENSION: GZTAR_FORMAT,
+                 BZTAR_EXTENSION: BZTAR_FORMAT,
+                 XZTAR_EXTENSION: BZTAR_FORMAT}
+
+# A list of specifically the extensions that are combinations of multiple different extensions
+L_COMPOUND_EXTENSIONS = [GZTAR_EXTENSION, BZTAR_EXTENSION, XZTAR_EXTENSION]
+
+# Formats which are supported by shutil's built-in archive utility
+D_SUPPORTED_ARCHIVE_FORMATS = {**D_ZIP_FORMATS, **D_TAR_FORMATS}
+
+L_UNSUPPORTED_ARCHIVE_EXTENSIONS = [".rar", ".7z"]
+
+L_ALL_ARCHIVE_EXTENSIONS = [*D_SUPPORTED_ARCHIVE_FORMATS.keys(), *L_UNSUPPORTED_ARCHIVE_EXTENSIONS]
+
+
+# Logging and Formatting
+# ----------------------
+
+# Number of character spaces allocated for flags/options
+
+# Get the terminal width so we can prettily print help text
+TERM_WIDTH, _ = shutil.get_terminal_size((80, 20))
+
+# Log formatting
+LOG_FORMAT = r'[%(asctime)s] %(levelname)s [%(name)s.%(funcName)s:%(lineno)d] %(message)s'
+TIMESTAMP_FORMAT = r"%Y-%m-%d %H:%M:%S"
+
+# Regex to match date/time format
+DATE_RE_RAW = r"\d{4}-[0-1]\d-[0-3]\d"
+TIME_RE_RAW = r"[0-2]\d:[0-5]\d:[0-5]\d"
+DATETIME_RE_RAW = f"{DATE_RE_RAW} {TIME_RE_RAW}"
+
+# Log mode info and settings
+LOG_FULL = "full"
+LOG_FULL_FORCE = "full-force"
+LOG_SIMPLE = "simple"
+LOG_STDOUT = "stdout"
+LOG_NONE = "none"
+
+LOG_MODE_DEFAULT = LOG_SIMPLE
+
+L_ALLOWED_LOG_MODES = (LOG_FULL, LOG_FULL_FORCE,  LOG_SIMPLE, LOG_STDOUT, LOG_NONE)
+
+LOG_EXT = ".log"
+OUTPUT_LOG_EXT = f"{LOG_EXT}.txt"
+
+# Settings for global logger
+GLOBAL_LOG_FILENAME = "./error_log.txt"
+GLOBAL_LOGGER_LEVEL = logging.ERROR
+
+# Settings for local logger
+LOCAL_LOGGER_NAME = "data-conversion"
+DEFAULT_LOCAL_LOGGER_LEVEL = logging.INFO
+DEFAULT_LISTING_LOG_FILE = "data-convert-list" + LOG_EXT
+
+# Converters and Related
+# ----------------------
+
+# Converter names are determined based on the modules present in the 'converters' package by the 'converter' module
+# This module contains constant dicts and lists of registered converters
+
+# Default converter - this must match the name of one of the registered converters
+CONVERTER_DEFAULT = 'Open Babel'
+
+# File format properties which are used to judge conversion quality - KEY is the key for it in the database, and LABEL
+# is how we want to print it out for the user
+QUAL_COMP_KEY = "composition"
+QUAL_COMP_LABEL = "Atomic composition is"
+QUAL_CONN_KEY = "connections"
+QUAL_CONN_LABEL = "Atomic connections are"
+QUAL_2D_KEY = "two_dim"
+QUAL_2D_LABEL = "2D atomic coordinates are"
+QUAL_3D_KEY = "three_dim"
+QUAL_3D_LABEL = "2D atomic coordinates are"
+
+D_QUAL_LABELS = {QUAL_COMP_KEY: QUAL_COMP_LABEL,
+                 QUAL_CONN_KEY: QUAL_CONN_LABEL,
+                 QUAL_2D_KEY: QUAL_2D_LABEL,
+                 QUAL_3D_KEY: QUAL_3D_LABEL}
+
+# Notes for conversion quality
+QUAL_NOTE_IN_UNKNOWN = ("Potential data extrapolation: {} represented in the output format but its representation in "
+                        "the input format is unknown")
+QUAL_NOTE_OUT_UNKNOWN = ("Potential data loss: {} represented in the input format, but its representation in the "
+                         "output format is unknown")
+QUAL_NOTE_BOTH_UNKNOWN = ("Potential data loss or extrapolation: {} potentially not supported in either or both of "
+                          "the input and output formats")
+QUAL_NOTE_IN_MISSING = "Potential data extrapolation: {} represented in the output format but not the input format"
+QUAL_NOTE_OUT_MISSING = "Potential data loss: {} represented in the input format but not the output format"
+
+# Conversion quality strings
+QUAL_UNKNOWN = 'unknown'
+QUAL_VERYGOOD = 'very good'
+QUAL_GOOD = 'good'
+QUAL_OKAY = 'okay'
+QUAL_POOR = 'poor'
+QUAL_VERYPOOR = 'very poor'
+
+# Errors
+# ------
+
+# HTTP status codes for various types of errors
+STATUS_CODE_BAD_METHOD = 405
+STATUS_CODE_SIZE = 413
+STATUS_CODE_GENERAL = 500
+
+# Error messages
+ERR_CONVERTER_NOT_RECOGNISED = "Converter {} not recognized. Allowed converters are: "
+ERR_WRONG_EXTENSIONS = "Input file '{file}' does not have expected extension '{ext}'"
+ERR_EMPTY_ARCHIVE = "No files to convert were contained in archive"
+ERR_CONVERSION_FAILED = ("File conversion failed for one or more files. Lines from the output log "
+                         "{} which indicate possible sources of error: ")