psdi-data-conversion 0.0.37__py3-none-any.whl → 0.0.38__py3-none-any.whl

psdi_data_conversion/app.py

@@ -19,6 +19,7 @@ import psdi_data_conversion
 from psdi_data_conversion import log_utility
 from psdi_data_conversion import constants as const
 from psdi_data_conversion.converter import run_converter
+from psdi_data_conversion.database import FormatInfo, get_format_info
 from psdi_data_conversion.file_io import split_archive_ext
 
 # Env var for the SHA of the latest commit
@@ -139,13 +140,38 @@ def convert():
     qualified_output_log = os.path.join(const.DEFAULT_DOWNLOAD_DIR,
                                         split_archive_ext(filename)[0] + const.OUTPUT_LOG_EXT)
 
+    # Determine the input and output formats
+    d_formats = {}
+    for format_label in "to", "from":
+        name = request.form[format_label]
+        full_note = request.form[format_label+"_full"]
+
+        l_possible_formats: list[FormatInfo] = get_format_info(name, which="all")
+
+        # If there's only one possible format, use that
+        if len(l_possible_formats) == 1:
+            d_formats[format_label] = l_possible_formats[0]
+            continue
+
+        # Otherwise, find the format with the matching note
+        found = False
+        for possible_format in l_possible_formats:
+            if possible_format.note in full_note:
+                d_formats[format_label] = possible_format
+                found = True
+                break
+        if not found:
+            print(f"Format '{name}' with full description '{full_note}' could not be found in database.",
+                  file=sys.stderr)
+            abort(const.STATUS_CODE_GENERAL)
+
     if (not service_mode) or (request.form['token'] == token and token != ''):
         try:
             conversion_output = run_converter(name=request.form['converter'],
                                               filename=qualified_filename,
                                               data=request.form,
-                                              to_format=request.form['to'],
-                                              from_format=request.form['from'],
+                                              to_format=d_formats["to"],
+                                              from_format=d_formats["from"],
                                               strict=(request.form['check_ext'] != "false"),
                                               log_mode=log_mode,
                                               log_level=log_level,
@@ -286,10 +312,10 @@ def main():
                         "variables and their defaults will instead control execution. These defaults will result in "
                         "the app running in production server mode.")
 
-    parser.add_argument("--max-file-size", type=float, default=const.DEFAULT_MAX_FILE_SIZE,
+    parser.add_argument("--max-file-size", type=float, default=const.DEFAULT_MAX_FILE_SIZE/const.MEGABYTE,
                         help="The maximum allowed filesize in MB - 0 (default) indicates no maximum")
 
-    parser.add_argument("--max-file-size-ob", type=float, default=const.DEFAULT_MAX_FILE_SIZE_OB,
+    parser.add_argument("--max-file-size-ob", type=float, default=const.DEFAULT_MAX_FILE_SIZE_OB/const.MEGABYTE,
                         help="The maximum allowed filesize in MB for the Open Babel converter, taking precendence over "
                         "the general maximum file size when Open Babel is used - 0 indicates no maximum. Default 1 MB.")
 

psdi_data_conversion/constants.py

@@ -45,10 +45,11 @@ MAX_FILESIZE_OB_EV = "MAX_FILESIZE_OB"
 # Files and Folders
 # -----------------
 
-# Maximum output file size in bytes
 MEGABYTE = 1024*1024
-DEFAULT_MAX_FILE_SIZE = 0*MEGABYTE
-DEFAULT_MAX_FILE_SIZE_OB = 1*MEGABYTE
+
+# Maximum output file size in bytes
+DEFAULT_MAX_FILE_SIZE = 0 * MEGABYTE
+DEFAULT_MAX_FILE_SIZE_OB = 1 * MEGABYTE
 
 DEFAULT_UPLOAD_DIR = './psdi_data_conversion/static/uploads'
 DEFAULT_DOWNLOAD_DIR = './psdi_data_conversion/static/downloads'
@@ -75,7 +76,7 @@ XZTAR_FORMAT = "xztar"
 D_TAR_FORMATS = {TAR_EXTENSION: TAR_FORMAT,
                  GZTAR_EXTENSION: GZTAR_FORMAT,
                  BZTAR_EXTENSION: BZTAR_FORMAT,
-                 XZTAR_EXTENSION: BZTAR_FORMAT}
+                 XZTAR_EXTENSION: XZTAR_FORMAT}
 
 # A list of specifically the extensions that are combinations of multiple different extensions
 L_COMPOUND_EXTENSIONS = [GZTAR_EXTENSION, BZTAR_EXTENSION, XZTAR_EXTENSION]
@@ -93,7 +94,7 @@ L_ALL_ARCHIVE_EXTENSIONS = [*D_SUPPORTED_ARCHIVE_FORMATS.keys(), *L_UNSUPPORTED_
 
 # Number of character spaces allocated for flags/options
 
-# Get the terminal width so we can prettily print help text
+# Get the terminal width so we can prettily print help text - default to 80 chars by 20 lines
 TERM_WIDTH, _ = shutil.get_terminal_size((80, 20))
 
 # Log formatting

psdi_data_conversion/converter.py

@@ -196,7 +196,7 @@ class FileConversionRunResult:
 
 
 def check_from_format(filename: str,
-                      from_format: str,
+                      from_format: str | int,
                       strict=False) -> bool:
     """Check that the filename for an input file ends with the expected extension
 
@@ -204,7 +204,7 @@ def check_from_format(filename: str,
     ----------
     filename : str
         The filename
-    from_format : str
+    from_format : str | int
         The expected format (extension)
     strict : bool, optional
         If True, will raise an exception on failure. Otherwise will print a warning and return False
@@ -220,14 +220,21 @@ def check_from_format(filename: str,
         If `strict` is True and the the file does not end with the expected exception
     """
 
+    # Get the name of the format
+    if isinstance(from_format, str):
+        from_format_name = from_format
+    else:
+        from psdi_data_conversion.database import get_format_info
+        from_format_name = get_format_info(from_format).name
+
     # Silently make sure `from_format` starts with a dot
-    if not from_format.startswith("."):
-        from_format = f".{from_format}"
+    if not from_format_name.startswith("."):
+        from_format_name = f".{from_format}"
 
-    if filename.endswith(from_format):
+    if filename.endswith(from_format_name):
         return True
 
-    msg = const.ERR_WRONG_EXTENSIONS.format(file=os.path.basename(filename), ext=from_format)
+    msg = const.ERR_WRONG_EXTENSIONS.format(file=os.path.basename(filename), ext=from_format_name)
 
     if strict:
         raise base.FileConverterInputException(msg)

psdi_data_conversion/converters/base.py

@@ -37,9 +37,13 @@ class FileConverterException(RuntimeError):
 
     def __init__(self,
                  *args,
-                 logged: bool = False):
+                 logged: bool = False,
+                 help: bool = False,
+                 msg_preformatted: bool = False):
         super().__init__(*args)
         self.logged = logged
+        self.help = help
+        self.msg_preformatted = msg_preformatted
 
 
 class FileConverterAbortException(FileConverterException):
@@ -76,23 +80,6 @@ class FileConverterInputException(FileConverterException):
     pass
 
 
-class FileConverterHelpException(FileConverterInputException):
-    """An exception class which indicates an error where we will likely want to help the user figure out how to
-    correctly use the CLI instead of simply printing a traceback
-    """
-
-    def __init__(self, *args, msg_preformatted=False):
-        """Init the exception, noting if the message should be treated as preformatted or not
-
-        Parameters
-        ----------
-        msg_preformatted : bool, optional
-            If True, indicates that the message of the exception has already been formatted. Default False
-        """
-        super().__init__(*args)
-        self.msg_preformatted = msg_preformatted
-
-
 if HTTPException is not None:
     l_abort_exceptions = (HTTPException, FileConverterAbortException)
 else:
@@ -157,6 +144,10 @@ class FileConverter:
     database_key_prefix: str | None = None
     """The prefix used in the database for keys related to this converter"""
 
+    supports_ambiguous_extensions: bool = False
+    """Whether or not this converter supports formats which share the same extension. This is used to enforce stricter
+    but less user-friendly requirements on format specification"""
+
     @abc.abstractmethod
     def _convert(self):
         """Run the conversion with the desired converter. This must be implemented for each converter class.
@@ -325,11 +316,10 @@ class FileConverter:
             else:
                 self.from_format = from_format
 
-            # Remove any leading periods from to/from_format
-            if self.to_format.startswith("."):
-                self.to_format = self.to_format[1:]
-            if self.from_format.startswith("."):
-                self.from_format = self.from_format[1:]
+            # Convert in and out formats to FormatInfo, and raise an exception if one is ambiguous
+            from psdi_data_conversion.database import disambiguate_formats
+            (self.from_format_info,
+             self.to_format_info) = disambiguate_formats(self.name, self.from_format, self.to_format)
 
             # Set placeholders for member variables which will be set when conversion is run
             self.in_size: int | None = None
@@ -348,7 +338,7 @@ class FileConverter:
 
             self.local_filename = os.path.split(self.in_filename)[1]
             self.filename_base = os.path.splitext(self.local_filename)[0]
-            self.out_filename = f"{self.download_dir}/{self.filename_base}.{self.to_format}"
+            self.out_filename = f"{self.download_dir}/{self.filename_base}.{self.to_format_info.name}"
 
             # Set up files to log to
             self._setup_loggers()
@@ -356,13 +346,16 @@ class FileConverter:
             # Check that the requested conversion is valid and warn of any issues unless suppressed
             if not no_check:
                 from psdi_data_conversion.database import get_conversion_quality
-                qual = get_conversion_quality(self.name, self.from_format, self.to_format)
+                qual = get_conversion_quality(self.name,
+                                              self.from_format_info.id,
+                                              self.to_format_info.id)
                 if not qual:
-                    raise FileConverterHelpException(f"Conversion from {self.from_format} to {self.to_format} "
-                                                     f"with {self.name} is not supported.")
+                    raise FileConverterInputException(f"Conversion from {self.from_format_info.name} to "
+                                                      f"{self.to_format_info.name} "
+                                                      f"with {self.name} is not supported.", help=True)
                 if qual.details:
                     msg = (":\nPotential data loss or extrapolation issues with the conversion from "
-                           f"{self.from_format} to {self.to_format}:\n")
+                           f"{self.from_format_info.name} to {self.to_format_info.name}:\n")
                     for detail_line in qual.details.split("\n"):
                         msg += f"- {detail_line}\n"
                     self.logger.warning(msg)
@@ -381,7 +374,7 @@ class FileConverter:
             # Try to run the standard abort method. There's a good chance this will fail though depending on what went
             # wrong when during init, so we fallback to printing the exception to stderr
             try:
-                if not isinstance(e, FileConverterHelpException):
+                if not (isinstance(e, FileConverterException) and e.help):
                     self.logger.error(f"Exception triggering an abort was raised while initializing the converter. "
                                       f"Exception was type '{type(e)}', with message: {str(e)}")
                     if e:
@@ -389,7 +382,7 @@ class FileConverter:
                 self._abort(message="The application encountered an error while initializing the converter:\n" +
                             traceback.format_exc(), e=e)
             except Exception as ee:
-                if isinstance(ee, (l_abort_exceptions, FileConverterHelpException)):
+                if isinstance(ee, l_abort_exceptions) or (isinstance(ee, FileConverterException) and ee.help):
                     # Don't catch a deliberate abort or help exception; let it pass through
                     raise
                 message = ("ERROR: The application encounted an error during initialization of the converter and "
@@ -486,7 +479,7 @@ class FileConverter:
                                       f"with message: {str(e)}")
                     e.logged = True
                 raise
-            if not isinstance(e, FileConverterHelpException):
+            if not (isinstance(e, FileConverterException) and e.help):
                 self.logger.error(f"Exception triggering an abort was raised while running the converter. Exception "
                                   f"was type '{type(e)}', with message: {str(e)}")
                 if e:
@@ -520,6 +513,18 @@ class FileConverter:
 
         """
 
+        def try_debug_log(msg, *args, **kwargs):
+            try:
+                self.logger.debug(msg, *args, **kwargs)
+            except AttributeError:
+                pass
+
+        def error_log(msg, *args, **kwargs):
+            try:
+                self.logger.error(msg, *args, **kwargs)
+            except AttributeError:
+                print(msg, file=sys.stderr)
+
         # Remove the input and output files if they exist
         if self.delete_input:
             self.logger.debug(f"Cleaning up input file {self.in_filename}")
@@ -527,33 +532,34 @@ class FileConverter:
                 os.remove(self.in_filename)
             except FileNotFoundError:
                 pass
+
         try:
             os.remove(self.out_filename)
-        except FileNotFoundError:
-            self.logger.debug("Application aborting; no output file found to clean up")
+        except (FileNotFoundError, AttributeError):
+            try_debug_log("Application aborting; no output file found to clean up")
         else:
-            self.logger.debug(f"Application aborting, so cleaning up output file {self.out_filename}")
+            try_debug_log(f"Application aborting, so cleaning up output file {self.out_filename}")
 
         # If we have a Help exception, override the message with its message
-        if isinstance(e, FileConverterHelpException):
-            self.logger.debug("Help exception triggered, so only using its message for output")
+        if isinstance(e, FileConverterException) and e.help:
+            try_debug_log("Help exception triggered, so only using its message for output")
             message = str(e)
 
         if message:
             # If we're adding a message in server mode, read in any prior logs, clear the log, write the message, then
             # write the prior logs
             if self.log_file is None:
-                self.logger.debug("Adding abort message to the top of the output log so it will be the first thing "
-                                  "read by the user")
+                try_debug_log("Adding abort message to the top of the output log so it will be the first thing "
+                              "read by the user")
                 prior_output_log = open(self.output_log, "r").read()
                 os.remove(self.output_log)
                 with open(self.output_log, "w") as fo:
                     fo.write(message + "\n")
                     fo.write(prior_output_log)
 
-            # Note this message in the dev logger as well
-            if not isinstance(e, FileConverterHelpException):
-                self.logger.error(message)
+            # Note this message in the error logger as well
+            if not (isinstance(e, FileConverterException) and e.help):
+                error_log(message)
                 if e:
                     e.logged = True
 
@@ -604,8 +610,8 @@ class FileConverter:
         # empty or whitespace will be stripped by the logger, so we use a lone colon, which looks least obtrusive
         return (":\n"
                 f"File name:         {self.filename_base}\n"
-                f"From:              {self.from_format}\n"
-                f"To:                {self.to_format}\n"
+                f"From:              {self.from_format_info.name} ({self.from_format_info.note})\n"
+                f"To:                {self.to_format} ({self.to_format_info.note})\n"
                 f"Converter:         {self.name}\n")
 
     def _log_success(self):
@@ -680,8 +686,8 @@ class FileConverter:
         from psdi_data_conversion.database import get_conversion_quality
 
         conversion_quality = get_conversion_quality(converter_name=self.name,
-                                                    in_format=self.from_format,
-                                                    out_format=self.to_format)
+                                                    in_format=self.from_format_info.id,
+                                                    out_format=self.to_format_info.id)
         if not conversion_quality:
             return "unknown"
         return conversion_quality.qual_str
@@ -694,10 +700,6 @@ class FileConverter:
 
         if self.delete_input:
             os.remove(self.in_filename)
-        if "from_full" in self.data:
-            self.from_format = self.data["from_full"]
-        if "to_full" in self.data:
-            self.to_format = self.data["to_full"]
         if "success" in self.data:
             self.quality = self.data["success"]
         else:
@@ -741,15 +743,16 @@ class ScriptFileConverter(FileConverter):
         # Check that all user-provided input passes security checks
         for user_args in [from_flags, to_flags, from_options, to_options]:
             if not string_is_safe(user_args):
-                raise FileConverterHelpException(f"Provided argument '{user_args}' does not pass security check - it "
-                                                 f"must match the regex {SAFE_STRING_RE.pattern}.")
+                raise FileConverterInputException(f"Provided argument '{user_args}' does not pass security check - it "
+                                                  f"must match the regex {SAFE_STRING_RE.pattern}.", help=True)
 
         env = {"DIST": get_dist()}
         if self.required_bin is not None:
             env["BIN_PATH"] = get_bin_path(self.required_bin)
 
-        process = subprocess.run(['sh', f'psdi_data_conversion/scripts/{self.script}', '--' + self.to_format,
-                                  self.in_filename, self.out_filename, from_flags, to_flags, from_options, to_options],
+        process = subprocess.run(['sh', f'psdi_data_conversion/scripts/{self.script}',
+                                  '--' + self.to_format_info.name, self.in_filename, self.out_filename, from_flags,
+                                  to_flags, from_options, to_options],
                                  env=env, capture_output=True, text=True)
 
         self.out = process.stdout

psdi_data_conversion/converters/c2x.py

@@ -27,6 +27,7 @@ class C2xFileConverter(ScriptFileConverter):
             "https://www.gnu.org/licenses/gpl-3.0.en.html. Its binaries are redistributed here under the terms of this "
             "license, and any further redistribution must also follow these terms. Its corresponding source code "
             "may be downloaded from https://www.c2x.org.uk/downloads/")
+    supports_ambiguous_extensions = True
 
 
 # Assign this converter to the `converter` variable - this lets the psdi_data_conversion.converter module detect and

psdi_data_conversion/converters/openbabel.py

@@ -9,7 +9,7 @@ from copy import deepcopy
 from openbabel import openbabel
 import py
 
-from psdi_data_conversion.converters.base import FileConverter, FileConverterHelpException
+from psdi_data_conversion.converters.base import FileConverter, FileConverterInputException
 from psdi_data_conversion.security import SAFE_STRING_RE, string_is_safe
 
 CONVERTER_OB = 'Open Babel'
@@ -30,8 +30,8 @@ def check_string_security(s: str):
     """Checks that a string is secure and raises an exception if it isn't.
     """
     if not string_is_safe(s):
-        raise FileConverterHelpException(f"Format option '{s}' does not pass security checks. It must pass the regex "
-                                         f"/{SAFE_STRING_RE.pattern}/.")
+        raise FileConverterInputException(f"Format option '{s}' does not pass security checks. It must pass the regex "
+                                          f"/{SAFE_STRING_RE.pattern}/.", help=True)
 
 
 def get_option_and_value(s: str):
@@ -60,16 +60,16 @@ def get_coord_gen(l_opts: list[str] | None) -> dict[str, str]:
 
     # No more than two arguments supplied to --coord-gen
     if l_opts is not None and len(l_opts) > 2:
-        raise FileConverterHelpException("At most two arguments may be provided to --coord-gen, the mode and "
-                                         "quality, e.g. '--coord-gen Gen3D best'")
+        raise FileConverterInputException("At most two arguments may be provided to --coord-gen, the mode and "
+                                          "quality, e.g. '--coord-gen Gen3D best'", help=True)
 
     # Coordinate generation options are valid
     if coord_gen not in L_ALLOWED_COORD_GENS:
-        raise FileConverterHelpException(f"Coordinate generation type '{coord_gen}' not recognised. Allowed "
-                                         f"types are: {L_ALLOWED_COORD_GENS}")
+        raise FileConverterInputException(f"Coordinate generation type '{coord_gen}' not recognised. Allowed "
+                                          f"types are: {L_ALLOWED_COORD_GENS}", help=True)
     if coord_gen_qual not in L_ALLOWED_COORD_GEN_QUALS:
-        raise FileConverterHelpException(f"Coordinate generation quality '{coord_gen_qual}' not recognised. "
-                                         f"Allowed qualities are: {L_ALLOWED_COORD_GEN_QUALS}")
+        raise FileConverterInputException(f"Coordinate generation quality '{coord_gen_qual}' not recognised. "
+                                          f"Allowed qualities are: {L_ALLOWED_COORD_GEN_QUALS}", help=True)
 
     return {COORD_GEN_KEY: coord_gen,
             COORD_GEN_QUAL_KEY: coord_gen_qual}
@@ -148,7 +148,7 @@ class OBFileConverter(FileConverter):
             stdouterr_ob = py.io.StdCaptureFD(in_=False)
 
             ob_conversion = openbabel.OBConversion()
-            ob_conversion.SetInAndOutFormats(self.from_format, self.to_format)
+            ob_conversion.SetInAndOutFormats(self.from_format_info.name, self.to_format_info.name)
 
             # Retrieve 'from' and 'to' option flags and arguments
             from_flags = self.data.get("from_flags", "")