psdi-data-conversion 0.0.36__py3-none-any.whl → 0.0.38__py3-none-any.whl

psdi_data_conversion/templates/index.htm

@@ -57,53 +57,67 @@
 
   <form name="gui" class="gui">
     <div class="max-width-box">
-      <p>Select 'from' and 'to' file formats in the 'Convert from/to' boxes, in either order. Typing where indicated
-        filters the options (case insensitive); for example, typing 'can' or 'NON' reduces the number of options to one:
-        'can: Canonical SMILES.' If you change your mind about a conversion, it is advisable to click on the 'Reset'
-        button. When both formats have been selected, converters able to carry out this conversion appear in the
-        'Conversion options' box along with an indication of conversion quality (many conversions have not yet been
-        tested) and whether or not the converter is supported on this site.</p>
-      <p>
-        <label for="searchFrom" id="fromLabel">Convert from:</label><br>
-        <input type="text" placeholder="-- type here to filter options --" size="12" id="searchFrom"
-          class="large-width"><br>
-        <select size="4" id="fromList" class="large-width"></select>
+      <p>Your one-stop shop for file format conversion, PSDI's Data Conversion Service enables users to seamlessly
+        convert between different file formats and assess the quality of proposed conversions. This is the web version
+        of our service. To find out more about Data Conversion and our other tools, click
+        <a href="https://resources.psdi.ac.uk/resource-themes/7bcc430a-fdc9-413e-bf32-bf163236430b" id="more"
+          target="_blank">here.</a>
       </p>
-      <div class="smallGap"></div>
-      <p>
-        <label for="searchTo" id="toLabel">Convert to:</label><br>
-        <input type="text" placeholder="-- type here to filter options --" size="12" id="searchTo"
-          class="large-width"><br>
-        <select size="4" id="toList" class="large-width"></select>
-      </p>
-      <input type="button" class="button" value=" Reset " name="resetButton" id="resetButton">
-      <br><br>
-      <p>Selecting a converter displays information
-        about it, along with a link to a relevant website. Selecting a supported converter affords the opportunity to
-        carry out a conversion on this site. Clicking on the 'Yes' button takes us to the 'Conversion' page. If a
-        file format has not been found, or if there are no converters capable of carrying out a required conversion
-        please click on 'Report Missing Format/Conversion' in the navigation bar.</p>
-      <p>
-        <label for="success">Select from available conversion
-          options:</label><br>
-        <select size="4" id="success" class="large-width"></select>
-      </p>
-      <span class="normalText">Show how the conversion quality was determined for the selected converter.</span>
-      <input type="button" class="button" value=" Show " name="showButton" id="showButton">
-      <br>
-      <h6>Converter details:</h6>
-      <p id="converter" , class="init-hidden">
-        <span id="name"></span>
-        <br><span id="description"></span>
-        <br><span id="url"></span>
-        <br><span id="info"></span>
-        <a id="visit" target="_blank">this website.</a>
-      <div id="formatWarning" class="init-hidden"></div>
-      <div id="offer" class="init-hidden">
-        <span id="question"></span>
-        <input type="button" class="button" value=" Yes " name="yesButton" id="yesButton">
+      <div id="format-selection">
+        <p>Select 'from' and 'to' file formats in the 'Convert from/to' boxes, in either order. Typing where indicated
+          filters the options (case insensitive); for example, typing 'can' or 'NON' reduces the number of options to
+          one:
+          'can: Canonical SMILES.' If you change your mind about a conversion, it is advisable to click on the 'Reset'
+          button. When both formats have been selected, converters able to carry out this conversion appear in the
+          'Conversion options' box along with an indication of conversion quality (many conversions have not yet been
+          tested) and whether or not the converter is supported on this site.</p>
+        <p>
+          <label for="searchFrom" id="fromLabel">Convert from:</label><br>
+          <input type="text" placeholder="-- type here to filter options --" size="12" id="searchFrom"
+            class="large-width"><br>
+          <select size="4" id="fromList" class="large-width"></select>
+        </p>
+        <div class="smallGap"></div>
+        <p>
+          <label for="searchTo" id="toLabel">Convert to:</label><br>
+          <input type="text" placeholder="-- type here to filter options --" size="12" id="searchTo"
+            class="large-width"><br>
+          <select size="4" id="toList" class="large-width"></select>
+        </p>
+        <input type="button" class="button" value=" Reset " name="resetButton" id="resetButton">
+        <br><br>
+        <p>Selecting a converter displays information
+          about it, along with a link to a relevant website. Selecting a supported converter affords the opportunity to
+          carry out a conversion on this site. Clicking on the 'Yes' button takes us to the 'Conversion' page. If a
+          file format has not been found, or if there are no converters capable of carrying out a required conversion
+          please click on 'Report Missing Format/Conversion' in the navigation bar.</p>
+        <p>
+          <label for="success">Select from available conversion
+            options:</label><br>
+          <select size="4" id="success" class="large-width"></select>
+        </p>
+        <span class="normalText">Show how the conversion quality was determined for the selected converter.</span>
+        <input type="button" class="button" value=" Show " name="showButton" id="showButton">
+        <br>
+        <h6>Converter details:</h6>
+        <p id="converter" , class="init-hidden">
+          <span id="name"></span>
+          <br><span id="description"></span>
+          <br><span id="url"></span>
+          <br><span id="info"></span>
+          <a id="visit" target="_blank">this website.</a>
+        <div id="formatWarning" class="init-hidden"></div>
+        <div id="offer" class="init-hidden">
+          <span id="question"></span>
+          <input type="button" class="button" value=" Yes " name="yesButton" id="yesButton">
+        </div>
+        </p>
+      </div>
+      <div id="database-error" class="init-hidden">
+        <p><strong>ERROR:</strong> Connection to the conversion database failed. Please try refreshing this page. If
+          this issue persists, please report it to us using the "Contact" link in the header. You can also download
+          a version of this service for local use using the "Download" link in the header if you'd prefer.</p>
       </div>
-      </p>
       <div class="medGap"></div>
     </div>
   </form>

psdi_data_conversion/testing/constants.py

@@ -6,7 +6,13 @@ Constants related to unit testing
 
 import os
 
-TEST_DATA_LOC = os.path.abspath("./test_data")
+# Locations relative to the root directory of the project - to ensure the files are found, tests should chdir to this
+# directory before searching for files
 
-INPUT_TEST_DATA_LOC = TEST_DATA_LOC
-OUTPUT_TEST_DATA_LOC = os.path.join(TEST_DATA_LOC, "output")
+TEST_DATA_LOC_IN_PROJECT = "./test_data"
+
+INPUT_TEST_DATA_LOC_IN_PROJECT = TEST_DATA_LOC_IN_PROJECT
+OUTPUT_TEST_DATA_LOC_IN_PROJECT = os.path.join(TEST_DATA_LOC_IN_PROJECT, "output")
+
+DEFAULT_ORIGIN = "http://127.0.0.1:5000"
+"""The default location of the server for running GUI unit tests"""

psdi_data_conversion/testing/conversion_callbacks.py

@@ -15,8 +15,8 @@ from tempfile import TemporaryDirectory
 from psdi_data_conversion.constants import DATETIME_RE_RAW
 from psdi_data_conversion.file_io import unpack_zip_or_tar
 from psdi_data_conversion.log_utility import string_with_placeholders_matches
-from psdi_data_conversion.testing.constants import OUTPUT_TEST_DATA_LOC
-from psdi_data_conversion.testing.utils import ConversionTestInfo, LibraryConversionTestInfo, check_file_match
+from psdi_data_conversion.testing.constants import OUTPUT_TEST_DATA_LOC_IN_PROJECT
+from psdi_data_conversion.testing.utils import ConversionTestInfo, check_file_match
 
 
 class MultiCallback:
@@ -326,7 +326,7 @@ class CheckException:
     """Callable class which checks an exception raised for its type, status code, and message. Tests will only be
     run on tests with the python library, as that's the only route that provides exceptions."""
 
-    ex_type: type[Exception]
+    ex_type: type[Exception] | None = None
     """The expected type of the raised exception (subclasses of it will also be allowed)"""
 
     ex_message: str | None = None
@@ -339,7 +339,8 @@ class CheckException:
     def __call__(self, test_info: ConversionTestInfo) -> str:
         """Perform the check on the exception"""
 
-        if not isinstance(test_info, LibraryConversionTestInfo):
+        # Skip check on CLA, since this won't catch any exceptions
+        if test_info.run_type == "cla":
             return ""
 
         # Confirm that an exception was indeed raised
@@ -350,8 +351,9 @@ class CheckException:
         l_errors: list[str] = []
 
         # Check the exception type
-        if not issubclass(exc_info.type, self.ex_type):
-            l_errors.append(f"ERROR: Raised exception is of type '{exc_info.type}', but expected '{self.ex_type}'")
+        if self.ex_type and not issubclass(exc_info.type, self.ex_type):
+            l_errors.append(f"ERROR: Raised exception is of type '{exc_info.type}', but expected '{self.ex_type}'. "
+                            f"Raised exception's message was: {str(exc_info)}")
 
         exc = exc_info.value
 
@@ -389,6 +391,6 @@ class MatchOutputFile:
     def __call__(self, test_info: ConversionTestInfo):
         """Run the check comparing the two files"""
 
-        qualified_ex_output_filename = os.path.join(OUTPUT_TEST_DATA_LOC, self.ex_output_filename)
+        qualified_ex_output_filename = os.path.join(OUTPUT_TEST_DATA_LOC_IN_PROJECT, self.ex_output_filename)
 
         return check_file_match(test_info.qualified_out_filename, qualified_ex_output_filename)

psdi_data_conversion/testing/conversion_test_specs.py

@@ -8,32 +8,45 @@ application, and GUI.
 
 from psdi_data_conversion import constants as const
 from psdi_data_conversion.converters.atomsk import CONVERTER_ATO
-from psdi_data_conversion.converters.base import (FileConverterAbortException, FileConverterHelpException,
-                                                  FileConverterInputException, FileConverterSizeException)
+from psdi_data_conversion.converters.base import (FileConverterAbortException, FileConverterInputException,
+                                                  FileConverterSizeException)
 from psdi_data_conversion.converters.c2x import CONVERTER_C2X
 from psdi_data_conversion.converters.openbabel import CONVERTER_OB, COORD_GEN_KEY, COORD_GEN_QUAL_KEY
+from psdi_data_conversion.database import FileConverterDatabaseException
 from psdi_data_conversion.testing.conversion_callbacks import (CheckArchiveContents, CheckException, CheckLogContents,
                                                                CheckLogContentsSuccess, CheckFileStatus,
                                                                CheckStderrContents, CheckStdoutContents,
                                                                MatchOutputFile, MultiCallback as MCB)
 from psdi_data_conversion.testing.utils import ConversionTestSpec as Spec
 
-
 l_all_test_specs: list[Spec] = []
 """All test specs defined in this file"""
 
-l_all_test_specs.append(Spec(name="Basic",
-                             filename=["1NE6.mmcif", "standard_test.cdxml",
+l_all_test_specs.append(Spec(name="Standard Single Test",
+                             filename="standard_test.cdxml",
+                             to_format="inchi",
+                             callback=MCB(CheckFileStatus(),
+                                          CheckLogContentsSuccess(),
+                                          MatchOutputFile("standard_test.inchi")),
+                             ))
+
+l_all_test_specs.append(Spec(name="Standard Multiple Tests",
+                             filename=["1NE6.mmcif",
                                        "hemoglobin.pdb", "aceticacid.mol", "nacl.cif",
                                        "hemoglobin.pdb", "hemoglobin.pdb", "nacl.cif",
                                        "hemoglobin.pdb", "hemoglobin.pdb", "nacl.cif",
                                        "ethanol.xyz"],
-                             to_format=["pdb", "inchi",
+                             to_format=["pdb-0",
                                         "cif", "mol2", "xyz",
                                         "cif", "xyz", "xyz",
                                         "cif", "xyz", "xyz",
                                         "cml"],
-                             converter_name=[CONVERTER_OB, CONVERTER_OB,
+                             from_format=[None,
+                                          None, None, None,
+                                          None, None, None,
+                                          "pdb-0", "pdb-0", None,
+                                          None],
+                             converter_name=[CONVERTER_OB,
                                              CONVERTER_OB, CONVERTER_OB, CONVERTER_OB,
                                              CONVERTER_ATO, CONVERTER_ATO, CONVERTER_ATO,
                                              CONVERTER_C2X, CONVERTER_C2X, CONVERTER_C2X,
@@ -57,24 +70,42 @@ l_all_test_specs.append(Spec(name="Archive",
                              filename=["caffeine-smi.zip",
                                        "caffeine-smi.tar",
                                        "caffeine-smi.tar.gz"],
+                             from_format="smi",
                              to_format="inchi",
                              callback=archive_callback,
                              ))
 """A test of converting a archives of files"""
 
-l_all_test_specs.append(Spec(name="Archive (wrong format)",
+l_all_test_specs.append(Spec(name="Archive (wrong format) - Library and CLA",
                              filename="caffeine-smi.zip",
                              to_format="inchi",
-                             conversion_kwargs=[{"from_format": "pdb"},
-                                                {"from_format": "pdb", "strict": True}],
+                             from_format=["pdb-0", "pdb-0"],
+                             conversion_kwargs=[{}, {"strict": True}],
                              expect_success=[True, False],
                              callback=[CheckStderrContents(const.ERR_WRONG_EXTENSIONS),
                                        CheckException(ex_type=FileConverterInputException,
                                                       ex_message=const.ERR_WRONG_EXTENSIONS)],
+                             compatible_with_gui=False,
                              ))
 """A test that if the user provides the wrong input format for files in an archive, and error will be output to stderr
 """
 
+l_all_test_specs.append(Spec(name="Archive (wrong format) - GUI",
+                             filename="caffeine-smi.zip",
+                             to_format="inchi",
+                             from_format=["pdb-0", "pdb-0"],
+                             conversion_kwargs=[{}, {"strict": True}],
+                             expect_success=[False, False],
+                             callback=CheckException(ex_type=FileConverterInputException,
+                                                     ex_message=const.ERR_WRONG_EXTENSIONS),
+                             compatible_with_library=False,
+                             compatible_with_cla=False,
+                             ))
+"""A test that if the user provides the wrong input format for files in an archive - variant for the GUI test, which is
+more strict
+"""
+
+
 l_all_test_specs.append(Spec(name="Log mode",
                              conversion_kwargs=[{"log_mode": const.LOG_NONE},
                                                 {"log_mode": const.LOG_STDOUT},
@@ -124,7 +155,7 @@ Not compatible with GUI tests, since the GUI doesn't support quiet mode
 
 l_all_test_specs.append(Spec(name="Open Babel Warning",
                              filename="1NE6.mmcif",
-                             to_format="pdb",
+                             to_format="pdb-0",
                              callback=CheckLogContentsSuccess(["Open Babel Warning",
                                                                "Failed to kekulize aromatic bonds",])
                              ))
@@ -138,8 +169,12 @@ l_all_test_specs.append(Spec(name="Invalid Converter",
                              converter_name="INVALID",
                              expect_success=False,
                              callback=invalid_converter_callback,
+                             compatible_with_gui=False,
                              ))
-"""A test that a proper error is returned if an invalid converter is requested"""
+"""A test that a proper error is returned if an invalid converter is requested
+
+Not compatible with GUI tests, since the GUI only offers valid converters to choose from
+"""
 
 quartz_quality_note_callback = CheckLogContentsSuccess(["WARNING",
                                                         const.QUAL_NOTE_OUT_MISSING.format(const.QUAL_2D_LABEL),
@@ -165,12 +200,17 @@ output and will be lost"""
 l_all_test_specs.append(Spec(name="Cleanup input",
                              conversion_kwargs={"delete_input": True},
                              callback=CheckFileStatus(expect_input_exists=False),
+                             compatible_with_gui=False,
                              ))
-"""A test that the input file to a conversion is deleted when cleanup is requested"""
+"""A test that the input file to a conversion is deleted when cleanup is requested.
+
+Not compatible with the GUI, since the GUI can't forcibly delete files uploaded from the user's computer
+"""
 
 l_all_test_specs.append(Spec(name="Failed conversion - bad input file",
                              filename=["quartz_err.xyz", "cyclopropane_err.mol"],
                              to_format=["inchi", "xyz"],
+                             from_format=[None, "mol-0"],
                              expect_success=False,
                              converter_name=[CONVERTER_OB, CONVERTER_C2X],
                              callback=[MCB(CheckFileStatus(expect_output_exists=False,
@@ -186,46 +226,103 @@ l_all_test_specs.append(Spec(name="Failed conversion - bad input file",
 
 quartz_error_ob_callback = CheckLogContents(["ERROR",
                                              "Problems reading an XYZ file: Could not read line #11, file error"])
-l_all_test_specs.append(Spec(name="Errors in logs",
+l_all_test_specs.append(Spec(name="Errors in logs - Library and CLA",
                              filename="quartz_err.xyz",
                              to_format="inchi",
                              converter_name=CONVERTER_OB,
                              expect_success=False,
                              callback=quartz_error_ob_callback,
+                             compatible_with_gui=False,
                              ))
-"""A test that when a conversion fails, logs are still produced and contain the expected error message"""
+"""A test that when a conversion fails in the library or CLA, logs are still produced and contain the expected error
+message"""
+
+l_all_test_specs.append(Spec(name="Errors in logs - GUI",
+                             filename="quartz_err.xyz",
+                             to_format="inchi",
+                             converter_name=CONVERTER_OB,
+                             expect_success=False,
+                             callback=CheckException(ex_type=FileConverterAbortException,
+                                                     ex_message=("Problems reading an XYZ file: Could not read line "
+                                                                 "#11, file error")),
+                             compatible_with_library=False,
+                             compatible_with_cla=False,
+                             ))
+"""A test that when a conversion fails in the GUI, the log message is output to the alert box"""
 
 l_all_test_specs.append(Spec(name="Failed conversion - invalid conversion",
                              filename=["Fapatite.ins", "nacl.mol"],
+                             from_format=["ins", "mol-0"],
                              to_format=["cml", "xyz"],
                              expect_success=False,
-                             converter_name=[CONVERTER_OB, CONVERTER_ATO],
+                             converter_name=[CONVERTER_C2X, CONVERTER_ATO],
                              callback=MCB(CheckFileStatus(expect_output_exists=False,
                                                           expect_log_exists=None),
-                                          CheckException(ex_type=FileConverterHelpException,
+                                          CheckException(ex_type=FileConverterDatabaseException,
                                                          ex_message="is not supported")),
+                             compatible_with_gui=False,
                              ))
-"""A test that a conversion that fails due an unsupported conversion will properly fail"""
+"""A test that a conversion that fails due an unsupported conversion will properly fail.
+
+Not compatible with the GUI, since the GUI only offers valid conversions.
+"""
 
-l_all_test_specs.append(Spec(name="Failed Conversion - wrong input type",
+l_all_test_specs.append(Spec(name="Blocked conversion - wrong input type",
                              filename="1NE6.mmcif",
                              to_format="cif",
-                             conversion_kwargs={"from_format": "pdb"},
+                             from_format="pdb-0",
+                             conversion_kwargs={"strict": True},
                              expect_success=False,
                              callback=MCB(CheckFileStatus(expect_output_exists=False,
-                                          expect_log_exists=None),
+                                                          expect_log_exists=None),
+                                          CheckException(ex_type=FileConverterInputException,
+                                                         ex_message=("The file extension is not {} or a zip or tar "
+                                                                     "archive extension"))),
+                             compatible_with_library=False,
+                             compatible_with_cla=False,
+                             ))
+"""A test that a conversion which is blocked in the GUI"""
+
+l_all_test_specs.append(Spec(name="Failed conversion - wrong input type",
+                             filename="1NE6.mmcif",
+                             to_format="cif",
+                             from_format="pdb-0",
+                             conversion_kwargs={"strict": False},
+                             expect_success=False,
+                             callback=MCB(CheckFileStatus(expect_output_exists=False,
+                                                          expect_log_exists=None),
                                           CheckException(ex_type=FileConverterAbortException,
-                                                         ex_message="not a valid {} file")),
+                                                         ex_message=("not a valid {} file"))),
                              ))
 """A test that a conversion which fails due to the wrong input file type will properly fail"""
 
-l_all_test_specs.append(Spec(name="Large files",
+l_all_test_specs.append(Spec(name="Large files - Library and CLA",
                              filename=["ch3cl-esp.cub", "benzyne.molden", "periodic_dmol3.outmol",
                                        "fullRhinovirus.pdb"],
                              to_format=["cdjson", "dmol", "mol", "cif"],
+                             from_format=[None, None, None, "pdb-0"],
+                             conversion_kwargs=[{}, {}, {}, {"strict": False}],
                              converter_name=[CONVERTER_OB, CONVERTER_OB, CONVERTER_OB, CONVERTER_C2X],
                              callback=CheckFileStatus(),
+                             compatible_with_gui=False,
                              ))
+"""Test that the library and CLA can process large files properly"""
+
+l_all_test_specs.append(Spec(name="Large files - GUI",
+                             filename=["ch3cl-esp.cub", "benzyne.molden",
+                                       "periodic_dmol3.outmol", "fullRhinovirus.pdb"],
+                             to_format=["cdjson", "dmol", "mol", "cif"],
+                             from_format=[None, None, None, "pdb-0"],
+                             converter_name=[CONVERTER_OB, CONVERTER_OB, CONVERTER_OB, CONVERTER_C2X],
+                             expect_success=[False, False, False, True],
+                             callback=[CheckException(ex_type=FileConverterInputException),
+                                       CheckException(ex_type=FileConverterInputException),
+                                       CheckException(ex_type=FileConverterInputException),
+                                       CheckFileStatus()],
+                             compatible_with_library=False,
+                             compatible_with_cla=False,
+                             ))
+"""Test that the GUI will refuse to process large files with OB, but will with other converters"""
 
 max_size_callback = MCB(CheckFileStatus(expect_output_exists=False),
                         CheckLogContents("file exceeds maximum size"),
@@ -234,17 +331,21 @@ max_size_callback = MCB(CheckFileStatus(expect_output_exists=False),
                                        ex_status_code=const.STATUS_CODE_SIZE))
 l_all_test_specs.append(Spec(name="Max size exceeded",
                              filename=["1NE6.mmcif", "caffeine-smi.tar.gz"],
-                             to_format="pdb",
+                             to_format="pdb-0",
                              conversion_kwargs=[{"max_file_size": 0.0001}, {"max_file_size": 0.0005}],
                              expect_success=False,
                              callback=max_size_callback,
                              compatible_with_cla=False,
+                             compatible_with_gui=False,
                              ))
 """A set of test conversion that the maximum size constraint is properly applied. In the first test, the input file
 will be greater than the maximum size, and the test should fail as soon as it checks it. In the second test, the input
 archive is smaller than the maximum size, but the unpacked files in it are greater, so it should fail midway through.
 
 Not compatible with CLA tests, since the CLA doesn't allow the imposition of a maximum size.
+
+Not compatible with GUI tests in current setup of test implementation, which doesn't let us set env vars to control
+things like maximum size on a per-test basis. May be possible to set up in the future though
 """
 
 
@@ -289,11 +390,11 @@ l_all_test_specs.append(Spec(name="Coord gen",
 """A set of tests which checks that coordinate generation options are processed correctly, by matching tests using them
 to expected output files"""
 
-l_library_test_specs = [x for x in l_all_test_specs if x.compatible_with_library]
+l_library_test_specs = [x for x in l_all_test_specs if x.compatible_with_library and not x.skip_all]
 """All test specs which are compatible with being run on the Python library"""
 
-l_cla_test_specs = [x for x in l_all_test_specs if x.compatible_with_cla]
+l_cla_test_specs = [x for x in l_all_test_specs if x.compatible_with_cla and not x.skip_all]
 """All test specs which are compatible with being run on the command-line application"""
 
-l_gui_test_specs = [x for x in l_all_test_specs if x.compatible_with_gui]
+l_gui_test_specs = [x for x in l_all_test_specs if x.compatible_with_gui and not x.skip_all]
 """All test specs which are compatible with being run on the GUI"""