psdi-data-conversion 0.0.35__py3-none-any.whl → 0.0.37__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- psdi_data_conversion/app.py +110 -10
- psdi_data_conversion/constants.py +2 -0
- psdi_data_conversion/converter.py +16 -6
- psdi_data_conversion/converters/atomsk.py +3 -1
- psdi_data_conversion/converters/base.py +99 -39
- psdi_data_conversion/converters/c2x.py +3 -1
- psdi_data_conversion/converters/openbabel.py +40 -1
- psdi_data_conversion/database.py +5 -0
- psdi_data_conversion/main.py +18 -10
- psdi_data_conversion/static/content/accessibility.htm +5 -5
- psdi_data_conversion/static/content/convert.htm +18 -13
- psdi_data_conversion/static/content/convertato.htm +40 -33
- psdi_data_conversion/static/content/convertc2x.htm +40 -33
- psdi_data_conversion/static/content/documentation.htm +4 -4
- psdi_data_conversion/static/content/download.htm +26 -10
- psdi_data_conversion/static/content/feedback.htm +4 -4
- psdi_data_conversion/static/content/index-versions/psdi-common-header.html +1 -1
- psdi_data_conversion/static/content/psdi-common-header.html +1 -1
- psdi_data_conversion/static/content/report.htm +9 -7
- psdi_data_conversion/static/javascript/common.js +20 -0
- psdi_data_conversion/static/javascript/convert.js +1 -2
- psdi_data_conversion/static/javascript/convert_common.js +80 -7
- psdi_data_conversion/static/javascript/convertato.js +1 -2
- psdi_data_conversion/static/javascript/convertc2x.js +1 -2
- psdi_data_conversion/static/javascript/format.js +12 -0
- psdi_data_conversion/static/javascript/report.js +6 -0
- psdi_data_conversion/static/styles/format.css +0 -6
- psdi_data_conversion/static/styles/psdi-common.css +10 -6
- psdi_data_conversion/templates/index.htm +10 -2
- psdi_data_conversion/testing/constants.py +6 -3
- psdi_data_conversion/testing/conversion_callbacks.py +7 -6
- psdi_data_conversion/testing/conversion_test_specs.py +333 -153
- psdi_data_conversion/testing/gui.py +366 -0
- psdi_data_conversion/testing/utils.py +108 -51
- {psdi_data_conversion-0.0.35.dist-info → psdi_data_conversion-0.0.37.dist-info}/METADATA +90 -51
- {psdi_data_conversion-0.0.35.dist-info → psdi_data_conversion-0.0.37.dist-info}/RECORD +39 -38
- {psdi_data_conversion-0.0.35.dist-info → psdi_data_conversion-0.0.37.dist-info}/WHEEL +1 -1
- {psdi_data_conversion-0.0.35.dist-info → psdi_data_conversion-0.0.37.dist-info}/entry_points.txt +1 -0
- {psdi_data_conversion-0.0.35.dist-info → psdi_data_conversion-0.0.37.dist-info}/licenses/LICENSE +0 -0
@@ -19,19 +19,55 @@ from unittest.mock import patch
|
|
19
19
|
import py
|
20
20
|
import pytest
|
21
21
|
|
22
|
+
import psdi_data_conversion
|
22
23
|
from psdi_data_conversion.constants import CONVERTER_DEFAULT, GLOBAL_LOG_FILENAME, LOG_NONE, OUTPUT_LOG_EXT
|
23
24
|
from psdi_data_conversion.converter import run_converter
|
24
25
|
from psdi_data_conversion.converters.openbabel import COORD_GEN_KEY, COORD_GEN_QUAL_KEY
|
25
26
|
from psdi_data_conversion.dist import LINUX_LABEL, get_dist
|
26
27
|
from psdi_data_conversion.file_io import is_archive, split_archive_ext
|
27
28
|
from psdi_data_conversion.main import main as data_convert_main
|
28
|
-
from psdi_data_conversion.testing.constants import
|
29
|
+
from psdi_data_conversion.testing.constants import (INPUT_TEST_DATA_LOC_IN_PROJECT, OUTPUT_TEST_DATA_LOC_IN_PROJECT,
|
30
|
+
TEST_DATA_LOC_IN_PROJECT)
|
31
|
+
|
32
|
+
|
33
|
+
def get_path_in_project(filename):
|
34
|
+
"""Get the realpath to a file contained within the project, given its project-relative path"""
|
35
|
+
|
36
|
+
old_cwd = os.getcwd()
|
37
|
+
|
38
|
+
try:
|
39
|
+
os.chdir(os.path.join(psdi_data_conversion.__path__[0], ".."))
|
40
|
+
realpath = os.path.realpath(filename)
|
41
|
+
|
42
|
+
finally:
|
43
|
+
# Change back to the previous directory
|
44
|
+
os.chdir(old_cwd)
|
45
|
+
|
46
|
+
return realpath
|
47
|
+
|
48
|
+
|
49
|
+
def get_test_data_loc():
|
50
|
+
"""Get the realpath of the base directory containing all data for tests"""
|
51
|
+
return get_path_in_project(TEST_DATA_LOC_IN_PROJECT)
|
52
|
+
|
53
|
+
|
54
|
+
def get_input_test_data_loc():
|
55
|
+
"""Get the realpath of the base directory containing input data for tests"""
|
56
|
+
return get_path_in_project(INPUT_TEST_DATA_LOC_IN_PROJECT)
|
57
|
+
|
58
|
+
|
59
|
+
def get_output_test_data_loc():
|
60
|
+
"""Get the realpath of the base directory containing expected output data for tests"""
|
61
|
+
return get_path_in_project(OUTPUT_TEST_DATA_LOC_IN_PROJECT)
|
29
62
|
|
30
63
|
|
31
64
|
@dataclass
|
32
65
|
class ConversionTestInfo:
|
33
66
|
"""Information about a tested conversion."""
|
34
67
|
|
68
|
+
run_type: str
|
69
|
+
"""One of "library", "cla", or "gui", describing which type of test run was performed"""
|
70
|
+
|
35
71
|
test_spec: SingleConversionTestSpec
|
36
72
|
"""The specification of the test conversion which was run to produce this"""
|
37
73
|
|
@@ -50,20 +86,23 @@ class ConversionTestInfo:
|
|
50
86
|
captured_stderr: str | None = None
|
51
87
|
"""Any output to stderr while the test was run"""
|
52
88
|
|
89
|
+
exc_info: pytest.ExceptionInfo | None = None
|
90
|
+
"""If the test conversion raised an exception, that exception's info, otherwise None"""
|
91
|
+
|
53
92
|
@property
|
54
93
|
def qualified_in_filename(self):
|
55
94
|
"""Get the fully-qualified name of the input file"""
|
56
|
-
return os.path.join(self.input_dir, self.test_spec.filename)
|
95
|
+
return os.path.realpath(os.path.join(self.input_dir, self.test_spec.filename))
|
57
96
|
|
58
97
|
@property
|
59
98
|
def qualified_out_filename(self):
|
60
99
|
"""Get the fully-qualified name of the output file"""
|
61
|
-
return os.path.join(self.output_dir, self.test_spec.out_filename)
|
100
|
+
return os.path.realpath(os.path.join(self.output_dir, self.test_spec.out_filename))
|
62
101
|
|
63
102
|
@property
|
64
103
|
def qualified_log_filename(self):
|
65
104
|
"""Get the fully-qualified name of the log file"""
|
66
|
-
return os.path.join(self.output_dir, self.test_spec.log_filename)
|
105
|
+
return os.path.realpath(os.path.join(self.output_dir, self.test_spec.log_filename))
|
67
106
|
|
68
107
|
@property
|
69
108
|
def qualified_global_log_filename(self):
|
@@ -71,28 +110,6 @@ class ConversionTestInfo:
|
|
71
110
|
return self.test_spec.global_log_filename
|
72
111
|
|
73
112
|
|
74
|
-
@dataclass
|
75
|
-
class LibraryConversionTestInfo(ConversionTestInfo):
|
76
|
-
"""Information about a tested conversion, specifically for when it was tested through a call to the library"""
|
77
|
-
|
78
|
-
exc_info: pytest.ExceptionInfo | None = None
|
79
|
-
"""If the test conversion raised an exception, that exception's info, otherwise None"""
|
80
|
-
|
81
|
-
|
82
|
-
@dataclass
|
83
|
-
class CLAConversionTestInfo(ConversionTestInfo):
|
84
|
-
"""Information about a tested conversion, specifically for when it was tested through a the command-line
|
85
|
-
application (CLA)
|
86
|
-
"""
|
87
|
-
|
88
|
-
|
89
|
-
@dataclass
|
90
|
-
class GUIConversionTestInfo(ConversionTestInfo):
|
91
|
-
"""Information about a tested conversion, specifically for when it was tested through the GUI (the local version of
|
92
|
-
the web app)
|
93
|
-
"""
|
94
|
-
|
95
|
-
|
96
113
|
@dataclass
|
97
114
|
class ConversionTestSpec:
|
98
115
|
"""Class providing a specification for a test file conversion.
|
@@ -102,13 +119,16 @@ class ConversionTestSpec:
|
|
102
119
|
(as if using zip on the multiple lists) to test each element in turn.
|
103
120
|
"""
|
104
121
|
|
122
|
+
name: str
|
123
|
+
"""The name of this test specification"""
|
124
|
+
|
105
125
|
filename: str | Iterable[str] = "nacl.cif"
|
106
126
|
"""The name of the input file, relative to the input test data location, or a list thereof"""
|
107
127
|
|
108
128
|
to_format: str | Iterable[str] = "pdb"
|
109
129
|
"""The format to test converting the input file to, or a list thereof"""
|
110
130
|
|
111
|
-
|
131
|
+
converter_name: str | Iterable[str] = CONVERTER_DEFAULT
|
112
132
|
"""The name of the converter to be used for the test, or a list thereof"""
|
113
133
|
|
114
134
|
conversion_kwargs: dict[str, Any] | Iterable[dict[str, Any]] = field(default_factory=dict)
|
@@ -118,12 +138,26 @@ class ConversionTestSpec:
|
|
118
138
|
expect_success: bool | Iterable[bool] = True
|
119
139
|
"""Whether or not to expect the test to succeed"""
|
120
140
|
|
141
|
+
skip: bool | Iterable[bool] = False
|
142
|
+
"""If set to true, this test will be skipped and not run. Can also be set individually for certain tests within an
|
143
|
+
array. This should typically only be used when debugging to skip working tests to more easily focus on non-working
|
144
|
+
tests"""
|
145
|
+
|
121
146
|
callback: (Callable[[ConversionTestInfo], str] |
|
122
147
|
Iterable[Callable[[ConversionTestInfo], str]] | None) = None
|
123
148
|
"""Function to be called after the conversion is performed to check in detail whether results are as expected. It
|
124
149
|
should take as its only argument a `ConversionTestInfo` and return a string. The string should be empty if the check
|
125
150
|
is passed and should explain the failure otherwise."""
|
126
151
|
|
152
|
+
compatible_with_library: bool = True
|
153
|
+
"""Whether or not this test spec is compatible with being run through the Python library, default True"""
|
154
|
+
|
155
|
+
compatible_with_cla: bool = True
|
156
|
+
"""Whether or not this test spec is compatible with being run through the command-line application, default True"""
|
157
|
+
|
158
|
+
compatible_with_gui: bool = True
|
159
|
+
"""Whether or not this test spec is compatible with being run through the GUI, default True"""
|
160
|
+
|
127
161
|
def __post_init__(self):
|
128
162
|
"""Regularize the lengths of all attribute lists, in case some were provided as single values and others as
|
129
163
|
lists, and set up initial values
|
@@ -131,7 +165,10 @@ class ConversionTestSpec:
|
|
131
165
|
|
132
166
|
# To ease maintainability, we get the list of this class's attributes automatically from its __dict__, excluding
|
133
167
|
# any which start with an underscore
|
134
|
-
self._l_attr_names: list[str] = [attr_name for attr_name in self.__dict__ if
|
168
|
+
self._l_attr_names: list[str] = [attr_name for attr_name in self.__dict__ if
|
169
|
+
not (attr_name.startswith("_") or
|
170
|
+
attr_name == "name" or
|
171
|
+
attr_name.startswith("compatible"))]
|
135
172
|
|
136
173
|
l_single_val_attrs = []
|
137
174
|
self._len: int = 1
|
@@ -172,6 +209,9 @@ class ConversionTestSpec:
|
|
172
209
|
if attr_name in l_single_val_attrs:
|
173
210
|
setattr(self, attr_name, [getattr(self, attr_name)]*self._len)
|
174
211
|
|
212
|
+
# Check if all tests should be skipped
|
213
|
+
self.skip_all = all(self.skip)
|
214
|
+
|
175
215
|
def __len__(self):
|
176
216
|
"""Get the length from the member - valid only after `__post_init__` has been called"""
|
177
217
|
return self._len
|
@@ -196,7 +236,7 @@ class SingleConversionTestSpec:
|
|
196
236
|
to_format: str
|
197
237
|
"""The format to test converting the input file to"""
|
198
238
|
|
199
|
-
|
239
|
+
converter_name: str | Iterable[str] = CONVERTER_DEFAULT
|
200
240
|
"""The name of the converter to be used for the test"""
|
201
241
|
|
202
242
|
conversion_kwargs: dict[str, Any] = field(default_factory=dict)
|
@@ -206,6 +246,9 @@ class SingleConversionTestSpec:
|
|
206
246
|
expect_success: bool = True
|
207
247
|
"""Whether or not to expect the test to succeed"""
|
208
248
|
|
249
|
+
skip: bool = False
|
250
|
+
"""If set to True, this test will be skipped, always returning success"""
|
251
|
+
|
209
252
|
callback: (Callable[[ConversionTestInfo], str] | None) = None
|
210
253
|
"""Function to be called after the conversion is performed to check in detail whether results are as expected. It
|
211
254
|
should take as its only argument a `ConversionTestInfo` and return a string. The string should be empty if the check
|
@@ -243,9 +286,14 @@ def run_test_conversion_with_library(test_spec: ConversionTestSpec):
|
|
243
286
|
with TemporaryDirectory("_input") as input_dir, TemporaryDirectory("_output") as output_dir:
|
244
287
|
# Iterate over the test spec to run each individual test it defines
|
245
288
|
for single_test_spec in test_spec:
|
289
|
+
if single_test_spec.skip:
|
290
|
+
print(f"Skipping single test spec {single_test_spec}")
|
291
|
+
continue
|
292
|
+
print(f"Running single test spec: {single_test_spec}")
|
246
293
|
_run_single_test_conversion_with_library(test_spec=single_test_spec,
|
247
294
|
input_dir=input_dir,
|
248
295
|
output_dir=output_dir)
|
296
|
+
print(f"Success for test spec: {single_test_spec}")
|
249
297
|
|
250
298
|
|
251
299
|
def _run_single_test_conversion_with_library(test_spec: SingleConversionTestSpec,
|
@@ -264,9 +312,9 @@ def _run_single_test_conversion_with_library(test_spec: SingleConversionTestSpec
|
|
264
312
|
"""
|
265
313
|
|
266
314
|
# Symlink the input file to the input directory
|
267
|
-
qualified_in_filename = os.path.join(input_dir, test_spec.filename)
|
315
|
+
qualified_in_filename = os.path.realpath(os.path.join(input_dir, test_spec.filename))
|
268
316
|
try:
|
269
|
-
os.symlink(os.path.join(
|
317
|
+
os.symlink(os.path.join(get_input_test_data_loc(), test_spec.filename),
|
270
318
|
qualified_in_filename)
|
271
319
|
except FileExistsError:
|
272
320
|
pass
|
@@ -279,7 +327,7 @@ def _run_single_test_conversion_with_library(test_spec: SingleConversionTestSpec
|
|
279
327
|
if test_spec.expect_success:
|
280
328
|
run_converter(filename=qualified_in_filename,
|
281
329
|
to_format=test_spec.to_format,
|
282
|
-
name=test_spec.
|
330
|
+
name=test_spec.converter_name,
|
283
331
|
upload_dir=input_dir,
|
284
332
|
download_dir=output_dir,
|
285
333
|
**test_spec.conversion_kwargs)
|
@@ -288,7 +336,7 @@ def _run_single_test_conversion_with_library(test_spec: SingleConversionTestSpec
|
|
288
336
|
with pytest.raises(Exception) as exc_info:
|
289
337
|
run_converter(filename=qualified_in_filename,
|
290
338
|
to_format=test_spec.to_format,
|
291
|
-
name=test_spec.
|
339
|
+
name=test_spec.converter_name,
|
292
340
|
upload_dir=input_dir,
|
293
341
|
download_dir=output_dir,
|
294
342
|
**test_spec.conversion_kwargs)
|
@@ -301,13 +349,14 @@ def _run_single_test_conversion_with_library(test_spec: SingleConversionTestSpec
|
|
301
349
|
|
302
350
|
# Compile output info for the test and call the callback function if one is provided
|
303
351
|
if test_spec.callback:
|
304
|
-
test_info =
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
352
|
+
test_info = ConversionTestInfo(run_type="library",
|
353
|
+
test_spec=test_spec,
|
354
|
+
input_dir=input_dir,
|
355
|
+
output_dir=output_dir,
|
356
|
+
success=success,
|
357
|
+
captured_stdout=stdout,
|
358
|
+
captured_stderr=stderr,
|
359
|
+
exc_info=exc_info)
|
311
360
|
callback_msg = test_spec.callback(test_info)
|
312
361
|
assert not callback_msg, callback_msg
|
313
362
|
|
@@ -324,9 +373,14 @@ def run_test_conversion_with_cla(test_spec: ConversionTestSpec):
|
|
324
373
|
with TemporaryDirectory("_input") as input_dir, TemporaryDirectory("_output") as output_dir:
|
325
374
|
# Iterate over the test spec to run each individual test it defines
|
326
375
|
for single_test_spec in test_spec:
|
376
|
+
if single_test_spec.skip:
|
377
|
+
print(f"Skipping single test spec {single_test_spec}")
|
378
|
+
continue
|
379
|
+
print(f"Running single test spec: {single_test_spec}")
|
327
380
|
_run_single_test_conversion_with_cla(test_spec=single_test_spec,
|
328
381
|
input_dir=input_dir,
|
329
382
|
output_dir=output_dir)
|
383
|
+
print(f"Success for test spec: {single_test_spec}")
|
330
384
|
|
331
385
|
|
332
386
|
def _run_single_test_conversion_with_cla(test_spec: SingleConversionTestSpec,
|
@@ -345,9 +399,9 @@ def _run_single_test_conversion_with_cla(test_spec: SingleConversionTestSpec,
|
|
345
399
|
"""
|
346
400
|
|
347
401
|
# Symlink the input file to the input directory
|
348
|
-
qualified_in_filename = os.path.join(input_dir, test_spec.filename)
|
402
|
+
qualified_in_filename = os.path.realpath(os.path.join(input_dir, test_spec.filename))
|
349
403
|
try:
|
350
|
-
os.symlink(os.path.join(
|
404
|
+
os.symlink(os.path.join(get_input_test_data_loc(), test_spec.filename),
|
351
405
|
qualified_in_filename)
|
352
406
|
except FileExistsError:
|
353
407
|
pass
|
@@ -359,7 +413,7 @@ def _run_single_test_conversion_with_cla(test_spec: SingleConversionTestSpec,
|
|
359
413
|
if test_spec.expect_success:
|
360
414
|
run_converter_through_cla(filename=qualified_in_filename,
|
361
415
|
to_format=test_spec.to_format,
|
362
|
-
name=test_spec.
|
416
|
+
name=test_spec.converter_name,
|
363
417
|
input_dir=input_dir,
|
364
418
|
output_dir=output_dir,
|
365
419
|
log_file=os.path.join(output_dir, test_spec.log_filename),
|
@@ -369,7 +423,7 @@ def _run_single_test_conversion_with_cla(test_spec: SingleConversionTestSpec,
|
|
369
423
|
with pytest.raises(SystemExit) as exc_info:
|
370
424
|
run_converter_through_cla(filename=qualified_in_filename,
|
371
425
|
to_format=test_spec.to_format,
|
372
|
-
name=test_spec.
|
426
|
+
name=test_spec.converter_name,
|
373
427
|
input_dir=input_dir,
|
374
428
|
output_dir=output_dir,
|
375
429
|
log_file=os.path.join(output_dir, test_spec.log_filename),
|
@@ -377,7 +431,7 @@ def _run_single_test_conversion_with_cla(test_spec: SingleConversionTestSpec,
|
|
377
431
|
# Get the success from whether or not the exit code is 0
|
378
432
|
success = not exc_info.value.code
|
379
433
|
|
380
|
-
qualified_out_filename = os.path.join(output_dir, test_spec.out_filename)
|
434
|
+
qualified_out_filename = os.path.realpath(os.path.join(output_dir, test_spec.out_filename))
|
381
435
|
|
382
436
|
# Determine success based on whether or not the output file exists with non-zero size
|
383
437
|
if not os.path.isfile(qualified_out_filename) or os.path.getsize(qualified_out_filename) == 0:
|
@@ -390,12 +444,13 @@ def _run_single_test_conversion_with_cla(test_spec: SingleConversionTestSpec,
|
|
390
444
|
|
391
445
|
# Compile output info for the test and call the callback function if one is provided
|
392
446
|
if test_spec.callback:
|
393
|
-
test_info =
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
447
|
+
test_info = ConversionTestInfo(run_type="cla",
|
448
|
+
test_spec=test_spec,
|
449
|
+
input_dir=input_dir,
|
450
|
+
output_dir=output_dir,
|
451
|
+
success=success,
|
452
|
+
captured_stdout=stdout,
|
453
|
+
captured_stderr=stderr)
|
399
454
|
callback_msg = test_spec.callback(test_info)
|
400
455
|
assert not callback_msg, callback_msg
|
401
456
|
|
@@ -426,6 +481,8 @@ def run_converter_through_cla(filename: str,
|
|
426
481
|
The directory which contains the output file
|
427
482
|
log_file : str
|
428
483
|
The desired name of the log file
|
484
|
+
conversion_kwargs : Any
|
485
|
+
Additional arguments describing the conversion
|
429
486
|
"""
|
430
487
|
|
431
488
|
# Start the argument string with the arguments we will always include
|
@@ -1,6 +1,6 @@
|
|
1
|
-
Metadata-Version: 2.
|
1
|
+
Metadata-Version: 2.4
|
2
2
|
Name: psdi_data_conversion
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.37
|
4
4
|
Summary: Chemistry file format conversion service, provided by PSDI
|
5
5
|
Project-URL: Homepage, https://data-conversion.psdi.ac.uk/
|
6
6
|
Project-URL: Documentation, https://psdi-uk.github.io/psdi-data-conversion/
|
@@ -278,7 +278,10 @@ This is the repository for the PSDI PF2 Chemistry File Format Conversion project
|
|
278
278
|
- [Installation and Setup](#installation-and-setup)
|
279
279
|
- [Running the App](#running-the-app)
|
280
280
|
- [Testing](#testing)
|
281
|
-
- [
|
281
|
+
- [Troubleshooting](#troubleshooting)
|
282
|
+
- [OSError: [Errno 24] Too many open files](#oserror-errno-24-too-many-open-files)
|
283
|
+
- [Errors running c2x or Atomsk converters](#errors-running-c2x-or-atomsk-converters)
|
284
|
+
- [Licensing](#licensing)
|
282
285
|
- [Contributors](#contributors)
|
283
286
|
- [Funding](#funding)
|
284
287
|
|
@@ -312,15 +315,17 @@ This is the repository for the PSDI PF2 Chemistry File Format Conversion project
|
|
312
315
|
- `test_data`
|
313
316
|
- (Files used for testing the project)
|
314
317
|
- `tests`
|
315
|
-
-
|
318
|
+
- `gui`
|
319
|
+
- (Unit tests for the GUI, aka the local version of the web app)
|
320
|
+
- `python`
|
321
|
+
- (Unit tests for the Python library and command-line application)
|
316
322
|
- `CHANGELOG.md` (Updates since initial public release)
|
317
323
|
- `CONTRIBUTING.md` (Guidelines and information for contributors to the project)
|
318
324
|
- `DOCKERFILE` (Dockerfile for image containerising PSDI's data conversion service)
|
319
|
-
- `LICENSE` (Apache
|
325
|
+
- `LICENSE` (Apache License version 2.0)
|
320
326
|
- `pyproject.toml` (Python project metadata and settings)
|
321
327
|
- `README.md` (This file)
|
322
328
|
- `requirements.txt` (Requirements for the web app deployment of this project)
|
323
|
-
- `run_local.sh` (Helper script to run the web app locally)
|
324
329
|
|
325
330
|
## Requirements
|
326
331
|
|
@@ -403,13 +408,21 @@ In addition to the dependencies listed above, this project uses the assets made
|
|
403
408
|
|
404
409
|
### Installation
|
405
410
|
|
406
|
-
The CLA and Python library are installed together. This
|
411
|
+
The CLA and Python library are installed together. This project is available on PyPI, and so can be installed via pip with:
|
412
|
+
|
413
|
+
```bash
|
414
|
+
pip install psdi-data-conversion
|
415
|
+
```
|
416
|
+
|
417
|
+
If you wish to install from source, this can be done most easily by cloning the project and then executing:
|
407
418
|
|
408
419
|
```bash
|
409
420
|
pip install .
|
410
421
|
```
|
411
422
|
|
412
|
-
|
423
|
+
from this project's directory. You can also replace the '.' in this command with the path to this project's directory to install it from elsewhere.
|
424
|
+
|
425
|
+
**Note:** This project uses git to determine the version number. If you clone the repository, you won't have to do anything special here, but if you get the source e.g. by extracting a release archive, you'll have to do one additional step before running the command above. If you have git installed, simply run `git init` in the project directory and it will be able to install. Otherwise, edit the project's `pyproject.toml` file to uncomment the line that sets a fixed version, and comment out the lines that set it up to determine the version from git - these are pointed out in the comments there.
|
413
426
|
|
414
427
|
Depending on your system, it may not be possible to install packages in this manner without creating a virtual environment to do so in. You can do this by first installing the `venv` module for Python3 with e.g.:
|
415
428
|
|
@@ -572,51 +585,25 @@ Enter https://data-conversion.psdi.ac.uk/ in a browser. Guidance on usage is giv
|
|
572
585
|
|
573
586
|
### Installation and Setup
|
574
587
|
|
575
|
-
|
588
|
+
This project is available on PyPI, and so can be installed via pip, including the necessary dependencies for the GUI, with:
|
576
589
|
|
577
590
|
```bash
|
578
|
-
pip install
|
591
|
+
pip install psdi-data-conversion'[gui]'
|
579
592
|
```
|
580
593
|
|
581
|
-
If
|
582
|
-
|
583
|
-
If you've cloned this repository, you can use the `run_local.sh` bash script to run the application. Otherwise (e.g. if you've installed from a wheel or PyPI), copy and paste the following into a script:
|
594
|
+
If you wish to install the project locally from source, this can be done most easily by cloning the project and then executing:
|
584
595
|
|
585
596
|
```bash
|
586
|
-
|
587
|
-
|
588
|
-
# The envvar MAX_FILESIZE can be used to set the maximum allowed filesize in MB - 0 indicates no maximum
|
589
|
-
if [ -z $MAX_FILESIZE ]; then
|
590
|
-
export MAX_FILESIZE=0
|
591
|
-
fi
|
592
|
-
|
593
|
-
# Logging control - "full" sets server-style logging, which is necessary to produce the output logs with the expected
|
594
|
-
# names. This should not be changed, or else errors will occur
|
595
|
-
export LOG_MODE=full
|
596
|
-
|
597
|
-
# The level to log at. Leave blank for defaults, which logs at INFO level for user output and ERROR level for the server
|
598
|
-
# log and stdout. If set to a different value here (e.g. DEBUG), all these channels will be set to that level
|
599
|
-
export LOG_LEVEL=
|
600
|
-
|
601
|
-
# The envvar SERVICE_MODE can be set to "true" to make this behave as if it's running as the public web service -
|
602
|
-
# uncomment the following line to enable that
|
603
|
-
# export SERVICE_MODE=true
|
604
|
-
|
605
|
-
# Uncomment the following line to enable debug mode
|
606
|
-
# export FLASK_ENV=development
|
607
|
-
|
608
|
-
# Execute a local run of the application from the proper path
|
609
|
-
|
610
|
-
PACKAGE_PATH=`python -c "import psdi_data_conversion; print(psdi_data_conversion.__path__[0])"`
|
611
|
-
cd $PACKAGE_PATH/..
|
612
|
-
python -m flask --app psdi_data_conversion/app.py run
|
597
|
+
pip install .'[gui]'
|
613
598
|
```
|
614
599
|
|
615
|
-
If
|
600
|
+
**Note:** This project uses git to determine the version number. If you clone the repository, you won't have to do anything special here, but if you get the source e.g. by extracting a release archive, you'll have to do one additional step before running the command above. If you have git installed, simply run `git init` in the project directory and it will be able to install. Otherwise, edit the project's `pyproject.toml` file to uncomment the line that sets a fixed version, and comment out the lines that set it up to determine the version from git - these are pointed out in the comments there.
|
601
|
+
|
602
|
+
If your system does not allow installation in this manner, it may be necessary to set up a virtual environment. See the instructions in the [command-line application installation](#installation) section above for how to do that, and then try to install again once you've set one up and activated it.
|
616
603
|
|
617
604
|
### Running the App
|
618
605
|
|
619
|
-
|
606
|
+
Once installed, the command-line script `psdi-data-convert-gui` will be made available, which can be called to start the server. You can then access the website by going to <http://127.0.0.1:5000> in a browser (this will also be printed in the terminal, and you can CTRL+click it there to open it in your default browser). Guidance for using the app is given on each page of it. When you're finished with the app, key CTRL+C in the terminal where you called the script to shut down the server, or, if the process was backgrounded, kill the appropriate process.
|
620
607
|
|
621
608
|
In case of problems when using Chrome, try opening Chrome from the command line:
|
622
609
|
open -a "Google Chrome.app" --args --allow-file-access-from-files
|
@@ -639,28 +626,80 @@ To test the CLA and Python library, install the optional testing requirements lo
|
|
639
626
|
|
640
627
|
```bash
|
641
628
|
pip install .'[test]'
|
642
|
-
pytest
|
629
|
+
pytest tests/python
|
643
630
|
```
|
644
631
|
|
645
632
|
To test the local version of the web app, install the GUI testing requirements locally (which also include the standard GUI requirements and standard testing requirements), start the server, and test by executing the GUI test script:
|
646
633
|
|
647
634
|
```bash
|
648
635
|
pip install .'[gui-test]'
|
649
|
-
|
650
|
-
|
651
|
-
|
652
|
-
|
636
|
+
pytest tests/gui
|
637
|
+
```
|
638
|
+
|
639
|
+
Both of these sets of tests can also be run together if desired through:
|
640
|
+
|
641
|
+
```bash
|
642
|
+
pip install .'[gui-test]'
|
643
|
+
pytest
|
644
|
+
```
|
645
|
+
|
646
|
+
## Troubleshooting
|
647
|
+
|
648
|
+
This section presents solutions for commonly-encountered issues.
|
649
|
+
|
650
|
+
### OSError: [Errno 24] Too many open files
|
651
|
+
|
652
|
+
You may see the error:
|
653
|
+
|
654
|
+
```
|
655
|
+
OSError: [Errno 24] Too many open files
|
656
|
+
```
|
657
|
+
|
658
|
+
while running the command-line application, using the Python library, or running tests This error is caused by a program hitting the limit of the number of open filehandles allowed by the OS. This limit is typically set to 1024 on Linux systems and 256 on MacOS systems, and thus this issue occurs much more often on the latter. You can see what your current limit is by running the command:
|
659
|
+
|
660
|
+
```bash
|
661
|
+
ulimit -a | grep "open files"
|
662
|
+
```
|
663
|
+
|
664
|
+
This limit can be temporarily increased for the current terminal session by running the command:
|
665
|
+
|
666
|
+
```bash
|
667
|
+
ulimit -n 1024 # Or another, higher number
|
668
|
+
```
|
669
|
+
|
670
|
+
First, try increasing the limit and then redo the operation which caused this error to see if this resolves it. If this does, you can make this change permanent in a few ways, the easiest of which is to add this command to your `.bashrc` file so it will be set for every new terminal session.
|
671
|
+
|
672
|
+
If you see this error when the filehandle limit is already set to a high value such as 1024, this may indicate the presence of a bug in the project which causes a leaking of filehandles, so please open an issue about it, pasting the error you get and the details of your system, particularly including your current filehandle limit.
|
673
|
+
|
674
|
+
### Errors running c2x or Atomsk converters
|
675
|
+
|
676
|
+
We provide support for the c2x and Atomsk converters by packing binaries which support common Linux and MacOS platforms with this project, but we cannot guarantee universal support for these binaries. In particular, they may rely on dynamically-linked libraries which aren't installed on your system.
|
677
|
+
|
678
|
+
Look through the error message you received for messages such as "Library not loaded" or "no such file", and see if they point to the name of a library which you can try installing. For instance, if you see that it's searching for `libquadmath.0.dylib` but not finding it, you can try installing this library. In this case, this library can be installed through apt with:
|
679
|
+
|
680
|
+
```bash
|
681
|
+
sudo apt install libquadmath0
|
653
682
|
```
|
654
683
|
|
655
|
-
|
684
|
+
or through brew via:
|
685
|
+
|
686
|
+
```bash
|
687
|
+
brew install gcc
|
688
|
+
```
|
689
|
+
|
690
|
+
Alternatively, you can run your own versions of the `c2x` and `atomsk` binaries with this project. Compile them yourself however you wish - see the projects at https://github.com/codenrx/c2x and https://github.com/pierrehirel/atomsk and follow their instructions to build a binary on your system. Once you've done so, add the binary to your `$PATH`, and this project will pick that up and use it in preference to the prepackaged binary.
|
691
|
+
|
692
|
+
On the other hand, it's possible that an error of this sort will occur if you have a non-working binary of one of these converters in your `$PATH`. If this might be the case, you can try removing it and see if the prepackaged binary works for you, or else recompile it to try to fix errors.
|
693
|
+
|
694
|
+
## Licensing
|
656
695
|
|
657
696
|
This project is provided under the Apache License version 2.0, the terms of which can be found in the file `LICENSE`.
|
658
697
|
|
659
|
-
This project redistributes compiled binaries for the Atomsk and c2x converters. These are both
|
698
|
+
This project redistributes compiled binaries for the Atomsk and c2x converters. These are both licensed under the
|
660
699
|
GNU General Public License version 3 and are redistributed per its terms. Any further redistribution of these binaries,
|
661
|
-
including redistribution of this project as a whole, including them, must also follow the terms of this
|
700
|
+
including redistribution of this project as a whole, including them, must also follow the terms of this license.
|
662
701
|
|
663
|
-
This requires conspicuously displaying notice of this
|
702
|
+
This requires conspicuously displaying notice of this license, providing the text of of the license (provided here in
|
664
703
|
the files `psdi_data_conversion/bin/LICENSE_C2X` and `psdi_data_conversion/bin/LICENSE_ATOMSK`), and appropriately
|
665
704
|
conveying the source code for each of these. Their respective source code may be found at:
|
666
705
|
|