emdbva 0.0.1.dev136__tar.gz → 0.0.1.dev139__tar.gz

{emdbva-0.0.1.dev136/emdbva.egg-info → emdbva-0.0.1.dev139}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: emdbva
-Version: 0.0.1.dev136
+Version: 0.0.1.dev139
 Summary: CryoEM validation toolkit
 Home-page: https://test.pypi.org/project/va/
 Author: Zhe Wang

{emdbva-0.0.1.dev136 → emdbva-0.0.1.dev139/emdbva.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: emdbva
-Version: 0.0.1.dev136
+Version: 0.0.1.dev139
 Summary: CryoEM validation toolkit
 Home-page: https://test.pypi.org/project/va/
 Author: Zhe Wang

{emdbva-0.0.1.dev136 → emdbva-0.0.1.dev139}/emdbva.egg-info/SOURCES.txt

@@ -21,6 +21,7 @@ va/metrics/contour_level_predicator.py
 va/metrics/emda_mmcc.py
 va/metrics/emringer.py
 va/metrics/inclusion.py
+va/metrics/map_data_validation.py
 va/metrics/overlap_percentage.py
 va/metrics/phaserandomization.py
 va/metrics/phenix_cc.py

{emdbva-0.0.1.dev136 → emdbva-0.0.1.dev139}/va/mainva.py

@@ -58,6 +58,7 @@ def allruns(validationobj, runs):
 
     # Projections
     if 'projection' in runs:
+        validationobj.map_data_validation()
         validationobj.new_projection()
         validationobj.relion_mask_evaluation()
 

emdbva-0.0.1.dev139/va/metrics/map_data_validation.py

@@ -0,0 +1,664 @@
+import io
+import os
+import re
+import warnings
+from collections import OrderedDict
+from pathlib import Path
+
+import mrcfile
+
+
+def _normalise_map_input(map_input):
+    if hasattr(map_input, "fullname"):
+        return map_input.fullname
+    if hasattr(map_input, "filename"):
+        return map_input.filename
+    return str(map_input)
+
+def _make_issue(code, category, severity, message, details=None, source="mrcfile.validate"):
+    """
+    Create one structured validation issue.
+
+    code:
+        Stable machine-readable problem name.
+
+    category:
+        Header/data/file area affected.
+
+    severity:
+        Usually "error" for validation failure, "warning" for weaker/unknown cases.
+
+    message:
+        Original mrcfile message, preserved for debugging.
+
+    details:
+        Parsed values extracted from the message.
+    """
+    return OrderedDict([
+        ("code", code),
+        ("category", category),
+        ("severity", severity),
+        ("source", source),
+        ("message", message),
+        ("details", details or OrderedDict()),
+    ])
+
+
+def _to_int(value):
+    try:
+        return int(value)
+    except Exception:
+        return value
+
+
+def _to_float(value):
+    try:
+        return float(value)
+    except Exception:
+        return value
+
+
+def _parse_validation_line(line, source="mrcfile.validate"):
+    """
+    Parse one line from mrcfile.validate() or one RuntimeWarning.
+
+    Any message not recognised by the parser is still kept in the JSON as
+    "unclassified_mrcfile_validation_message" so future mrcfile versions do
+    not silently lose information.
+    """
+    line = line.strip()
+
+    # 1. MRC map ID/header map field
+    match = re.match(
+        r"^Map ID string is incorrect: found (?P<found>.*), should be (?P<expected>.*)$",
+        line,
+    )
+    if match:
+        return _make_issue(
+            "map_id_incorrect",
+            "header.map",
+            "warning",
+            line,
+            OrderedDict([
+                ("found", match.group("found")),
+                ("expected", match.group("expected")),
+            ]),
+            source,
+        )
+
+    if line == "Map ID string not found - not an MRC file, or file is corrupt":
+        return _make_issue(
+            "map_id_missing_or_corrupt",
+            "header.map",
+            "warning",
+            line,
+            source=source,
+        )
+
+    # 2. Machine stamp
+    match = re.match(r"^Invalid machine stamp: (?P<machine_stamp>.*)$", line)
+    if match:
+        return _make_issue(
+            "machine_stamp_invalid",
+            "header.machst",
+            "error",
+            line,
+            OrderedDict([
+                ("machine_stamp", match.group("machine_stamp")),
+            ]),
+            source,
+        )
+
+    match = re.match(
+        r"^Machine stamp '(?P<machine_stamp>.*)' does not match the apparent byte order '(?P<byte_order>.*)'$",
+        line,
+    )
+    if match:
+        return _make_issue(
+            "machine_stamp_byte_order_mismatch",
+            "header.machst",
+            "warning",
+            line,
+            OrderedDict([
+                ("machine_stamp", match.group("machine_stamp")),
+                ("byte_order", match.group("byte_order")),
+            ]),
+            source,
+        )
+
+    # 3. Mode
+    match = re.match(r"^Invalid mode: (?P<mode>[-+]?\d+)$", line)
+    if match:
+        return _make_issue(
+            "mode_invalid",
+            "header.mode",
+            "error",
+            line,
+            OrderedDict([
+                ("mode", _to_int(match.group("mode"))),
+            ]),
+            source,
+        )
+
+    match = re.match(r"^Unrecognised mode '(?P<mode>.*)' - data block cannot be read$", line)
+    if match:
+        return _make_issue(
+            "mode_unrecognised_data_unreadable",
+            "header.mode",
+            "error",
+            line,
+            OrderedDict([
+                ("mode", match.group("mode")),
+            ]),
+            source,
+        )
+
+    # 4. Negative header fields
+    match = re.match(
+        r"^Header field '(?P<field>nx|ny|nz|mx|my|mz|ispg|nlabl)' is negative$",
+        line,
+    )
+    if match:
+        return _make_issue(
+            "header_field_negative",
+            "header.dimension_or_count",
+            "error",
+            line,
+            OrderedDict([
+                ("field", match.group("field")),
+            ]),
+            source,
+        )
+
+    match = re.match(r"^Cell dimension '(?P<axis>x|y|z)' is negative$", line)
+    if match:
+        return _make_issue(
+            "cell_dimension_negative",
+            "header.cella",
+            "error",
+            line,
+            OrderedDict([
+                ("axis", match.group("axis")),
+            ]),
+            source,
+        )
+
+    # 5. Axis mapping
+    match = re.match(
+        r"^Invalid axis mapping: found (?P<found>.*), should be \[1, 2, 3\]$",
+        line,
+    )
+    if match:
+        return _make_issue(
+            "axis_mapping_invalid",
+            "header.mapc_mapr_maps",
+            "error",
+            line,
+            OrderedDict([
+                ("found", match.group("found")),
+                ("expected", "[1, 2, 3]"),
+            ]),
+            source,
+        )
+
+    # 6. Volume stack
+    match = re.match(
+        r"^Error in dimensions for volume stack: nz should be divisible by mz\. "
+        r"Found nz = (?P<nz>[-+]?\d+), mz = (?P<mz>[-+]?\d+)\)?$",
+        line,
+    )
+    if match:
+        return _make_issue(
+            "volume_stack_dimensions_invalid",
+            "header.nz_mz_ispg",
+            "error",
+            line,
+            OrderedDict([
+                ("nz", _to_int(match.group("nz"))),
+                ("mz", _to_int(match.group("mz"))),
+            ]),
+            source,
+        )
+
+    # 7. Header labels
+    if line == "Error in header labels: empty labels appear between text-containing labels":
+        return _make_issue(
+            "header_labels_empty_between_text",
+            "header.label",
+            "error",
+            line,
+            source=source,
+        )
+
+    match = re.match(
+        r"^Error in header labels: nlabl is (?P<nlabl>[-+]?\d+) "
+        r"but (?P<label_count>[-+]?\d+) labels contain text$",
+        line,
+    )
+    if match:
+        return _make_issue(
+            "header_labels_nlabl_mismatch",
+            "header.nlabl",
+            "error",
+            line,
+            OrderedDict([
+                ("nlabl", _to_int(match.group("nlabl"))),
+                ("actual_label_count", _to_int(match.group("label_count"))),
+            ]),
+            source,
+        )
+
+    # 8. MRC format version
+    match = re.match(
+        r"^File does not declare MRC format version 20140 or 20141: nversion = (?P<nversion>[-+]?\d+)$",
+        line,
+    )
+    if match:
+        return _make_issue(
+            "mrc_format_version_invalid",
+            "header.nversion",
+            "warning",
+            line,
+            OrderedDict([
+                ("nversion", _to_int(match.group("nversion"))),
+                ("expected", [20140, 20141]),
+            ]),
+            source,
+        )
+
+    # 9. Extended header type
+    match = re.match(
+        r"^Extended header type is undefined or unrecognised: exttyp = '(?P<exttyp>.*)'$",
+        line,
+    )
+    if match:
+        return _make_issue(
+            "extended_header_type_invalid",
+            "header.exttyp",
+            "error",
+            line,
+            OrderedDict([
+                ("exttyp", match.group("exttyp")),
+                ("expected_known_types", ["CCP4", "MRCO", "SERI", "AGAR", "FEI1", "FEI2", "HDF5"]),
+            ]),
+            source,
+        )
+
+    # 10. Data statistics
+    match = re.match(
+        r"^Data statistics appear to be inaccurate: RMS deviation is "
+        r"(?P<actual>[-+0-9.eE]+) but the value in the header is (?P<header>[-+0-9.eE]+)$",
+        line,
+    )
+    if match:
+        return _make_issue(
+            "data_statistics_rms_mismatch",
+            "header.rms",
+            "error",
+            line,
+            OrderedDict([
+                ("actual", _to_float(match.group("actual"))),
+                ("header", _to_float(match.group("header"))),
+            ]),
+            source,
+        )
+
+    match = re.match(
+        r"^Data statistics appear to be inaccurate: minimum is "
+        r"(?P<actual>[-+0-9.eE]+) but the value in the header is (?P<header>[-+0-9.eE]+)$",
+        line,
+    )
+    if match:
+        return _make_issue(
+            "data_statistics_minimum_mismatch",
+            "header.dmin",
+            "error",
+            line,
+            OrderedDict([
+                ("actual", _to_float(match.group("actual"))),
+                ("header", _to_float(match.group("header"))),
+            ]),
+            source,
+        )
+
+    match = re.match(
+        r"^Data statistics appear to be inaccurate: maximum is "
+        r"(?P<actual>[-+0-9.eE]+) but the value in the header is (?P<header>[-+0-9.eE]+)$",
+        line,
+    )
+    if match:
+        return _make_issue(
+            "data_statistics_maximum_mismatch",
+            "header.dmax",
+            "error",
+            line,
+            OrderedDict([
+                ("actual", _to_float(match.group("actual"))),
+                ("header", _to_float(match.group("header"))),
+            ]),
+            source,
+        )
+
+    match = re.match(
+        r"^Data statistics appear to be inaccurate: mean is "
+        r"(?P<actual>[-+0-9.eE]+) but the value in the header is (?P<header>[-+0-9.eE]+)$",
+        line,
+    )
+    if match:
+        return _make_issue(
+            "data_statistics_mean_mismatch",
+            "header.dmean",
+            "error",
+            line,
+            OrderedDict([
+                ("actual", _to_float(match.group("actual"))),
+                ("header", _to_float(match.group("header"))),
+            ]),
+            source,
+        )
+
+    # 11. File size
+    match = re.match(
+        r"^File is larger than expected\. Actual size: (?P<actual_bytes>\d+) bytes; "
+        r"expected size: (?P<expected_bytes>\d+) bytes \(calculated from header\)$",
+        line,
+    )
+    if match:
+        return _make_issue(
+            "file_size_larger_than_expected",
+            "file.size",
+            "error",
+            line,
+            OrderedDict([
+                ("actual_bytes", _to_int(match.group("actual_bytes"))),
+                ("expected_bytes", _to_int(match.group("expected_bytes"))),
+            ]),
+            source,
+        )
+
+    if line == "Data block could not be read - file size not checked":
+        return _make_issue(
+            "data_block_unreadable_file_size_not_checked",
+            "data",
+            "error",
+            line,
+            source=source,
+        )
+
+    # 12. Extended header/data read warnings
+    match = re.match(
+        r"^Expected (?P<expected_bytes>\d+) bytes in extended header but could only read (?P<actual_bytes>\d+)$",
+        line,
+    )
+    if match:
+        return _make_issue(
+            "extended_header_too_small",
+            "extended_header.size",
+            "error",
+            line,
+            OrderedDict([
+                ("expected_bytes", _to_int(match.group("expected_bytes"))),
+                ("actual_bytes", _to_int(match.group("actual_bytes"))),
+            ]),
+            source,
+        )
+
+    match = re.match(
+        r"^Expected (?P<expected_bytes>\d+) bytes in data block but could only read (?P<actual_bytes>\d+)$",
+        line,
+    )
+    if match:
+        return _make_issue(
+            "data_block_too_small",
+            "data.size",
+            "error",
+            line,
+            OrderedDict([
+                ("expected_bytes", _to_int(match.group("expected_bytes"))),
+                ("actual_bytes", _to_int(match.group("actual_bytes"))),
+            ]),
+            source,
+        )
+
+    match = re.match(
+        r"^Expected (?P<expected_bytes>\d+) bytes in data block but limit is (?P<limit_bytes>\d+)$",
+        line,
+    )
+    if match:
+        return _make_issue(
+            "data_block_exceeds_read_limit",
+            "data.size",
+            "error",
+            line,
+            OrderedDict([
+                ("expected_bytes", _to_int(match.group("expected_bytes"))),
+                ("limit_bytes", _to_int(match.group("limit_bytes"))),
+            ]),
+            source,
+        )
+
+    if line == "Couldn't read enough bytes for MRC header":
+        return _make_issue(
+            "mrc_header_too_small",
+            "header.size",
+            "error",
+            line,
+            source=source,
+        )
+
+    # 13. Exception-like final traceback lines
+    match = re.match(r"^(?P<exception_type>[A-Za-z_][A-Za-z0-9_]*Error): (?P<exception_message>.*)$", line)
+    if match:
+        return _make_issue(
+            "exception_during_validation",
+            "exception",
+            "error",
+            line,
+            OrderedDict([
+                ("exception_type", match.group("exception_type")),
+                ("exception_message", match.group("exception_message")),
+            ]),
+            source,
+        )
+
+    # Fallback. Keep unknown messages instead of losing them.
+    return _make_issue(
+        "unclassified_mrcfile_validation_message",
+        "unclassified",
+        "warning",
+        line,
+        source=source,
+    )
+
+
+def _split_mrcfile_output(output):
+    """
+    Convert mrcfile.validate() text output into individual validation messages.
+
+    We remove non-problem lines like:
+      Checking if ... is a valid MRC2014 file...
+      File appears to be valid.
+    """
+    lines = []
+
+    for raw_line in output.splitlines():
+        line = raw_line.strip()
+
+        if not line:
+            continue
+
+        if line.startswith("Checking if ") and line.endswith(" is a valid MRC2014 file..."):
+            continue
+
+        if line == "File appears to be valid.":
+            continue
+
+        # If traceback is printed, keep only meaningful final error line.
+        if line == "Traceback (most recent call last):":
+            continue
+
+        if line.startswith("File ") and ", line " in line:
+            continue
+
+        if line.startswith("^"):
+            continue
+
+        lines.append(line)
+
+    return lines
+
+
+def _deduplicate_issues(issues):
+    """
+    Remove duplicates.
+
+    The same problem can sometimes appear both in mrcfile.validate() text output
+    and in captured RuntimeWarning messages.
+    """
+    unique_issues = []
+    seen = set()
+
+    for issue in issues:
+        key = (
+            issue.get("code"),
+            issue.get("category"),
+            issue.get("message"),
+        )
+
+        if key in seen:
+            continue
+
+        seen.add(key)
+        unique_issues.append(issue)
+
+    return unique_issues
+
+
+def _parse_messages_and_warnings(messages_text, warning_messages):
+    issues = []
+
+    for line in _split_mrcfile_output(messages_text):
+        issues.append(_parse_validation_line(line, source="mrcfile.validate"))
+
+    for warning_message in warning_messages:
+        warning_message = warning_message.strip()
+        if not warning_message:
+            continue
+
+        issues.append(_parse_validation_line(warning_message, source="RuntimeWarning"))
+
+    return _deduplicate_issues(issues)
+
+
+
+def validate_single_map(map_input):
+    map_path = Path(_normalise_map_input(map_input))
+    messages = io.StringIO()
+    warning_messages = []
+
+    result = OrderedDict([
+        ("file", str(map_path)),
+        ("exists", map_path.exists()),
+        ("valid", False),
+
+        # New structured fields
+        ("issue_count", 0),
+        ("issues", []),
+
+        # Keep old raw fields for backward compatibility/debugging
+        ("messages", ""),
+        ("warnings", []),
+        ("error", None),
+    ])
+
+    if not map_path.exists():
+        result["error"] = "File does not exist"
+        result["issues"] = [
+            _make_issue(
+                "file_not_found",
+                "file.path",
+                "error",
+                "File does not exist",
+                OrderedDict([
+                    ("path", str(map_path)),
+                ]),
+                source="validationanalysis",
+            )
+        ]
+        result["issue_count"] = len(result["issues"])
+        return result
+
+    try:
+        with warnings.catch_warnings(record=True) as caught_warnings:
+            warnings.simplefilter("always", RuntimeWarning)
+
+            mrcfile_valid = mrcfile.validate(str(map_path), print_file=messages)
+
+            warning_messages = [
+                str(warning.message)
+                for warning in caught_warnings
+                if str(warning.message).strip()
+            ]
+
+        messages_text = messages.getvalue().strip()
+        issues = _parse_messages_and_warnings(messages_text, warning_messages)
+        has_error = any(issue.get("severity") == "error" for issue in issues)
+
+        result["mrcfile_valid"] = mrcfile_valid
+        result["no_known_critical_error"] = not has_error
+        # result["messages"] = messages_text
+        # result["warnings"] = warning_messages
+        result["issues"] = issues
+        result["issue_count"] = len(issues)
+
+    except Exception as exc:
+        messages_text = messages.getvalue().strip()
+        exception_message = str(exc)
+
+        issues = _parse_messages_and_warnings(messages_text, warning_messages)
+        issues.append(
+            _make_issue(
+                "mrcfile_validate_exception",
+                "exception",
+                "error",
+                exception_message,
+                OrderedDict([
+                    ("exception_type", exc.__class__.__name__),
+                ]),
+                source="validationanalysis",
+            )
+        )
+
+        result["error"] = exception_message
+        result["messages"] = messages_text
+        result["warnings"] = warning_messages
+        result["issues"] = _deduplicate_issues(issues)
+        result["issue_count"] = len(result["issues"])
+
+    return result
+
+
+def run_map_data_validation(map_inputs):
+    map_results = OrderedDict()
+
+    for map_input in map_inputs:
+        if map_input is None:
+            continue
+
+        map_path = _normalise_map_input(map_input)
+        map_name = os.path.basename(map_path)
+
+        # Avoid overwriting when two inputs have the same basename.
+        # For example:
+        #   /path1/map.mrc
+        #   /path2/map.mrc
+        if map_name in map_results:
+            map_name = map_path
+
+        map_results[map_name] = validate_single_map(map_input)
+
+    return OrderedDict([
+        ("data_integrity", OrderedDict([
+            ("map_data_integrity", map_results)
+        ]))
+    ])

{emdbva-0.0.1.dev136 → emdbva-0.0.1.dev139}/va/validationanalysis.py

@@ -77,6 +77,7 @@ from va.metrics.qscore import *
 from va.metrics.inclusion import *
 from va.metrics.connected_percentage import *
 from va.metrics.overlap_percentage import *
+from metrics.map_data_validation import run_map_data_validation
 import va
 
 try:
@@ -1158,6 +1159,53 @@ class ValidationAnalysis:
             viewer.new_surface_view_chimerax(primary_input_map, primary_input_contour, 'mask', '',
                                            mask_name, mask_contour)
 
+    @profile_peak_memory()
+    def map_data_validation(self):
+        """
+        Validate map/header data using mrcfile.validate().
+
+        Output JSON structure:
+            data_validation -> map_data_validation -> mapname -> validation results
+        """
+
+        start = timeit.default_timer()
+        map_inputs = []
+
+        for attr in ("map", "rawmap", "hmodd", "hmeven"):
+            try:
+                value = getattr(self, attr, None)
+                if value is not None:
+                    map_inputs.append(value)
+            except AttributeError:
+                pass
+
+        try:
+            result_dict = run_map_data_validation(map_inputs)
+
+            out_json = os.path.join(
+                self.workdir,
+                f"{self.mapname}_map_data_validation.json"
+            )
+
+            with codecs.open(out_json, "w", encoding="utf-8") as f:
+                json.dump(
+                    result_dict,
+                    f,
+                    ensure_ascii=False,
+                    indent=2
+                )
+
+            print("Map data validation results were collected.")
+
+        except Exception:
+            err = "Map data validation error: {}.".format(sys.exc_info()[1])
+            sys.stderr.write(err + "\n")
+
+        end = timeit.default_timer()
+        print("Map data validation time: %s" % (end - start))
+        print("------------------------------------")
+
+        return None
 
     # Surface Chimera way
     # def new_surface_view_chimerax(self, input_map, input_contour, type='surface', raw='', mask_map=None,

{emdbva-0.0.1.dev136 → emdbva-0.0.1.dev139}/va/version.py

@@ -18,5 +18,5 @@ under the License.
 
 """
 
-__version__ = '0.0.1.dev136'
+__version__ = '0.0.1.dev139'
 __em_statistics_version__ = '202505.v01'