emdbva 0.0.1.dev143__tar.gz → 0.0.1.dev145__tar.gz

{emdbva-0.0.1.dev143 → emdbva-0.0.1.dev145}/MANIFEST.in

@@ -1,5 +1,6 @@
 include va/qscores.csv
 include va/utils/cl_weights.pth
+recursive-include va/schemas *.json
 exclude va/symmetry.py
 exclude va/onedepva.py
 exclude va/PATHS.py

{emdbva-0.0.1.dev143/emdbva.egg-info → emdbva-0.0.1.dev145}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: emdbva
-Version: 0.0.1.dev143
+Version: 0.0.1.dev145
 Summary: CryoEM validation toolkit
 Home-page: https://test.pypi.org/project/va/
 Author: Zhe Wang

{emdbva-0.0.1.dev143 → emdbva-0.0.1.dev145/emdbva.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: emdbva
-Version: 0.0.1.dev143
+Version: 0.0.1.dev145
 Summary: CryoEM validation toolkit
 Home-page: https://test.pypi.org/project/va/
 Author: Zhe Wang

{emdbva-0.0.1.dev143 → emdbva-0.0.1.dev145}/emdbva.egg-info/SOURCES.txt

@@ -16,6 +16,8 @@ va/prepareandrun_codon_airflow.py
 va/qscores.csv
 va/validationanalysis.py
 va/version.py
+va/audit/__init__.py
+va/audit/cif_revision.py
 va/metrics/__init__.py
 va/metrics/bars.py
 va/metrics/connected_percentage.py
@@ -36,6 +38,9 @@ va/metrics/smoc.py
 va/metrics/strudel.py
 va/metrics/surfaces.py
 va/metrics/threedfsc.py
+va/schemas/__init__.py
+va/schemas/emdb_entry_full.schema.json
+va/schemas/loader.py
 va/utils/Checker.py
 va/utils/ChimeraxViews.py
 va/utils/MapProcessor.py

{emdbva-0.0.1.dev143 → emdbva-0.0.1.dev145}/setup.py

@@ -19,6 +19,9 @@ setup(
     name="emdbva",
     version=__version__,
     packages=find_packages(),
+    package_data={
+        'va.schemas': ['*.json'],
+    },
     exclude_package_data={
         'va': ['test/*', 'core/*', 'proc/*', 'validation/*', 'test_data/*'],
     },

emdbva-0.0.1.dev145/va/audit/__init__.py

@@ -0,0 +1,13 @@
+"""Audit helpers used to make conservative VA execution decisions."""
+
+from va.audit.cif_revision import (
+    audit_model_cifs_for_preparation,
+    should_run_va_for_cif,
+    write_cif_revision_audit_report,
+)
+
+__all__ = [
+    'audit_model_cifs_for_preparation',
+    'should_run_va_for_cif',
+    'write_cif_revision_audit_report',
+]

emdbva-0.0.1.dev145/va/audit/cif_revision.py

@@ -0,0 +1,324 @@
+"""Gemmi-based audit revision checks for model CIF files."""
+
+import os
+import re
+import json
+
+import gemmi
+
+
+_CATEGORY = '_pdbx_audit_revision_details'
+_FIELDS = (
+    'ordinal',
+    'revision_ordinal',
+    'data_content_type',
+    'provider',
+    'type',
+    'description',
+    'details',
+)
+
+DEFAULT_SKIP_RULES = [
+    {
+        'type': 'Remediation',
+        'description': 'Metalloprotein remediation',
+        'decision': 'skip',
+        'reason_code': 'metalloprotein_remediation',
+    },
+    {
+        'type': 'Remediation',
+        'description': 'Carbohydrate remediation',
+        'decision': 'skip',
+        'reason_code': 'carbohydrate_remediation',
+    },
+]
+
+
+def _normalise_value(value):
+    """Normalize one Gemmi CIF value for display and comparison."""
+    if value is None:
+        return None
+    value = str(value).strip()
+    if len(value) >= 2 and value[0] == value[-1] and value[0] in "'\"":
+        value = value[1:-1].strip()
+    value = re.sub(r'\s+', ' ', value)
+    if value in ('?', '.') or not value:
+        return None
+    return value
+
+
+def _normalise_for_match(value):
+    """Return a normalized case-insensitive comparison value."""
+    value = _normalise_value(value)
+    return value.casefold() if value is not None else None
+
+
+def _first_block(document):
+    """Return the first data block across supported Gemmi versions."""
+    try:
+        return document[0]
+    except (TypeError, IndexError):
+        return document.sole_block()
+
+
+def _loop_from_block(block):
+    """Return the audit loop or ``None`` when the category is not looped."""
+    for field in _FIELDS:
+        try:
+            column = block.find_loop(f'{_CATEGORY}.{field}')
+        except Exception:
+            column = None
+        if column is not None:
+            return column.get_loop() if hasattr(column, 'get_loop') else column
+    return None
+
+
+def _rows_from_loop(loop):
+    """Convert a Gemmi loop into normalized row dictionaries."""
+    tags = [str(tag) for tag in getattr(loop, 'tags', [])]
+    values = list(getattr(loop, 'values', []))
+    if not tags:
+        return []
+
+    width = len(tags)
+    rows = []
+    for offset in range(0, len(values), width):
+        row = {}
+        for field in _FIELDS:
+            tag = f'{_CATEGORY}.{field}'
+            try:
+                index = tags.index(tag)
+            except ValueError:
+                row[field] = None
+                continue
+            value_index = offset + index
+            row[field] = _normalise_value(values[value_index]) if value_index < len(values) else None
+        rows.append(row)
+    return rows
+
+
+def _rows_from_scalars(block):
+    """Read a single audit row when the category is not represented as a loop."""
+    row = {}
+    found = False
+    for field in _FIELDS:
+        try:
+            value = block.find_value(f'{_CATEGORY}.{field}')
+        except Exception:
+            value = None
+        row[field] = _normalise_value(value)
+        found = found or row[field] is not None
+    return [row] if found else []
+
+
+def _issue(code, message, path):
+    """Build a non-fatal audit issue."""
+    return {
+        'code': code,
+        'severity': 'warning',
+        'path': path,
+        'message': message,
+    }
+
+
+def _rule_matches(row, rule):
+    """Return whether a rule exactly matches the relevant row fields."""
+    return (
+        _normalise_for_match(row.get('type')) == _normalise_for_match(rule.get('type'))
+        and _normalise_for_match(row.get('description')) == _normalise_for_match(rule.get('description'))
+    )
+
+
+def _base_decision(rows, issues):
+    """Return the common report structure for parsed audit rows."""
+    return {
+        'run_validation': True,
+        'decision': 'run',
+        'reason_code': 'unmatched_revision',
+        'message': 'No known VA-skip audit rule matched; VA will run.',
+        'matched_rules': [],
+        'issues': issues,
+        'audit_revision_details': {
+            'row_count': len(rows),
+            'rows': rows,
+        },
+    }
+
+
+def should_run_va_for_cif(cif_path, rules=None):
+    """Return a conservative VA execution decision for one model CIF.
+
+    Args:
+        cif_path (str): Path to a model CIF/mmCIF file.
+        rules (list or None): Optional replacement list of skip-rule mappings.
+
+    Returns:
+        dict: A stable run/skip decision with parsed audit rows and warnings.
+    """
+    if not cif_path or not os.path.isfile(cif_path):
+        return {
+            'run_validation': True,
+            'decision': 'run',
+            'reason_code': 'missing_cif',
+            'message': 'Model CIF file is missing; VA will run without audit gating.',
+            'matched_rules': [],
+            'issues': [_issue('missing_cif', f'CIF file does not exist: {cif_path}', str(cif_path))],
+            'audit_revision_details': {'row_count': 0, 'rows': []},
+        }
+
+    try:
+        document = gemmi.cif.read_file(cif_path)
+        block = _first_block(document)
+        loop = _loop_from_block(block)
+        rows = _rows_from_loop(loop) if loop is not None else _rows_from_scalars(block)
+    except Exception as exc:
+        return {
+            'run_validation': True,
+            'decision': 'run',
+            'reason_code': 'parse_error',
+            'message': 'CIF audit data could not be parsed; VA will run.',
+            'matched_rules': [],
+            'issues': [_issue('parse_error', str(exc), str(cif_path))],
+            'audit_revision_details': {'row_count': 0, 'rows': []},
+        }
+
+    if not rows:
+        return {
+            'run_validation': True,
+            'decision': 'run',
+            'reason_code': 'missing_audit_category',
+            'message': 'No audit revision details were found; VA will run.',
+            'matched_rules': [],
+            'issues': [_issue(
+                'missing_audit_category',
+                f'{_CATEGORY} is absent or has no rows.',
+                _CATEGORY,
+            )],
+            'audit_revision_details': {'row_count': 0, 'rows': []},
+        }
+
+    report = _base_decision(rows, [])
+    active_rules = DEFAULT_SKIP_RULES if rules is None else rules
+    matched_rules = []
+    matched_row_indexes = set()
+    for index, row in enumerate(rows):
+        for rule in active_rules:
+            if _rule_matches(row, rule):
+                matched_rules.append(dict(rule))
+                matched_row_indexes.add(index)
+
+    initial_type = _normalise_for_match('Initial release')
+    unrelated_rows = [
+        row for index, row in enumerate(rows)
+        if index not in matched_row_indexes
+        and _normalise_for_match(row.get('type')) not in (None, initial_type)
+    ]
+
+    if matched_rules and not unrelated_rows:
+        report.update({
+            'run_validation': False,
+            'decision': 'skip',
+            'reason_code': 'known_remediation_skip',
+            'message': 'A known remediation-only audit rule matched; VA validation will be skipped.',
+            'matched_rules': matched_rules,
+        })
+        return report
+
+    if len(rows) == 1 and _normalise_for_match(rows[0].get('type')) == initial_type:
+        report.update({
+            'reason_code': 'initial_release_only',
+            'message': 'Only an initial release audit revision was found; VA will run.',
+        })
+    return report
+
+
+def _model_cif_paths(prepobj):
+    """Resolve model CIF paths using the locations used by ``read_model``."""
+    model_names = getattr(prepobj, 'model', None) or []
+    if isinstance(model_names, str):
+        model_names = [model_names]
+
+    paths = []
+    for model_name in model_names:
+        if not model_name:
+            continue
+        if os.path.isabs(model_name):
+            paths.append(model_name)
+            continue
+        working_path = os.path.join(prepobj.vadir, model_name)
+        paths.append(working_path if os.path.isfile(working_path) else model_name)
+    return paths
+
+
+def _aggregate_model_reports(model_paths, model_reports):
+    """Aggregate per-model decisions using the conservative all-skip policy."""
+    all_skip = bool(model_reports) and all(
+        report.get('decision') == 'skip' for report in model_reports
+    )
+    issues = [issue for report in model_reports for issue in report.get('issues', [])]
+    matched_rules = [rule for report in model_reports for rule in report.get('matched_rules', [])]
+    rows = [
+        row for report in model_reports
+        for row in report.get('audit_revision_details', {}).get('rows', [])
+    ]
+    return {
+        'run_validation': not all_skip,
+        'decision': 'skip' if all_skip else 'run',
+        'reason_code': 'known_remediation_skip' if all_skip else 'model_audit_reviewed',
+        'message': (
+            'All model CIFs matched known remediation rules; VA validation will be skipped.'
+            if all_skip else 'At least one model CIF did not clearly match a skip rule; VA will run.'
+        ),
+        'matched_rules': matched_rules,
+        'issues': issues,
+        'audit_revision_details': {'row_count': len(rows), 'rows': rows},
+        'model_count': len(model_paths),
+        'model_reports': model_reports,
+    }
+
+
+def audit_model_cifs_for_preparation(prepobj):
+    """Audit all configured model CIFs for a prepared VA entry.
+
+    Args:
+        prepobj: Prepared VA input object with ``model`` and ``vadir`` attributes.
+
+    Returns:
+        dict: Aggregate audit decision and per-model reports. Missing or
+            malformed CIF data produces a run decision with warnings.
+    """
+    model_paths = _model_cif_paths(prepobj)
+    if not model_paths:
+        return {
+            'run_validation': True,
+            'decision': 'run',
+            'reason_code': 'no_model',
+            'message': 'No model was supplied; CIF audit gating was not applied.',
+            'matched_rules': [],
+            'issues': [],
+            'audit_revision_details': {'row_count': 0, 'rows': []},
+            'model_count': 0,
+            'model_reports': [],
+        }
+
+    model_reports = [should_run_va_for_cif(path) for path in model_paths]
+    return _aggregate_model_reports(model_paths, model_reports)
+
+
+def write_cif_revision_audit_report(prepobj, report):
+    """Write the audit fragment consumed by the existing JSON merger.
+
+    Args:
+        prepobj: Prepared VA input object with ``vadir`` and ``mapname``.
+        report (dict): Aggregate CIF audit decision.
+
+    Returns:
+        str: Path to the written audit report fragment.
+    """
+    report_path = os.path.join(
+        prepobj.vadir,
+        '{}_cif_revision_audit.json'.format(os.path.basename(prepobj.mapname)),
+    )
+    with open(report_path, 'w', encoding='utf-8') as output:
+        json.dump({'audit_validation': {'cif_revision_audit': report}}, output, indent=2)
+    return report_path

{emdbva-0.0.1.dev143 → emdbva-0.0.1.dev145}/va/mainva.py

@@ -40,6 +40,10 @@ from va.validationanalysis import ValidationAnalysis
 from va.preparation import PreParation as prep
 from va.metrics.strudel import run_strudel
 from va.utils.log_utils import LogRedirector
+from va.audit.cif_revision import (
+    audit_model_cifs_for_preparation,
+    write_cif_revision_audit_report,
+)
 import threading
 try:
     from PATHS import MAP_SERVER_PATH
@@ -335,6 +339,7 @@ def reconfigure_logging(temp_logger, buffer, log_file):
 
     temp_logger.info(f"Logging reconfigured to use file: {log_file}")
 
+
 # @profile
 def main():
     """Run the main VA command-line entry point.
@@ -348,6 +353,14 @@ def main():
     # Preparation
     # temp_logger, buffer = setup_temporary_logging()
     prepobj = prep()
+
+    if not prepobj.args.positions:
+        audit_report = audit_model_cifs_for_preparation(prepobj)
+        write_cif_revision_audit_report(prepobj, audit_report)
+        if not audit_report['run_validation']:
+            print(audit_report['message'])
+            return None
+
     # path_to_logs = f'{prepobj.vadir}/{prepobj.mapname}_va.log'
     # reconfigure_logging(temp_logger, buffer, path_to_logs)
     position = prepobj.args.positions

{emdbva-0.0.1.dev143 → emdbva-0.0.1.dev145}/va/preparation.py

@@ -53,6 +53,7 @@ from va.version import __version__
 from va.utils.misc import out_json, create_symbolic_link, keep_three_significant_digits
 from va.utils.MapProcessor import MapProcessor
 from memory_profiler import profile
+from va.schemas.loader import SchemaLoadError, load_json_schema, schema_candidate_paths, schema_file_path
 
 # JSON Schema (optional dependency; code will still run if missing)
 try:
@@ -314,22 +315,22 @@ class PreParation:
         copyfile(source, outpath)
         print(f'Wrote input example to {outpath}')
 
+    def _schema_file_paths(self):
+        """Return schema paths in package-first, source-tree-fallback order."""
+        return schema_candidate_paths()
+
     def _schema_file_path(self):
-        """Return the on-disk path to docs/emdb_entry_full.schema.json."""
-        # preparation.py lives under va/; the schema is stored in sibling docs/.
-        return os.path.join(os.path.dirname(os.path.dirname(__file__)), 'docs', 'emdb_entry_full.schema.json')
+        """Return the first available output-schema path."""
+        return schema_file_path()
 
     def _load_json_schema(self):
-        """Load a JSON Schema from docs/emdb_entry_full.schema.json. Returns dict or None."""
-        path = self._schema_file_path()
-        if not os.path.isfile(path):
-            sys.stderr.write(f'[validation] Schema file not found: {path}\n')
-            return None
+        """Load the packaged output schema, falling back to the source-tree copy."""
+        self._schema_load_error = None
         try:
-            with open(path, 'r', encoding='utf-8') as f:
-                return json.load(f)
-        except Exception as e:
-            sys.stderr.write(f'[validation] Could not load schema "{path}": {e}\n')
+            return load_json_schema()
+        except SchemaLoadError as exc:
+            self._schema_load_error = str(exc)
+            sys.stderr.write(self._schema_load_error + '\n')
             return None
 
     @staticmethod
@@ -373,7 +374,7 @@ class PreParation:
         """
         schema = self._load_json_schema()
         if schema is None:
-            return False, ['Schema file not found or unreadable.']
+            return False, [getattr(self, '_schema_load_error', 'Schema file not found or unreadable.')]
 
         Validator = self._choose_validator(schema)
         if Validator is None:

emdbva-0.0.1.dev145/va/schemas/__init__.py

@@ -0,0 +1 @@
+"""Packaged JSON schemas used by VA at runtime."""