emdbva 0.0.1.dev144__tar.gz → 0.0.1.dev146__tar.gz

{emdbva-0.0.1.dev144/emdbva.egg-info → emdbva-0.0.1.dev146}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: emdbva
-Version: 0.0.1.dev144
+Version: 0.0.1.dev146
 Summary: CryoEM validation toolkit
 Home-page: https://test.pypi.org/project/va/
 Author: Zhe Wang

{emdbva-0.0.1.dev144 → emdbva-0.0.1.dev146/emdbva.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: emdbva
-Version: 0.0.1.dev144
+Version: 0.0.1.dev146
 Summary: CryoEM validation toolkit
 Home-page: https://test.pypi.org/project/va/
 Author: Zhe Wang

{emdbva-0.0.1.dev144 → emdbva-0.0.1.dev146}/emdbva.egg-info/SOURCES.txt

@@ -16,6 +16,8 @@ va/prepareandrun_codon_airflow.py
 va/qscores.csv
 va/validationanalysis.py
 va/version.py
+va/audit/__init__.py
+va/audit/cif_revision.py
 va/metrics/__init__.py
 va/metrics/bars.py
 va/metrics/connected_percentage.py

emdbva-0.0.1.dev146/va/audit/__init__.py

@@ -0,0 +1,13 @@
+"""Audit helpers used to make conservative VA execution decisions."""
+
+from va.audit.cif_revision import (
+    audit_model_cifs_for_preparation,
+    should_run_va_for_cif,
+    write_cif_revision_audit_report,
+)
+
+__all__ = [
+    'audit_model_cifs_for_preparation',
+    'should_run_va_for_cif',
+    'write_cif_revision_audit_report',
+]

emdbva-0.0.1.dev146/va/audit/cif_revision.py

@@ -0,0 +1,324 @@
+"""Gemmi-based audit revision checks for model CIF files."""
+
+import os
+import re
+import json
+
+import gemmi
+
+
+_CATEGORY = '_pdbx_audit_revision_details'
+_FIELDS = (
+    'ordinal',
+    'revision_ordinal',
+    'data_content_type',
+    'provider',
+    'type',
+    'description',
+    'details',
+)
+
+DEFAULT_SKIP_RULES = [
+    {
+        'type': 'Remediation',
+        'description': 'Metalloprotein remediation',
+        'decision': 'skip',
+        'reason_code': 'metalloprotein_remediation',
+    },
+    {
+        'type': 'Remediation',
+        'description': 'Carbohydrate remediation',
+        'decision': 'skip',
+        'reason_code': 'carbohydrate_remediation',
+    },
+]
+
+
+def _normalise_value(value):
+    """Normalize one Gemmi CIF value for display and comparison."""
+    if value is None:
+        return None
+    value = str(value).strip()
+    if len(value) >= 2 and value[0] == value[-1] and value[0] in "'\"":
+        value = value[1:-1].strip()
+    value = re.sub(r'\s+', ' ', value)
+    if value in ('?', '.') or not value:
+        return None
+    return value
+
+
+def _normalise_for_match(value):
+    """Return a normalized case-insensitive comparison value."""
+    value = _normalise_value(value)
+    return value.casefold() if value is not None else None
+
+
+def _first_block(document):
+    """Return the first data block across supported Gemmi versions."""
+    try:
+        return document[0]
+    except (TypeError, IndexError):
+        return document.sole_block()
+
+
+def _loop_from_block(block):
+    """Return the audit loop or ``None`` when the category is not looped."""
+    for field in _FIELDS:
+        try:
+            column = block.find_loop(f'{_CATEGORY}.{field}')
+        except Exception:
+            column = None
+        if column is not None:
+            return column.get_loop() if hasattr(column, 'get_loop') else column
+    return None
+
+
+def _rows_from_loop(loop):
+    """Convert a Gemmi loop into normalized row dictionaries."""
+    tags = [str(tag) for tag in getattr(loop, 'tags', [])]
+    values = list(getattr(loop, 'values', []))
+    if not tags:
+        return []
+
+    width = len(tags)
+    rows = []
+    for offset in range(0, len(values), width):
+        row = {}
+        for field in _FIELDS:
+            tag = f'{_CATEGORY}.{field}'
+            try:
+                index = tags.index(tag)
+            except ValueError:
+                row[field] = None
+                continue
+            value_index = offset + index
+            row[field] = _normalise_value(values[value_index]) if value_index < len(values) else None
+        rows.append(row)
+    return rows
+
+
+def _rows_from_scalars(block):
+    """Read a single audit row when the category is not represented as a loop."""
+    row = {}
+    found = False
+    for field in _FIELDS:
+        try:
+            value = block.find_value(f'{_CATEGORY}.{field}')
+        except Exception:
+            value = None
+        row[field] = _normalise_value(value)
+        found = found or row[field] is not None
+    return [row] if found else []
+
+
+def _issue(code, message, path):
+    """Build a non-fatal audit issue."""
+    return {
+        'code': code,
+        'severity': 'warning',
+        'path': path,
+        'message': message,
+    }
+
+
+def _rule_matches(row, rule):
+    """Return whether a rule exactly matches the relevant row fields."""
+    return (
+        _normalise_for_match(row.get('type')) == _normalise_for_match(rule.get('type'))
+        and _normalise_for_match(row.get('description')) == _normalise_for_match(rule.get('description'))
+    )
+
+
+def _base_decision(rows, issues):
+    """Return the common report structure for parsed audit rows."""
+    return {
+        'run_validation': True,
+        'decision': 'run',
+        'reason_code': 'unmatched_revision',
+        'message': 'No known VA-skip audit rule matched; VA will run.',
+        'matched_rules': [],
+        'issues': issues,
+        'audit_revision_details': {
+            'row_count': len(rows),
+            'rows': rows,
+        },
+    }
+
+
+def should_run_va_for_cif(cif_path, rules=None):
+    """Return a conservative VA execution decision for one model CIF.
+
+    Args:
+        cif_path (str): Path to a model CIF/mmCIF file.
+        rules (list or None): Optional replacement list of skip-rule mappings.
+
+    Returns:
+        dict: A stable run/skip decision with parsed audit rows and warnings.
+    """
+    if not cif_path or not os.path.isfile(cif_path):
+        return {
+            'run_validation': True,
+            'decision': 'run',
+            'reason_code': 'missing_cif',
+            'message': 'Model CIF file is missing; VA will run without audit gating.',
+            'matched_rules': [],
+            'issues': [_issue('missing_cif', f'CIF file does not exist: {cif_path}', str(cif_path))],
+            'audit_revision_details': {'row_count': 0, 'rows': []},
+        }
+
+    try:
+        document = gemmi.cif.read_file(cif_path)
+        block = _first_block(document)
+        loop = _loop_from_block(block)
+        rows = _rows_from_loop(loop) if loop is not None else _rows_from_scalars(block)
+    except Exception as exc:
+        return {
+            'run_validation': True,
+            'decision': 'run',
+            'reason_code': 'parse_error',
+            'message': 'CIF audit data could not be parsed; VA will run.',
+            'matched_rules': [],
+            'issues': [_issue('parse_error', str(exc), str(cif_path))],
+            'audit_revision_details': {'row_count': 0, 'rows': []},
+        }
+
+    if not rows:
+        return {
+            'run_validation': True,
+            'decision': 'run',
+            'reason_code': 'missing_audit_category',
+            'message': 'No audit revision details were found; VA will run.',
+            'matched_rules': [],
+            'issues': [_issue(
+                'missing_audit_category',
+                f'{_CATEGORY} is absent or has no rows.',
+                _CATEGORY,
+            )],
+            'audit_revision_details': {'row_count': 0, 'rows': []},
+        }
+
+    report = _base_decision(rows, [])
+    active_rules = DEFAULT_SKIP_RULES if rules is None else rules
+    matched_rules = []
+    matched_row_indexes = set()
+    for index, row in enumerate(rows):
+        for rule in active_rules:
+            if _rule_matches(row, rule):
+                matched_rules.append(dict(rule))
+                matched_row_indexes.add(index)
+
+    initial_type = _normalise_for_match('Initial release')
+    unrelated_rows = [
+        row for index, row in enumerate(rows)
+        if index not in matched_row_indexes
+        and _normalise_for_match(row.get('type')) not in (None, initial_type)
+    ]
+
+    if matched_rules and not unrelated_rows:
+        report.update({
+            'run_validation': False,
+            'decision': 'skip',
+            'reason_code': 'known_remediation_skip',
+            'message': 'A known remediation-only audit rule matched; VA validation will be skipped.',
+            'matched_rules': matched_rules,
+        })
+        return report
+
+    if len(rows) == 1 and _normalise_for_match(rows[0].get('type')) == initial_type:
+        report.update({
+            'reason_code': 'initial_release_only',
+            'message': 'Only an initial release audit revision was found; VA will run.',
+        })
+    return report
+
+
+def _model_cif_paths(prepobj):
+    """Resolve model CIF paths using the locations used by ``read_model``."""
+    model_names = getattr(prepobj, 'model', None) or []
+    if isinstance(model_names, str):
+        model_names = [model_names]
+
+    paths = []
+    for model_name in model_names:
+        if not model_name:
+            continue
+        if os.path.isabs(model_name):
+            paths.append(model_name)
+            continue
+        working_path = os.path.join(prepobj.vadir, model_name)
+        paths.append(working_path if os.path.isfile(working_path) else model_name)
+    return paths
+
+
+def _aggregate_model_reports(model_paths, model_reports):
+    """Aggregate per-model decisions using the conservative all-skip policy."""
+    all_skip = bool(model_reports) and all(
+        report.get('decision') == 'skip' for report in model_reports
+    )
+    issues = [issue for report in model_reports for issue in report.get('issues', [])]
+    matched_rules = [rule for report in model_reports for rule in report.get('matched_rules', [])]
+    rows = [
+        row for report in model_reports
+        for row in report.get('audit_revision_details', {}).get('rows', [])
+    ]
+    return {
+        'run_validation': not all_skip,
+        'decision': 'skip' if all_skip else 'run',
+        'reason_code': 'known_remediation_skip' if all_skip else 'model_audit_reviewed',
+        'message': (
+            'All model CIFs matched known remediation rules; VA validation will be skipped.'
+            if all_skip else 'At least one model CIF did not clearly match a skip rule; VA will run.'
+        ),
+        'matched_rules': matched_rules,
+        'issues': issues,
+        'audit_revision_details': {'row_count': len(rows), 'rows': rows},
+        'model_count': len(model_paths),
+        'model_reports': model_reports,
+    }
+
+
+def audit_model_cifs_for_preparation(prepobj):
+    """Audit all configured model CIFs for a prepared VA entry.
+
+    Args:
+        prepobj: Prepared VA input object with ``model`` and ``vadir`` attributes.
+
+    Returns:
+        dict: Aggregate audit decision and per-model reports. Missing or
+            malformed CIF data produces a run decision with warnings.
+    """
+    model_paths = _model_cif_paths(prepobj)
+    if not model_paths:
+        return {
+            'run_validation': True,
+            'decision': 'run',
+            'reason_code': 'no_model',
+            'message': 'No model was supplied; CIF audit gating was not applied.',
+            'matched_rules': [],
+            'issues': [],
+            'audit_revision_details': {'row_count': 0, 'rows': []},
+            'model_count': 0,
+            'model_reports': [],
+        }
+
+    model_reports = [should_run_va_for_cif(path) for path in model_paths]
+    return _aggregate_model_reports(model_paths, model_reports)
+
+
+def write_cif_revision_audit_report(prepobj, report):
+    """Write the audit fragment consumed by the existing JSON merger.
+
+    Args:
+        prepobj: Prepared VA input object with ``vadir`` and ``mapname``.
+        report (dict): Aggregate CIF audit decision.
+
+    Returns:
+        str: Path to the written audit report fragment.
+    """
+    report_path = os.path.join(
+        prepobj.vadir,
+        '{}_cif_revision_audit.json'.format(os.path.basename(prepobj.mapname)),
+    )
+    with open(report_path, 'w', encoding='utf-8') as output:
+        json.dump({'audit_validation': {'cif_revision_audit': report}}, output, indent=2)
+    return report_path

{emdbva-0.0.1.dev144 → emdbva-0.0.1.dev146}/va/mainva.py

@@ -40,6 +40,10 @@ from va.validationanalysis import ValidationAnalysis
 from va.preparation import PreParation as prep
 from va.metrics.strudel import run_strudel
 from va.utils.log_utils import LogRedirector
+from va.audit.cif_revision import (
+    audit_model_cifs_for_preparation,
+    write_cif_revision_audit_report,
+)
 import threading
 try:
     from PATHS import MAP_SERVER_PATH
@@ -335,6 +339,7 @@ def reconfigure_logging(temp_logger, buffer, log_file):
 
     temp_logger.info(f"Logging reconfigured to use file: {log_file}")
 
+
 # @profile
 def main():
     """Run the main VA command-line entry point.
@@ -348,6 +353,14 @@ def main():
     # Preparation
     # temp_logger, buffer = setup_temporary_logging()
     prepobj = prep()
+
+    if not prepobj.args.positions:
+        audit_report = audit_model_cifs_for_preparation(prepobj)
+        write_cif_revision_audit_report(prepobj, audit_report)
+        if not audit_report['run_validation']:
+            print(audit_report['message'])
+            return None
+
     # path_to_logs = f'{prepobj.vadir}/{prepobj.mapname}_va.log'
     # reconfigure_logging(temp_logger, buffer, path_to_logs)
     position = prepobj.args.positions

{emdbva-0.0.1.dev144 → emdbva-0.0.1.dev146}/va/metrics/resmap.py

@@ -180,7 +180,7 @@ def resmap_filecheck(mapone, output_path):
     return check if check else False
 
 
-def resmap_chimerax(mapone, output_path):
+def relion_local_res_chimerax(orgmap, mapone, output_path):
     """
         Generate chimerax cmd for ResMap results
 
@@ -188,10 +188,10 @@ def resmap_chimerax(mapone, output_path):
         None. ChimeraX command files may be written to the output directory.
     """
 
-    mapname = os.path.basename(mapone)
     output_chimerax_file = '{}{}_chimerax.cxc'.format(output_path, os.path.basename(mapone))
-    orgmap = '{}{}_ori.map'.format(output_path, os.path.splitext(mapname)[0])
-    resmap = '{}{}_ori_resmap.map'.format(output_path, os.path.splitext(mapname)[0])
+    orgmap = orgmap
+    # resmap = '{}{}_ori_resmap.map'.format(output_path, os.path.splitext(mapname)[0])
+    resmap = mapone
     header = mrcfile.open(mapone, mode='r', header_only=True)
     voxsizes = header.voxel_size.tolist()
     if all(element == voxsizes[0] for element in voxsizes):
@@ -305,12 +305,12 @@ def run_resmap_chimerax(bindisplay, locCHIMERA, cxcfile):
         if not bindisplay:
             subprocess.check_call(locCHIMERA + " --offscreen --nogui " + cxcfile, cwd=os.path.dirname(cxcfile),
                                   shell=True)
-            print('Animated PNG for ResMap result has been produced.')
+            print('Animated PNG for local resolution result has been produced.')
         else:
             subprocess.check_call(locCHIMERA + " " + cxcfile, cwd=os.path.dirname(cxcfile), shell=True)
-            print('Animated PNG for ResMap result has been produced.')
+            print('Animated PNG for local resolution result has been produced.')
     except subprocess.CalledProcessError as suberr:
-        err = 'Saving ResMap local resolution animated png error: {}.'.format(suberr)
+        err = 'Saving local resolution animated png error: {}.'.format(suberr)
         errlist.append(err)
         sys.stderr.write(err + '\n')
 

{emdbva-0.0.1.dev144 → emdbva-0.0.1.dev146}/va/prepareandrun_codon_airflow.py

@@ -627,7 +627,14 @@ class prepareandrun():
             logging.info('Validation command completed.')
             self.data_sync()
         else:
-            logging.info('Validation command failed; skipping data sync.')
+            logging.error(
+                'Validation command failed with exit code %s; skipping data sync. '
+                'See stdout: %s and stderr: %s',
+                return_code,
+                stdout_path,
+                stderr_path,
+            )
+            raise subprocess.CalledProcessError(return_code, orgcmd)
 
         # predmem = self.memmsg(self.mapsize)
         # bsub -M predmem .....
@@ -665,7 +672,7 @@ class prepareandrun():
 
     def data_sync(self):
         """
-            sync data at corresponding folder to staging and production folders
+            sync data at corresponding folder to public production folders
 
         :return: None
         """
@@ -673,94 +680,82 @@ class prepareandrun():
         result_folder = '{}{}/va/'.format(MAP_SERVER_PATH, sub_dir)
         id = list(self.idsubdirs.keys())[0]
         prod_folder = '{}/{}/va'.format(VA_PROD_PATH, sub_dir)
+        fallback_prod_folder = prod_folder.replace('/nfs/public/', '/nfs/ebi/public/', 1)
         logging.info(prod_folder)
+        logging.info(fallback_prod_folder)
         alljsonfile = glob.glob('{}emd_{}_all.json'.format(result_folder, id))
-        if alljsonfile:
+        if not alljsonfile:
+            logging.info('No all json output, job failed to the end')
+            return None
+
+        rsync_filters = (
+            "--include '*/' "
+            "--include '*_rawmap.map' "
+            "--include '*_mask.mrc' "
+            "--exclude '*_locres.mrc' "
+            "--exclude '*.map' "
+            "--exclude '*.mrc' "
+            "--exclude '*.gz' "
+            "--exclude 'emd-*.cif' "
+            "--include '*' "
+        )
+
+        def sync_command(target_folder):
             quoted_result = shlex.quote(result_folder)
-            quoted_prod = shlex.quote(prod_folder)
-
-            rsync_filters = (
-                "--include '*/' "
-                "--include '*_rawmap.map' "
-                "--include '*_mask.mrc' "
-                "--exclude '*_locres.mrc' "
-                "--exclude '*.map' "
-                "--exclude '*.mrc' "
-                "--exclude '*.gz' "
-                "--exclude 'emd-*.cif' "
-                "--include '*' "
+            quoted_target = shlex.quote(target_folder.rstrip('/'))
+            return (
+                'mkdir -p {target} && '
+                'rsync -avzrhP {filters}{source} {target}/'
+            ).format(
+                target=quoted_target,
+                filters=rsync_filters,
+                source=quoted_result,
             )
 
-            sync_cmd = (
-                f"rsync -avzrhP {rsync_filters}"
-                f"{quoted_result} {quoted_prod}"
+        def sbatch_command(job_name, stdout_name, stderr_name, target_folder):
+            wrap_cmd = sync_command(target_folder)
+            return (
+                'sbatch --wait -p datamover --mem=10G --time 12:00:00 '
+                '-J {job_name} --mail-user=pdb_em@ebi.ac.uk '
+                '--mail-type=END --mail-type=FAIL '
+                '-o {stdout_path} -e {stderr_path} --wrap="{wrap_cmd}"'
+            ).format(
+                job_name=shlex.quote(job_name),
+                stdout_path=shlex.quote('{}{}'.format(result_folder, stdout_name)),
+                stderr_path=shlex.quote('{}{}'.format(result_folder, stderr_name)),
+                wrap_cmd=wrap_cmd,
             )
-            prod_stdout = f"{result_folder}{id}_prodsync_stdout.txt"
-            prod_stderr = f"{result_folder}{id}_prodsync_stderr.txt"
-
-            # for codon staging, production and dev
-            try:
-                subprocess.check_call(['ssh', 'wp-p1m2-e1',
-                                       'ls ' + prod_folder],
-                                      stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-                prod_rc = self.run_shell_command(sync_cmd, prod_stdout, prod_stderr)
-                if prod_rc == 0:
-                    logging.info('Target folder exists and data sync to production has completed')
-                else:
-                    logging.info('Target folder exists but data sync to production failed')
-
-                # sync data to HL
-                fallback_prod_folder = prod_folder
-                quoted_fallback_prod_folder = shlex.quote(fallback_prod_folder)
-                quoted_fallback_source = shlex.quote(fallback_prod_folder.rstrip('/') + '/')
-                quoted_fallback_dest = 'hl2-ws-login:' + shlex.quote(fallback_prod_folder.rstrip('/') + '/')
-
-                fallback_sync_cmd = (
-                    f"ssh hl2-ws-login 'mkdir -p {quoted_fallback_prod_folder}' && "
-                    f"rsync -avzrhP {rsync_filters}"
-                    f"{quoted_fallback_source} {quoted_fallback_dest}"
-                )
-                fallback_stdout = f"{result_folder}{id}_fallbacksync_stdout.txt"
-                fallback_stderr = f"{result_folder}{id}_fallbacksync_stderr.txt"
-                fallback_rc = self.run_shell_command(fallback_sync_cmd, fallback_stdout, fallback_stderr)
-                if fallback_rc == 0:
-                    logging.info("Fallback validation_analysis sync has completed")
-                else:
-                    logging.info("Fallback validation_analysis sync failed")
 
-            except subprocess.CalledProcessError as e:
-                try:
-                    subprocess.check_call(['ssh wp-p1m2-e1 "mkdir -p ' + prod_folder + '"'], stdout=subprocess.PIPE,
-                                          stderr=subprocess.PIPE, shell=True)
-                    prod_rc = self.run_shell_command(sync_cmd, prod_stdout, prod_stderr)
-                    if prod_rc == 0:
-                        logging.info('Target folder was created and data sync to production has completed')
-                    else:
-                        logging.info('Target folder was created but data sync to production failed')
-
-                    # sync data to HL
-                    fallback_prod_folder = prod_folder
-                    quoted_fallback_prod_folder = shlex.quote(fallback_prod_folder)
-                    quoted_fallback_source = shlex.quote(fallback_prod_folder.rstrip('/') + '/')
-                    quoted_fallback_dest = 'hl2-ws-login:' + shlex.quote(fallback_prod_folder.rstrip('/') + '/')
-
-                    fallback_sync_cmd = (
-                        f"ssh hl2-ws-login 'mkdir -p {quoted_fallback_prod_folder}' && "
-                        f"rsync -avzrhP {rsync_filters}"
-                        f"{quoted_fallback_source} {quoted_fallback_dest}"
-                    )
-                    fallback_stdout = f"{result_folder}{id}_fallbacksync_stdout.txt"
-                    fallback_stderr = f"{result_folder}{id}_fallbacksync_stderr.txt"
-                    fallback_rc = self.run_shell_command(fallback_sync_cmd, fallback_stdout, fallback_stderr)
-                    if fallback_rc == 0:
-                        logging.info("Fallback validation_analysis sync has completed")
-                    else:
-                        logging.info("Fallback validation_analysis sync failed")
-
-                except subprocess.CalledProcessError as e:
-                    logging.info('Target folder not properly produced.')
-        else:
-            logging.info('No all json output, job failed to the end')
+        sync_jobs = [
+            (
+                'production',
+                sbatch_command(
+                    'vasync_{}'.format(id),
+                    '{}_prodsync_stdout.txt'.format(id),
+                    '{}_prodsync_stderr.txt'.format(id),
+                    prod_folder,
+                ),
+            ),
+            (
+                'fallback',
+                sbatch_command(
+                    'vafallbacksync_{}'.format(id),
+                    '{}_fallbacksync_stdout.txt'.format(id),
+                    '{}_fallbacksync_stderr.txt'.format(id),
+                    fallback_prod_folder,
+                ),
+            ),
+        ]
+
+        for label, job_cmd in sync_jobs:
+            logging.info('Submitting %s sync job...', label)
+            logging.info(job_cmd)
+            subprocess.check_call(
+                ['ssh', 'codon-slurm-login', job_cmd],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+            logging.info('%s sync job has been submitted', label)
 
         return None
 
@@ -1069,7 +1064,6 @@ def main():
 
     if not myobj.onlycopy and not myobj.onlysync:
         myobj.runcmd()
-        myobj.data_sync()
     else:
         logging.info('Only copy files. Please make sure use --update t/Ture/1')