hitlist 2.1.2__tar.gz → 2.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hitlist-2.1.2/hitlist.egg-info → hitlist-2.1.4}/PKG-INFO +1 -1
- hitlist-2.1.4/hitlist/_source_scan.py +361 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/builder.py +59 -60
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/provenance.py +85 -8
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/scanner.py +46 -8
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/supplement.py +19 -1
- hitlist-2.1.4/hitlist/supplement_peptidoforms.py +111 -0
- hitlist-2.1.4/hitlist/version.py +1 -0
- {hitlist-2.1.2 → hitlist-2.1.4/hitlist.egg-info}/PKG-INFO +1 -1
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist.egg-info/SOURCES.txt +4 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/pyproject.toml +2 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_benchmark_network.py +99 -3
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_provenance.py +27 -1
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_scanner.py +48 -0
- hitlist-2.1.4/tests/test_source_scan.py +365 -0
- hitlist-2.1.4/tests/test_supplement_peptidoforms.py +150 -0
- hitlist-2.1.2/hitlist/version.py +0 -1
- {hitlist-2.1.2 → hitlist-2.1.4}/LICENSE +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/MANIFEST.in +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/README.md +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/__init__.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/aggregate.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/apm.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/assay_adjudication.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/assays.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/atlas.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/binding_assay_context.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/binding_context_availability.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/bulk_fractionation.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/bulk_proteomics.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/bulk_replicates.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/cache_inventory.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/canine_bulk.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/canine_ligands.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/canine_orfdata.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/canine_peptide_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/cell_name_parser.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/cli.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/cli_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/cli_evidence_store.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/cli_help.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/cli_uniprot.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/condition_categories.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/conditions.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/context_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/cta_expression.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/curation.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/curation_yaml.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/__init__.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/assay_adjudications.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/atlas_community_crosswalk.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/bulk_proteomics/__init__.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/bulk_proteomics/detectability.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/bulk_proteomics/sources.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/canine_bulk.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/canine_ligands.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/cell_lines.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/condition_vocabulary.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/data_assets.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/binding_assay_protocols.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/digestion_assets.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/digestion_contract.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/dilution_observer.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/goldberg_mapp.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/goldberg_mapp_assets.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/hap1_assets.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/hap1_contract.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/intact_degradomics.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/mapp_assets.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/mapp_contract.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/mapp_controlled_assets.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/mapp_controlled_contract.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/mapp_pbmc_assets.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/mapp_pbmc_contract.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/ms_calibration_assets.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/ms_calibration_contract.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/nanaware_lymph.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/native_peptidomics.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/nontryptic_calibration.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/pmid_overrides_1_78.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/precursor_context_assets.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/proteometools_ms1_core.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/pxd057588_assets.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/pxd057588_contract.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/pxd057588_retrieval_receipts.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/pxd057588_workbook_inventory.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/supporting_v4_assets.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/tmab_assets.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/evidence_revisions/wolf_levy_companion_crosswalk_audit.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/gene_sets.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/hla_ligand_atlas.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/line_expression/__init__.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/line_expression/apm_rna_build_report.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/line_expression/ensembl_gene_id_symbol.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/line_expression/gm12878_encode_rnaseq_tpm.csv.gz +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/line_expression/hap1_depmap24q4_gene_tpm.csv.gz +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/line_expression/sources.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/line_expression_anchors.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/monoallelic_lines.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/peptide_attributions/connelley_2022_bola_a19.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/peptide_attributions/sarkizova_2020_patient_cohort.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/pmid_overrides.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/pxd013455.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/pxd013455_publication.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/species_contexts.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/specimen_lineage.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/supplementary/abelin_2019_dr_tissue.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/supplementary/abelin_2019_maptac_class1.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/supplementary/gomez_zepeda_2024_hela.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/supplementary/gomez_zepeda_2024_jy.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/supplementary/gomez_zepeda_2024_plasma.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/supplementary/gomez_zepeda_2024_raji.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/supplementary/gomez_zepeda_2024_skmel37.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/supplementary/shapiro_2025_hap1_b2m_ko.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/supplementary/shapiro_2025_hap1_calr_ko.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/supplementary/shapiro_2025_hap1_canx_ko.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/supplementary/shapiro_2025_hap1_erap1_ko.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/supplementary/shapiro_2025_hap1_ganab_ko.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/supplementary/shapiro_2025_hap1_irf2_ko.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/supplementary/shapiro_2025_hap1_pdia3_ko.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/supplementary/shapiro_2025_hap1_sppl3_ko.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/supplementary/shapiro_2025_hap1_tap1_ko.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/supplementary/shapiro_2025_hap1_tap2_ko.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/supplementary/shapiro_2025_hap1_tapbp_ko.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/supplementary/shapiro_2025_hap1_wildtype.csv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/supplementary.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/tcell_audit_sources.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/tissue_blacklist.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/tissue_categories.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/data/uniprot_references.yaml +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/dataset.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/detectability.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/digestion_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/digestion_processing_package.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/digestion_recipe.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/dilution_observer.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/dilution_observer_compiler.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/dilution_observer_metadata.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/dilution_observer_schema.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/distribution_recipe.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/downloads.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/evidence_audit.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/evidence_bindings.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/evidence_bundle.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/evidence_catalog.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/evidence_package.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/evidence_recipe.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/evidence_snapshot.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/evidence_store.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/export.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/expression_inputs.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/external_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/external_successor.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/genes.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/goldberg_candidates.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/goldberg_composition.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/goldberg_mapp.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/goldberg_workbooks.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/gomez_arm_projection.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/gomez_prediction_panel.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/gomez_source_lineage.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/hap1_quantitative.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/hap1_response_recipe.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/indexer.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/intact_degradomics.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/intact_degradomics_schema.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/intact_degradomics_workbook.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/intact_kinetics.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/intact_organ_method.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/intact_organ_transfer.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/line_expression.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/lineage.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/lymph_composition.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/managed_assets.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/mapp_canonical.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/mapp_composition.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/mapp_controlled_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/mapp_controlled_recipe.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/mapp_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/mapp_pbmc_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/mappings.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/material_context.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/mhc_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/ms_calibration_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/ms_calibration_response.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/nanaware_lymph.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/native_contrast.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/native_contrast_compiler.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/native_contrast_schema.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/native_peptidomics.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/native_peptidomics_compiler.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/native_peptidomics_reference.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/native_peptidomics_schema.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/nontryptic_calibration.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/nontryptic_calibration_compiler.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/nontryptic_calibration_schema.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/nontryptic_workbooks.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/observations.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/pandas_utils.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/parquet_io.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/pbmc_composition.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/peptide_modifications.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/pmhc_query.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/precursor_context_package.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/precursor_recipe.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/predict.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/proteome.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/proteometools_ms1.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/proteometools_response.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/pxd013455.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/pxd013455_download.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/pxd013455_publication.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/pxd057588_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/qc.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/reference_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/report.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/sample_identity.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/samples.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/sequence_reference_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/species_contexts.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/species_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/species_expression.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/split_audit.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/supporting_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/supporting_numeric_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/tcell_assays.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/tcell_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/tcell_functional.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/tcell_multimer.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/tcell_peptides.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/tcell_processing_package.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/tcell_training.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/tissue_blacklist.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/tmab_peptidomics.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/tmab_response_recipe.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/training_bundle.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist/uniprot.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist.egg-info/dependency_links.txt +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist.egg-info/entry_points.txt +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist.egg-info/requires.txt +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/hitlist.egg-info/top_level.txt +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/scripts/verify_binding_assay_context.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/scripts/verify_canine_peptide_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/scripts/verify_digestion_arithmetic.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/scripts/verify_digestion_roundtrip.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/scripts/verify_digestion_timecourses.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/setup.cfg +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/setup.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/data/canine_orfdata/README.md +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/data/canine_orfdata/manifest.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/data/canine_orfdata/protein.tsv +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/data/canine_orfdata/search.fasta +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/data/canine_peptide_evidence/README.md +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/data/canine_peptide_evidence/dataset.sqlite.gz +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/data/canine_peptide_evidence/expected.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/data/canine_peptide_evidence/manifest.json +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_aggregate.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_allele_candidate_audit.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_allele_candidates.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_allele_identity.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_apm.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_apm_host_expression.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_apply_winners_vectorized.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_arm_resolution.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_assay_adjudication.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_assays.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_atlas_crosswalk.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_atlas_lineage.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_attribution_query_context.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_binding_assay_context.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_binding_context_availability.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_build_smoke.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_build_uniprot_reference.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_builder.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_bulk_fractionation.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_bulk_proteomics.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_bulk_proteomics_unit.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_bulk_replicates.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_cache_current.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_cache_inventory.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_canine_bulk.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_canine_ligands.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_canine_orfdata.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_canine_peptide_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_canine_peptide_runtime.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_categorical_consumers.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_cell_name_parser.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_check_dev_install.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_ci_corpus.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_cli_build_group.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_cli_data_dirs.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_cli_export_cleanup.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_cli_help.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_cli_mhc_allele_split.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_cli_pmhc_samples.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_cli_version.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_compare_corpus_generations.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_compare_reliability_outputs.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_condition_categories.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_conditions.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_context_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_corpus_qualification.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_corpus_v3_arm_contexts.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_cta_evidence_bundle.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_cta_expression.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_curated_erap2_lines.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_curated_study_sources.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_curation.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_curation_sanity_pass.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_curation_source_followup.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_curation_yaml.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_data_assets.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_data_dir_resolution.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_dataset_generations.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_deploy_script.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_depmap_expression_fetch.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_deposited_elution_conditions.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_detectability.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_digestion_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_digestion_processing_package.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_digestion_recipe.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_dilution_observer.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_download_http.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_download_to_file.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_downloads_timeout.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_ensembl_cache_only.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_evidence_catalog.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_evidence_package.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_evidence_recipes.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_evidence_snapshot.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_evidence_store.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_exclude_from_ms.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_export.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_export_filter_parity.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_export_join_lifetime.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_export_species_filters.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_expression_source_provenance.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_external_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_external_successor.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_genes.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_genotype_attribution.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_goldberg_composition.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_goldberg_mapp.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_gomez_arm_projection.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_gomez_prediction_panel.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_gomez_source_lineage.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_hap1_quantitative.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_hap1_response_recipe.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_human_cohort_qualification.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_intact_degradomics.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_intact_kinetics.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_intact_organ_method.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_intact_organ_transfer.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_line_expression.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_lineage_audit.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_local_supplement.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_long_peptide_mapping.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_lymph_composition.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_mapp_canonical.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_mapp_controlled_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_mapp_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_mapp_pbmc_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_mapping_worker_cache.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_mappings.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_material_context.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_mhc_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_mhc_typing_contract.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_mhcgnomes_floor_check.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_ms_calibration_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_ms_calibration_response.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_multi_proteome_flanking.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_mutant_sample_genotypes.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_nanaware_lymph.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_native_contrast.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_native_peptidomics.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_netmhcpan_invocation.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_nontryptic_calibration.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_nontryptic_workbooks.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_observations.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_pbmc_composition.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_peptide_attribution_scope.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_peptide_modifications.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_pmhc_query.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_precursor_context_package.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_predict.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_profile_build.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_proteome.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_proteome_cache_resolution.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_proteometools_ms1.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_public_api.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_pxd013455.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_qc.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_reassign_genotypes.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_release_artifacts.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_release_diagnostics.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_report.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_sample_attribution_audit.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_sample_group.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_sample_identity.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_sample_inventory.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_sample_override_and_note.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_samples.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_sequence_reference_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_source_verification.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_species_contexts.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_species_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_study_identity_corrections.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_supplement.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_supplementary_build_e2e.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_supporting_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_supporting_numeric_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_tcell_assays.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_tcell_availability.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_tcell_evidence.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_tcell_processing_package.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_test_script.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_tissue_blacklist.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_tmab_peptidomics.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_tmab_response_recipe.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_tmab_semantics.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_training_binding_filters.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_training_bundle.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_training_identity.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_training_preflight.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_training_qc_metadata.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_uniprot.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_unit_isolation.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_versioned_datasets.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_wolf_levy_crosswalk_audit.py +0 -0
- {hitlist-2.1.2 → hitlist-2.1.4}/tests/test_xdist_cache.py +0 -0
|
@@ -0,0 +1,361 @@
|
|
|
1
|
+
"""Bounded independent IEDB/CEDAR workers for the full-data builder."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import io
|
|
6
|
+
import json
|
|
7
|
+
import multiprocessing
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
import subprocess
|
|
11
|
+
import sys
|
|
12
|
+
import tempfile
|
|
13
|
+
import time
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
from .provenance import ContributorCollector, ProvenanceStorageError
|
|
17
|
+
|
|
18
|
+
_GIB = 1024**3
|
|
19
|
+
_RESULT_BYTES = 64 * 1024
|
|
20
|
+
_PARTITIONS = ("ms", "binding", "other", "empty")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _cgroup_directories(controller):
|
|
24
|
+
"""Resolve membership inside its mount and include every visible ancestor."""
|
|
25
|
+
memberships = {}
|
|
26
|
+
for line in Path("/proc/self/cgroup").read_text().splitlines():
|
|
27
|
+
_, controllers, location = line.split(":", 2)
|
|
28
|
+
for name in controllers.split(","):
|
|
29
|
+
memberships[name] = location
|
|
30
|
+
for line in Path("/proc/self/mountinfo").read_text().splitlines():
|
|
31
|
+
before, after = line.split(" - ", 1)
|
|
32
|
+
fields, filesystem = before.split(), after.split()
|
|
33
|
+
unified = filesystem[0] == "cgroup2"
|
|
34
|
+
if unified:
|
|
35
|
+
if controller in memberships:
|
|
36
|
+
continue # A hybrid host can keep this controller on v1.
|
|
37
|
+
location = memberships.get("")
|
|
38
|
+
elif filesystem[0] == "cgroup" and controller in filesystem[2].split(","):
|
|
39
|
+
location = memberships.get(controller)
|
|
40
|
+
else:
|
|
41
|
+
continue
|
|
42
|
+
if location is None:
|
|
43
|
+
continue
|
|
44
|
+
root, mount = (
|
|
45
|
+
Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value))
|
|
46
|
+
for value in fields[3:5]
|
|
47
|
+
)
|
|
48
|
+
member = Path(location)
|
|
49
|
+
if ".." in member.parts:
|
|
50
|
+
raise ValueError("Unresolved cgroup namespace")
|
|
51
|
+
current = mount / member.relative_to(root)
|
|
52
|
+
directories = []
|
|
53
|
+
while True:
|
|
54
|
+
directories.append(current)
|
|
55
|
+
if current == mount:
|
|
56
|
+
return unified, directories
|
|
57
|
+
current = current.parent
|
|
58
|
+
raise ValueError(f"No resolved {controller} cgroup mount")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _available_memory():
|
|
62
|
+
"""Conservative free capacity; unknown probes always select serial execution."""
|
|
63
|
+
try:
|
|
64
|
+
if sys.platform == "linux":
|
|
65
|
+
available = next(
|
|
66
|
+
int(line.split()[1]) * 1024
|
|
67
|
+
for line in Path("/proc/meminfo").read_text().splitlines()
|
|
68
|
+
if line.startswith("MemAvailable:")
|
|
69
|
+
)
|
|
70
|
+
unified, directories = _cgroup_directories("memory")
|
|
71
|
+
names = (
|
|
72
|
+
("memory.max", "memory.current")
|
|
73
|
+
if unified
|
|
74
|
+
else ("memory.limit_in_bytes", "memory.usage_in_bytes")
|
|
75
|
+
)
|
|
76
|
+
for directory in directories:
|
|
77
|
+
if not (directory / names[0]).exists():
|
|
78
|
+
continue # The controller can be disabled at an ancestor.
|
|
79
|
+
limit = (directory / names[0]).read_text().strip()
|
|
80
|
+
if limit != "max":
|
|
81
|
+
available = min(
|
|
82
|
+
available, max(0, int(limit) - int((directory / names[1]).read_text()))
|
|
83
|
+
)
|
|
84
|
+
return available
|
|
85
|
+
if sys.platform == "darwin":
|
|
86
|
+
probe = subprocess.run(["vm_stat"], capture_output=True, text=True, check=True)
|
|
87
|
+
pages = {}
|
|
88
|
+
for line in probe.stdout.splitlines():
|
|
89
|
+
name, separator, value = line.partition(":")
|
|
90
|
+
if separator and name in {"Pages free", "Pages speculative"}:
|
|
91
|
+
pages[name] = int(value.strip().rstrip("."))
|
|
92
|
+
return (pages["Pages free"] + pages.get("Pages speculative", 0)) * os.sysconf(
|
|
93
|
+
"SC_PAGE_SIZE"
|
|
94
|
+
)
|
|
95
|
+
except (OSError, ValueError, KeyError, IndexError, StopIteration, subprocess.SubprocessError):
|
|
96
|
+
return None
|
|
97
|
+
return None
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _available_cpus():
|
|
101
|
+
count = os.cpu_count() or 1
|
|
102
|
+
try:
|
|
103
|
+
if sys.platform == "linux":
|
|
104
|
+
count = min(count, len(os.sched_getaffinity(0)))
|
|
105
|
+
unified, directories = _cgroup_directories("cpu")
|
|
106
|
+
for directory in directories:
|
|
107
|
+
if unified:
|
|
108
|
+
if not (directory / "cpu.max").exists():
|
|
109
|
+
continue
|
|
110
|
+
quota, period = (directory / "cpu.max").read_text().split()
|
|
111
|
+
if quota != "max":
|
|
112
|
+
count = min(count, max(1, int(quota) // int(period)))
|
|
113
|
+
elif (directory / "cpu.cfs_quota_us").exists():
|
|
114
|
+
quota = int((directory / "cpu.cfs_quota_us").read_text())
|
|
115
|
+
period = int((directory / "cpu.cfs_period_us").read_text())
|
|
116
|
+
if quota > 0:
|
|
117
|
+
count = min(count, max(1, quota // period))
|
|
118
|
+
except (OSError, ValueError, KeyError, IndexError, ZeroDivisionError):
|
|
119
|
+
return 1
|
|
120
|
+
return count
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _admit_workers(sources):
|
|
124
|
+
requested = int(
|
|
125
|
+
os.environ.get("HITLIST_SCAN_WORKERS", os.environ.get("HITLIST_BUILD_WORKERS", "2"))
|
|
126
|
+
)
|
|
127
|
+
if requested < 1:
|
|
128
|
+
raise ValueError("HITLIST_SCAN_WORKERS must be a positive integer")
|
|
129
|
+
if requested == 1 or len(sources) != 2 or _available_cpus() < 2:
|
|
130
|
+
return False
|
|
131
|
+
available = _available_memory()
|
|
132
|
+
if available is None:
|
|
133
|
+
return False
|
|
134
|
+
try:
|
|
135
|
+
needed = 4 * _GIB + sum(max(28 * _GIB, 3 * path.stat().st_size) for _, path in sources)
|
|
136
|
+
except OSError:
|
|
137
|
+
return False
|
|
138
|
+
return available >= needed
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
class _QuotaSink(io.RawIOBase):
|
|
142
|
+
"""Enforce the worker's aggregate IPC reservation before every file write."""
|
|
143
|
+
|
|
144
|
+
def __init__(self, handle, remaining, collector):
|
|
145
|
+
super().__init__()
|
|
146
|
+
self.handle, self.remaining, self.collector = handle, remaining, collector
|
|
147
|
+
|
|
148
|
+
def writable(self):
|
|
149
|
+
return True
|
|
150
|
+
|
|
151
|
+
def tell(self):
|
|
152
|
+
return self.handle.tell()
|
|
153
|
+
|
|
154
|
+
def write(self, data):
|
|
155
|
+
if len(data) > self.remaining[0]:
|
|
156
|
+
raise self.collector._storage_error(
|
|
157
|
+
"source table buffers reached their scratch reservation"
|
|
158
|
+
)
|
|
159
|
+
self.collector._check_space(force=True)
|
|
160
|
+
written = self.handle.write(data)
|
|
161
|
+
self.remaining[0] -= written
|
|
162
|
+
return written
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _write_result(directory, result):
|
|
166
|
+
payload = (json.dumps(result) + "\n").encode()
|
|
167
|
+
if len(payload) > _RESULT_BYTES:
|
|
168
|
+
raise ProvenanceStorageError("Source worker result exceeds its scratch reservation")
|
|
169
|
+
(directory / "result.json").write_bytes(payload)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _worker(name, source, directory, quota, reserve, data_directory):
|
|
173
|
+
import pyarrow as pa
|
|
174
|
+
|
|
175
|
+
from . import scanner
|
|
176
|
+
from .builder import _scan_source_tables
|
|
177
|
+
from .downloads import set_data_dir
|
|
178
|
+
|
|
179
|
+
directory = Path(directory)
|
|
180
|
+
set_data_dir(data_directory)
|
|
181
|
+
scanner._tqdm = None
|
|
182
|
+
try:
|
|
183
|
+
with ContributorCollector(
|
|
184
|
+
scratch_dir=directory, max_scratch_bytes=quota, min_free_bytes=reserve
|
|
185
|
+
) as collector:
|
|
186
|
+
tables = list(_scan_source_tables(name, Path(source), collector))
|
|
187
|
+
tables[-1] = pa.Table.from_pandas(tables[-1], preserve_index=False)
|
|
188
|
+
remaining, written = [quota], []
|
|
189
|
+
for partition, table in zip(_PARTITIONS, tables):
|
|
190
|
+
if table is None:
|
|
191
|
+
continue
|
|
192
|
+
with (
|
|
193
|
+
(directory / f"{partition}.arrow").open("wb") as handle,
|
|
194
|
+
pa.PythonFile(_QuotaSink(handle, remaining, collector), mode="w") as sink,
|
|
195
|
+
pa.ipc.new_stream(
|
|
196
|
+
sink,
|
|
197
|
+
table.schema,
|
|
198
|
+
options=pa.ipc.IpcWriteOptions(compression="zstd", use_threads=False),
|
|
199
|
+
) as writer,
|
|
200
|
+
):
|
|
201
|
+
writer.write_table(table)
|
|
202
|
+
written.append(partition)
|
|
203
|
+
sources = collector._export_snapshot(directory / "graph.sqlite")
|
|
204
|
+
result = {"status": "success", "partitions": written, "sources": sources}
|
|
205
|
+
collector._check_space(force=True)
|
|
206
|
+
_write_result(directory, result)
|
|
207
|
+
except BaseException as error:
|
|
208
|
+
result = {
|
|
209
|
+
"status": "failed",
|
|
210
|
+
"error": "capacity" if isinstance(error, ProvenanceStorageError) else "worker",
|
|
211
|
+
"message": f"{type(error).__name__}: {error}"[:4000],
|
|
212
|
+
}
|
|
213
|
+
_write_result(directory, result)
|
|
214
|
+
raise
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _wait_workers(jobs):
|
|
218
|
+
pending = list(jobs)
|
|
219
|
+
while pending:
|
|
220
|
+
for process, directory in list(pending):
|
|
221
|
+
process.join(timeout=0)
|
|
222
|
+
if process.exitcode is None:
|
|
223
|
+
continue
|
|
224
|
+
if process.exitcode:
|
|
225
|
+
result_path = directory / "result.json"
|
|
226
|
+
result = json.loads(result_path.read_text()) if result_path.exists() else {}
|
|
227
|
+
message = result.get(
|
|
228
|
+
"message", f"Source worker exited with code {process.exitcode}"
|
|
229
|
+
)
|
|
230
|
+
if result.get("error") == "capacity":
|
|
231
|
+
raise ProvenanceStorageError(message)
|
|
232
|
+
raise RuntimeError(message)
|
|
233
|
+
pending.remove((process, directory))
|
|
234
|
+
if pending:
|
|
235
|
+
time.sleep(0.1)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _stop_workers(jobs):
|
|
239
|
+
for process, _ in jobs:
|
|
240
|
+
if process.is_alive():
|
|
241
|
+
process.terminate()
|
|
242
|
+
for process, _ in jobs:
|
|
243
|
+
process.join(timeout=5)
|
|
244
|
+
if process.is_alive():
|
|
245
|
+
process.kill()
|
|
246
|
+
process.join()
|
|
247
|
+
process.close()
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _parallel_sources(sources, provenance, quota, original_limit):
|
|
251
|
+
import pyarrow as pa
|
|
252
|
+
|
|
253
|
+
from .downloads import data_dir
|
|
254
|
+
|
|
255
|
+
results = []
|
|
256
|
+
with tempfile.TemporaryDirectory(
|
|
257
|
+
prefix="hitlist-scan-", dir=provenance.scratch_dir
|
|
258
|
+
) as temporary:
|
|
259
|
+
root = Path(temporary)
|
|
260
|
+
jobs = []
|
|
261
|
+
try:
|
|
262
|
+
for name, source in sources:
|
|
263
|
+
directory = root / name
|
|
264
|
+
directory.mkdir()
|
|
265
|
+
process = multiprocessing.get_context("spawn").Process(
|
|
266
|
+
target=_worker,
|
|
267
|
+
args=(
|
|
268
|
+
name,
|
|
269
|
+
str(source),
|
|
270
|
+
str(directory),
|
|
271
|
+
quota,
|
|
272
|
+
provenance.min_free_bytes,
|
|
273
|
+
str(data_dir()),
|
|
274
|
+
),
|
|
275
|
+
)
|
|
276
|
+
process.start()
|
|
277
|
+
jobs.append((process, directory))
|
|
278
|
+
try:
|
|
279
|
+
_wait_workers(jobs)
|
|
280
|
+
except ProvenanceStorageError:
|
|
281
|
+
# Nothing has been imported yet. Release all child reservations
|
|
282
|
+
# before a serial retry using the original collector budget.
|
|
283
|
+
return None
|
|
284
|
+
finally:
|
|
285
|
+
_stop_workers(jobs)
|
|
286
|
+
parent_limit = original_limit - 4 * quota - 2 * _RESULT_BYTES
|
|
287
|
+
for name, source in sources:
|
|
288
|
+
directory = root / name
|
|
289
|
+
result = json.loads((directory / "result.json").read_text())
|
|
290
|
+
if (
|
|
291
|
+
result.get("status") != "success"
|
|
292
|
+
or set(result.get("sources", {})) != {name}
|
|
293
|
+
or result["sources"][name].get("path") != str(source)
|
|
294
|
+
or not set(result.get("partitions", ())) <= set(_PARTITIONS)
|
|
295
|
+
or "empty" not in result.get("partitions", ())
|
|
296
|
+
or len(result["partitions"]) != len(set(result["partitions"]))
|
|
297
|
+
):
|
|
298
|
+
raise ValueError("Invalid source worker result")
|
|
299
|
+
expected_files = {"graph.sqlite", "result.json"} | {
|
|
300
|
+
f"{partition}.arrow" for partition in result["partitions"]
|
|
301
|
+
}
|
|
302
|
+
if {path.name for path in directory.iterdir()} != expected_files:
|
|
303
|
+
raise ValueError("Source worker file inventory differs from its result")
|
|
304
|
+
tables = []
|
|
305
|
+
for partition in _PARTITIONS:
|
|
306
|
+
path = directory / f"{partition}.arrow"
|
|
307
|
+
if partition not in result["partitions"]:
|
|
308
|
+
tables.append(None)
|
|
309
|
+
continue
|
|
310
|
+
# Owned buffers allow removal on every supported platform;
|
|
311
|
+
# memory-mapped tables would keep the reservation's files open.
|
|
312
|
+
with path.open("rb") as handle:
|
|
313
|
+
tables.append(pa.ipc.open_stream(handle).read_all())
|
|
314
|
+
path.unlink()
|
|
315
|
+
tables[-1] = tables[-1].to_pandas()
|
|
316
|
+
parent_limit += quota
|
|
317
|
+
provenance._set_scratch_limit(parent_limit)
|
|
318
|
+
graph = directory / "graph.sqlite"
|
|
319
|
+
provenance._import_snapshot(graph, result["sources"])
|
|
320
|
+
graph.unlink()
|
|
321
|
+
parent_limit += quota
|
|
322
|
+
provenance._set_scratch_limit(parent_limit)
|
|
323
|
+
results.append((name, tuple(tables)))
|
|
324
|
+
return results
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def scan_sources(paths, provenance):
|
|
328
|
+
"""Return deterministic source partitions with serial fallback before import."""
|
|
329
|
+
from .builder import _scan_source_tables
|
|
330
|
+
|
|
331
|
+
sources = [(name, Path(paths[name])) for name in ("iedb", "cedar") if name in paths]
|
|
332
|
+
original_limit = provenance.max_scratch_bytes
|
|
333
|
+
quota = (original_limit - 2 * _RESULT_BYTES) // (5 * 4096) * 4096
|
|
334
|
+
n_pages = provenance.db.execute("PRAGMA page_count").fetchone()[0]
|
|
335
|
+
if not provenance.sources and quota >= n_pages * 4096 and _admit_workers(sources):
|
|
336
|
+
provenance._set_scratch_limit(original_limit - 4 * quota - 2 * _RESULT_BYTES)
|
|
337
|
+
print("Scanning IEDB and CEDAR with two source workers.")
|
|
338
|
+
results = _parallel_sources(sources, provenance, quota, original_limit)
|
|
339
|
+
provenance._set_scratch_limit(original_limit)
|
|
340
|
+
if results is not None:
|
|
341
|
+
return results
|
|
342
|
+
print(
|
|
343
|
+
"Source worker scratch reservations were insufficient; retrying source scans serially."
|
|
344
|
+
)
|
|
345
|
+
results = []
|
|
346
|
+
for name, path in sources:
|
|
347
|
+
*tables, empty = _scan_source_tables(name, path, provenance)
|
|
348
|
+
results.append((name, (*tables, _owned_empty_schema(empty))))
|
|
349
|
+
return results
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def _owned_empty_schema(frame):
|
|
353
|
+
"""Preserve an empty schema without retaining its source's Arrow buffers."""
|
|
354
|
+
import pandas as pd
|
|
355
|
+
|
|
356
|
+
if len(frame):
|
|
357
|
+
raise ValueError("Expected an empty source schema")
|
|
358
|
+
empty = frame.copy()
|
|
359
|
+
for column, dtype in frame.dtypes.items():
|
|
360
|
+
empty[column] = pd.Series(index=empty.index, dtype=dtype)
|
|
361
|
+
return empty
|
|
@@ -545,8 +545,8 @@ def _drop_supplementary_duplicates(
|
|
|
545
545
|
supp: pd.DataFrame, obs: pd.DataFrame, *, provenance=None
|
|
546
546
|
) -> pd.DataFrame:
|
|
547
547
|
"""Anti-join supplementary rows against IEDB/CEDAR rows on
|
|
548
|
-
``(peptide, mhc_restriction, pmid)`` — IEDB/CEDAR wins when the
|
|
549
|
-
|
|
548
|
+
``(peptide, mhc_restriction, pmid, peptidoform)`` — IEDB/CEDAR wins when the
|
|
549
|
+
identity is shared (#45 supplements are intentionally redundant
|
|
550
550
|
when the paper is also in IEDB).
|
|
551
551
|
|
|
552
552
|
Replaces an earlier ``set(zip(...))`` Python set that held ~4 M
|
|
@@ -575,7 +575,12 @@ def _drop_supplementary_duplicates(
|
|
|
575
575
|
"""
|
|
576
576
|
if supp.empty:
|
|
577
577
|
return supp
|
|
578
|
-
|
|
578
|
+
from .supplement_peptidoforms import peptidoform_keys
|
|
579
|
+
|
|
580
|
+
supp = supp.assign(_peptidoform_key=peptidoform_keys(supp))
|
|
581
|
+
if not obs.empty:
|
|
582
|
+
obs = obs.assign(_peptidoform_key=peptidoform_keys(obs))
|
|
583
|
+
join_cols = ["peptide", "mhc_restriction", "pmid", "_peptidoform_key"]
|
|
579
584
|
if obs.empty:
|
|
580
585
|
obs_keys = pd.DataFrame({c: pd.Series(dtype="object") for c in join_cols})
|
|
581
586
|
else:
|
|
@@ -604,7 +609,7 @@ def _drop_supplementary_duplicates(
|
|
|
604
609
|
return (
|
|
605
610
|
supp.merge(obs_keys, on=join_cols, how="left", indicator=True)
|
|
606
611
|
.query("_merge == 'left_only'")
|
|
607
|
-
.drop(columns="_merge")
|
|
612
|
+
.drop(columns=["_merge", "_peptidoform_key"])
|
|
608
613
|
.reset_index(drop=True)
|
|
609
614
|
)
|
|
610
615
|
|
|
@@ -870,6 +875,46 @@ def build_observations(
|
|
|
870
875
|
return published["observations.parquet"]
|
|
871
876
|
|
|
872
877
|
|
|
878
|
+
def _scan_source_tables(name, path, provenance):
|
|
879
|
+
"""Apply the same source scan and partition normalization in either process."""
|
|
880
|
+
import pyarrow as pa
|
|
881
|
+
|
|
882
|
+
from .assays import ANNOTATION_COLUMNS, annotate_assays
|
|
883
|
+
from .scanner import scan
|
|
884
|
+
|
|
885
|
+
print(f"Scanning {name} ({path.name})...")
|
|
886
|
+
df = scan(
|
|
887
|
+
peptides=None,
|
|
888
|
+
iedb_path=path if name == "iedb" else None,
|
|
889
|
+
cedar_path=path if name == "cedar" else None,
|
|
890
|
+
mhc_species=None,
|
|
891
|
+
classify_source=True,
|
|
892
|
+
provenance=provenance,
|
|
893
|
+
)
|
|
894
|
+
df["source"] = name
|
|
895
|
+
if not set(ANNOTATION_COLUMNS) <= set(df):
|
|
896
|
+
annotate_assays(df)
|
|
897
|
+
empty_schema = df.iloc[:0].copy()
|
|
898
|
+
frames = (
|
|
899
|
+
df[df["is_ms_observation"]].copy(),
|
|
900
|
+
df[df["is_binding_assay"]].copy(),
|
|
901
|
+
df[~(df["is_ms_observation"] | df["is_binding_assay"])].copy(),
|
|
902
|
+
)
|
|
903
|
+
del df
|
|
904
|
+
n_ms, n_binding, n_other = map(len, frames)
|
|
905
|
+
tables = []
|
|
906
|
+
for frame in frames:
|
|
907
|
+
# Preserve the existing nullable PMID conversion and per-source category
|
|
908
|
+
# order before Arrow transport; mixed raw PMIDs remain in provenance.
|
|
909
|
+
if "pmid" in frame:
|
|
910
|
+
frame["pmid"] = pd.to_numeric(frame["pmid"], errors="coerce").astype("Int64")
|
|
911
|
+
_compress_categoricals(frame)
|
|
912
|
+
for frame in frames:
|
|
913
|
+
tables.append(pa.Table.from_pandas(frame, preserve_index=False) if len(frame) else None)
|
|
914
|
+
print(f" {n_ms:,} MS + {n_binding:,} binding + {n_other:,} other assay rows from {name}")
|
|
915
|
+
return (*tables, empty_schema)
|
|
916
|
+
|
|
917
|
+
|
|
873
918
|
def _build_observations(
|
|
874
919
|
paths,
|
|
875
920
|
out_path,
|
|
@@ -886,7 +931,6 @@ def _build_observations(
|
|
|
886
931
|
|
|
887
932
|
from .assays import ANNOTATION_COLUMNS, annotate_assays
|
|
888
933
|
from .provenance import contributors_path, file_digest
|
|
889
|
-
from .scanner import scan
|
|
890
934
|
|
|
891
935
|
# Buffer per-source partitions as pyarrow Tables, not pandas frames.
|
|
892
936
|
# Two reasons:
|
|
@@ -908,61 +952,16 @@ def _build_observations(
|
|
|
908
952
|
other_tables: list[pa.Table] = []
|
|
909
953
|
empty_schema = pd.DataFrame()
|
|
910
954
|
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
provenance=provenance,
|
|
922
|
-
)
|
|
923
|
-
df["source"] = name
|
|
924
|
-
|
|
925
|
-
# A false binding flag never establishes MS modality. Retain all
|
|
926
|
-
# remaining assays independently, including explicit negative MS.
|
|
927
|
-
if not set(ANNOTATION_COLUMNS) <= set(df):
|
|
928
|
-
annotate_assays(df)
|
|
929
|
-
empty_schema = df.iloc[:0].copy()
|
|
930
|
-
ms_df = df[df["is_ms_observation"]].copy()
|
|
931
|
-
bd_df = df[df["is_binding_assay"]].copy()
|
|
932
|
-
other_df = df[~(df["is_ms_observation"] | df["is_binding_assay"])].copy()
|
|
933
|
-
del df # free the source scan's frame before the next source's scan
|
|
934
|
-
n_ms, n_bd, n_other = len(ms_df), len(bd_df), len(other_df)
|
|
935
|
-
|
|
936
|
-
# Normalize ``pmid`` to ``Int64`` *per-partition* before the
|
|
937
|
-
# Arrow conversion. The scanner emits ``pmid`` as object dtype
|
|
938
|
-
# with ``""`` for missing rows and integer-like strings for
|
|
939
|
-
# present ones; pyarrow's ``Table.from_pandas`` infers a single
|
|
940
|
-
# type per column and chokes on the mixed shapes (#232). The
|
|
941
|
-
# full-frame normalization at the bottom of this function is
|
|
942
|
-
# kept as defensive scaffolding but is now a no-op.
|
|
943
|
-
for frame in (ms_df, bd_df, other_df):
|
|
944
|
-
if "pmid" in frame.columns:
|
|
945
|
-
frame["pmid"] = pd.to_numeric(frame["pmid"], errors="coerce").astype("Int64")
|
|
946
|
-
|
|
947
|
-
# Compress per-partition before Arrow conversion so dictionary-encoded
|
|
948
|
-
# categories survive the round-trip (Arrow ``DictionaryArray`` ↔
|
|
949
|
-
# pandas ``category`` is preserved by ``pa.Table.from_pandas`` /
|
|
950
|
-
# ``Table.to_pandas``).
|
|
951
|
-
_compress_categoricals(ms_df)
|
|
952
|
-
_compress_categoricals(bd_df)
|
|
953
|
-
_compress_categoricals(other_df)
|
|
954
|
-
|
|
955
|
-
# Convert to Arrow eagerly and free the pandas frame. Holding the
|
|
956
|
-
# accumulated partitions as Arrow keeps the per-source memory cost
|
|
957
|
-
# ~5x lower than pandas.
|
|
958
|
-
if n_ms:
|
|
959
|
-
ms_tables.append(pa.Table.from_pandas(ms_df, preserve_index=False))
|
|
960
|
-
if n_bd:
|
|
961
|
-
binding_tables.append(pa.Table.from_pandas(bd_df, preserve_index=False))
|
|
962
|
-
if n_other:
|
|
963
|
-
other_tables.append(pa.Table.from_pandas(other_df, preserve_index=False))
|
|
964
|
-
del ms_df, bd_df, other_df
|
|
965
|
-
print(f" {n_ms:,} MS + {n_bd:,} binding + {n_other:,} other assay rows from {name}")
|
|
955
|
+
from ._source_scan import scan_sources
|
|
956
|
+
|
|
957
|
+
for _, (ms_table, binding_table, other_table, source_schema) in scan_sources(paths, provenance):
|
|
958
|
+
empty_schema = source_schema
|
|
959
|
+
if ms_table is not None:
|
|
960
|
+
ms_tables.append(ms_table)
|
|
961
|
+
if binding_table is not None:
|
|
962
|
+
binding_tables.append(binding_table)
|
|
963
|
+
if other_table is not None:
|
|
964
|
+
other_tables.append(other_table)
|
|
966
965
|
|
|
967
966
|
if not ms_tables and not binding_tables and not other_tables:
|
|
968
967
|
raise RuntimeError("No data scanned.")
|
|
@@ -7,6 +7,7 @@ does not require another corpus-sized Python collection.
|
|
|
7
7
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
|
+
import contextlib
|
|
10
11
|
import errno
|
|
11
12
|
import json
|
|
12
13
|
import math
|
|
@@ -230,7 +231,7 @@ class ContributorCollector:
|
|
|
230
231
|
CREATE TABLE payload_blocks (id INTEGER PRIMARY KEY, payload BLOB, raw_size INTEGER);
|
|
231
232
|
CREATE TABLE records (id TEXT PRIMARY KEY, dataset TEXT, row_number INTEGER,
|
|
232
233
|
block_id INTEGER, byte_offset INTEGER, byte_length INTEGER,
|
|
233
|
-
pmid TEXT);
|
|
234
|
+
pmid TEXT) WITHOUT ROWID;
|
|
234
235
|
CREATE TABLE edges (source TEXT, target TEXT, relation INTEGER, label TEXT);
|
|
235
236
|
CREATE INDEX edge_target ON edges(target);
|
|
236
237
|
CREATE TABLE retained (id TEXT PRIMARY KEY) WITHOUT ROWID;
|
|
@@ -282,22 +283,39 @@ class ContributorCollector:
|
|
|
282
283
|
CSV readers return only strings. The general record method still owns
|
|
283
284
|
nested/typed supplementary payloads. Reusing JSON's string encoder here
|
|
284
285
|
preserves its exact escaping, spaces, key order and unmapped cells.
|
|
285
|
-
|
|
286
|
+
Only the preceding bounded row is retained to reuse unchanged cells.
|
|
286
287
|
"""
|
|
287
288
|
prefixes = tuple(
|
|
288
289
|
(json.dumps(name) + ": ", index) for name, index in sorted(columns.items())
|
|
289
290
|
)
|
|
290
291
|
pmid_index = columns.get("pmid")
|
|
291
292
|
escape = json.encoder.encode_basestring_ascii
|
|
293
|
+
previous = ()
|
|
294
|
+
encoded = []
|
|
295
|
+
mapped = []
|
|
292
296
|
|
|
293
297
|
def record_row(row_number, row_values):
|
|
294
|
-
|
|
298
|
+
nonlocal previous, encoded, mapped
|
|
299
|
+
if len(previous) == len(row_values) and mapped:
|
|
300
|
+
for index, (value, old) in enumerate(zip(row_values, previous)):
|
|
301
|
+
if value != old:
|
|
302
|
+
encoded[index] = escape(value)
|
|
303
|
+
for position, (prefix, index) in enumerate(prefixes):
|
|
304
|
+
if index < len(row_values) and row_values[index] != previous[index]:
|
|
305
|
+
mapped[position] = prefix + encoded[index]
|
|
306
|
+
else:
|
|
307
|
+
encoded = list(map(escape, row_values))
|
|
308
|
+
mapped = [
|
|
309
|
+
prefix + (encoded[index] if index < len(encoded) else '""')
|
|
310
|
+
for prefix, index in prefixes
|
|
311
|
+
]
|
|
295
312
|
n_columns = len(encoded)
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
313
|
+
payload = ("{" + ", ".join(mapped) + "}\n[" + ", ".join(encoded) + "]").encode()
|
|
314
|
+
if len(payload) <= self._CAPTURE_BATCH_BYTES:
|
|
315
|
+
# A caller may reuse and mutate its CSV row list.
|
|
316
|
+
previous = tuple(row_values)
|
|
317
|
+
else:
|
|
318
|
+
previous, encoded, mapped = (), [], []
|
|
301
319
|
pmid = (
|
|
302
320
|
row_values[pmid_index] if pmid_index is not None and pmid_index < n_columns else ""
|
|
303
321
|
)
|
|
@@ -368,6 +386,65 @@ class ContributorCollector:
|
|
|
368
386
|
raise ValueError("Contributor payload slice exceeds its block")
|
|
369
387
|
return block[offset : offset + length]
|
|
370
388
|
|
|
389
|
+
def _set_scratch_limit(self, n_bytes):
|
|
390
|
+
"""Apply a temporary reservation within the caller's total scratch budget."""
|
|
391
|
+
n_pages = n_bytes // 4096
|
|
392
|
+
if self.db.execute("PRAGMA page_count").fetchone()[0] > n_pages:
|
|
393
|
+
raise self._storage_error("existing graph exceeds its reserved scratch space")
|
|
394
|
+
actual = self.db.execute(f"PRAGMA max_page_count={n_pages}").fetchone()[0]
|
|
395
|
+
if actual != n_pages:
|
|
396
|
+
raise self._storage_error("could not apply the scratch reservation")
|
|
397
|
+
self.max_scratch_bytes = n_pages * 4096
|
|
398
|
+
|
|
399
|
+
def _export_snapshot(self, path):
|
|
400
|
+
"""Move a closed, disposable worker graph; this is not a public artifact."""
|
|
401
|
+
path = Path(path)
|
|
402
|
+
if path.exists():
|
|
403
|
+
raise ValueError("Contributor snapshot destination already exists")
|
|
404
|
+
self._flush_payloads()
|
|
405
|
+
self.db.commit()
|
|
406
|
+
self.db.close()
|
|
407
|
+
self.db = None
|
|
408
|
+
(Path(self._temporary.name) / "records.sqlite").replace(path)
|
|
409
|
+
return self.sources
|
|
410
|
+
|
|
411
|
+
def _import_snapshot(self, path, sources):
|
|
412
|
+
"""Merge a private source worker's graph without changing public identities."""
|
|
413
|
+
if self.sources.keys() & sources.keys():
|
|
414
|
+
raise ValueError("Contributor snapshot repeats a source namespace")
|
|
415
|
+
self._flush_payloads()
|
|
416
|
+
self.db.commit()
|
|
417
|
+
self._execute("ATTACH DATABASE ? AS captured", (str(path),))
|
|
418
|
+
try:
|
|
419
|
+
next_block = self._execute(
|
|
420
|
+
"SELECT COALESCE(MAX(id) + 1, 0) FROM captured.payload_blocks"
|
|
421
|
+
).fetchone()[0]
|
|
422
|
+
self._execute(
|
|
423
|
+
"INSERT INTO payload_blocks SELECT id + ?, payload, raw_size "
|
|
424
|
+
"FROM captured.payload_blocks",
|
|
425
|
+
(self._block_id,),
|
|
426
|
+
)
|
|
427
|
+
self._execute(
|
|
428
|
+
"INSERT INTO records SELECT id, dataset, row_number, block_id + ?, "
|
|
429
|
+
"byte_offset, byte_length, pmid FROM captured.records",
|
|
430
|
+
(self._block_id,),
|
|
431
|
+
)
|
|
432
|
+
self._execute(
|
|
433
|
+
"INSERT INTO edges SELECT source, target, relation, label FROM captured.edges"
|
|
434
|
+
)
|
|
435
|
+
self.db.commit()
|
|
436
|
+
self._block_id += next_block
|
|
437
|
+
self.sources.update(sources)
|
|
438
|
+
except BaseException:
|
|
439
|
+
# Preserve the import error if cleanup also encounters a damaged DB.
|
|
440
|
+
# The enclosing build discards this graph and never publishes it.
|
|
441
|
+
with contextlib.suppress(sqlite3.Error):
|
|
442
|
+
self.db.rollback()
|
|
443
|
+
self.db.execute("DETACH DATABASE captured")
|
|
444
|
+
raise
|
|
445
|
+
else:
|
|
446
|
+
self.db.execute("DETACH DATABASE captured")
|
|
447
|
+
|
|
371
448
|
def observe(self, record_id: str, sample_label="") -> str:
|
|
372
449
|
node = f"observation:{record_id}"
|
|
373
450
|
if sample_label:
|