descriptron-core 2.6.0__tar.gz → 2.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/PKG-INFO +1 -1
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/pyproject.toml +1 -1
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biosyslit_rag_retrieval_v2.py +42 -0
- descriptron_core-2.7.0/src/descriptron_core/tools/descriptron_check_cross_image_copies_v1.py +200 -0
- descriptron_core-2.7.0/src/descriptron_core/tools/descriptron_labelled_plates_v1.py +290 -0
- descriptron_core-2.7.0/src/descriptron_core/tools/descriptron_reexamine_v2.py +923 -0
- descriptron_core-2.7.0/src/descriptron_core/tools/generate_species_plates.py +299 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/run_full_pipeline_v2.py +17 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/.gitignore +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/LICENSE +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/NOTICE +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/README.md +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/__init__.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/cli.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/data/biorag_prompts/biorag_system_prompts_v1.txt +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/data/biorag_prompts/biorag_system_prompts_v2.txt +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/data/biorag_prompts/diaphorina_user_prompts.txt +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/data/biorag_prompts/ontologies/aism.obo +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/data/biorag_prompts/ontologies/pato.obo +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/data/biorag_prompts/schemas/sdd_1.1/SDD.xsd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/data/biorag_prompts/schemas/sdd_1.1/SDD_EnumLib.xsd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/data/biorag_prompts/schemas/sdd_1.1/SDD_TypeLib.xsd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/data/biorag_prompts/schemas/sdd_1.1/UBIF_BasePattern.xsd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/data/biorag_prompts/schemas/sdd_1.1/UBIF_CoreExtensions.xsd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/data/biorag_prompts/schemas/sdd_1.1/UBIF_CoreOntology.xsd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/data/biorag_prompts/schemas/sdd_1.1/UBIF_EnumLib.xsd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/data/biorag_prompts/schemas/sdd_1.1/UBIF_LinneanCore.xsd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/data/biorag_prompts/schemas/sdd_1.1/UBIF_TypeLib.xsd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/data/biorag_prompts/schemas/sdd_1.1/w3c-schemata/xml.xsd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/data/biorag_prompts/schemas/taxpub/sample_treatment_deposit_example.xml +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/data/biorag_prompts/schemas/taxpub/sample_zookeys_26056_tp.xml +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/data/biorag_prompts/schemas/taxpub/tax-treatment-NS0-v1_flat.dtd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/data/biorag_prompts/taxon_profiles/TEMPLATE_taxon_profile.yaml +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/data/biorag_prompts/taxon_profiles/diaphorina_taxon_profile.yaml +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/FHS_and_CLAHE_V21.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_add_discrete_characters_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_annotation_reorient_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_annotation_screen_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_autapomorphy_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_calibrate_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_calibration_figures_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_character_gate_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_character_reliability_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_character_robustness_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_collaborator_forms_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_computed_sets_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_confabulation_checker_v2.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_congruence_compare_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_description_refiner_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_descriptive_scoring_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_feature_policy.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_frame_orientation_calibrate_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_frame_orientation_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_gate_compare_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_graph_identify_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_graph_support_figure_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_homology_frame_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_instrument_compare_v3.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_key_builder_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_key_feature_filter_v2.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_key_fuzzy_figures_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_key_fuzzy_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_key_holdout_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_key_jackknife_support_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_label_conflict_check_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_llm_backend.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_make_taxon_profile_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_methods_summary_figure_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_novelty_score_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_ontology_annotator_v2.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_outgroup_batch_check_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_outline_shape_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_prompts/biorag_system_prompts_v1.txt +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_prompts/biorag_system_prompts_v2.txt +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_prompts/diaphorina_user_prompts.txt +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_prompts/ontologies/aism.obo +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_prompts/ontologies/pato.obo +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_prompts/schemas/sdd_1.1/SDD.xsd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_prompts/schemas/sdd_1.1/SDD_EnumLib.xsd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_prompts/schemas/sdd_1.1/SDD_TypeLib.xsd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_prompts/schemas/sdd_1.1/UBIF_BasePattern.xsd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_prompts/schemas/sdd_1.1/UBIF_CoreExtensions.xsd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_prompts/schemas/sdd_1.1/UBIF_CoreOntology.xsd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_prompts/schemas/sdd_1.1/UBIF_EnumLib.xsd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_prompts/schemas/sdd_1.1/UBIF_LinneanCore.xsd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_prompts/schemas/sdd_1.1/UBIF_TypeLib.xsd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_prompts/schemas/sdd_1.1/w3c-schemata/xml.xsd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_prompts/schemas/taxpub/sample_treatment_deposit_example.xml +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_prompts/schemas/taxpub/sample_zookeys_26056_tp.xml +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_prompts/schemas/taxpub/tax-treatment-NS0-v1_flat.dtd +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_prompts/taxon_profiles/TEMPLATE_taxon_profile.yaml +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_prompts/taxon_profiles/diaphorina_taxon_profile.yaml +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_provenance_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_rebuild_downstream_v1.sh +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_reliability_figures_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_rename_species_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_rerun_descriptive_tables_v1.sh +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_retest_compare_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_specimen_id.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_subjective_checks_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_type_material_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_vlm_characters_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_vlm_heatmap_figures_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_vlm_reliability_figure_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biorag_vlm_roi_figures_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/biosyslit_rag_retrieval.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/build_monograph_exports_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/build_species_treatment_docx_v2.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/coco_combiner_V13.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/coco_converter_v24.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/coi_species_tree_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/color_extraction_with_names-v8.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/color_phenomics_homology_v2_1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/compile_specimen_data.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/descriptron_centerline.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/descriptron_character_signal_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/descriptron_coco_tools.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/descriptron_convert.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/descriptron_credentials.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/descriptron_joints.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/descriptron_metadata.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/descriptron_phylo.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/descriptron_phylo_figure_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/descriptron_reexamine_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/descriptron_rosetta.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/descriptron_shape_stats.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/descriptron_trait_stats.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/dna_vs_morphology_v1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/landmark_gpa_V1.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/landmark_gpa_V2.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/make_reproducibility_bundle.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/semi_landmark_and_kpts_procrustesV34_GPA.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/semi_landmark_and_kpts_procrustesV42_GPA.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/texture_phenomics_from_coco.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/texture_phenomics_homology.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/validate_centerline_lengths.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/validate_phylo_published_waldron2025.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/validate_phylo_real_tree.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/validate_phylo_traits.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/validate_pipeline_gm_vs_geomorph.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/validate_semilandmarks_vs_geomorph.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/validate_shape_stats_vs_geomorph.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/validate_shape_stats_vs_rrpp.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/validate_texture_glcm.py +0 -0
- {descriptron_core-2.6.0 → descriptron_core-2.7.0}/src/descriptron_core/tools/validate_trait_stats.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: descriptron-core
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.7.0
|
|
4
4
|
Summary: Evidence-tiered species descriptions, keys and audits from phenomic matrices
|
|
5
5
|
Project-URL: Homepage, https://github.com/alexrvandam/Descriptron
|
|
6
6
|
Project-URL: Repository, https://github.com/alexrvandam/Descriptron
|
|
@@ -64,6 +64,48 @@ from biorag_llm_backend import load_prompt_library, make_llm_client # noqa: E40
|
|
|
64
64
|
|
|
65
65
|
V2_VERSION = "2.0"
|
|
66
66
|
|
|
67
|
+
# ── retrieval log (opt-in): BIORAG_RETRIEVAL_LOG=<file.jsonl> records every literature search and every passage
|
|
68
|
+
# returned (source, DOI, section, score), so a run can show WHERE in the literature the model was primed from.
|
|
69
|
+
# BIORAG_RETRIEVAL_CONTEXT (e.g. the species code) is copied into each record. v1 is not modified: the two
|
|
70
|
+
# functions are wrapped here.
|
|
71
|
+
_RLOG = os.environ.get("BIORAG_RETRIEVAL_LOG")
|
|
72
|
+
if _RLOG:
|
|
73
|
+
import datetime as _dt
|
|
74
|
+
import json as _json
|
|
75
|
+
|
|
76
|
+
def _rlog_write(rec):
|
|
77
|
+
rec = {"time": _dt.datetime.now().isoformat(timespec="seconds"),
|
|
78
|
+
"context": os.environ.get("BIORAG_RETRIEVAL_CONTEXT", ""), **rec}
|
|
79
|
+
with open(_RLOG, "a") as _f:
|
|
80
|
+
_f.write(_json.dumps(rec, ensure_ascii=False) + "\n")
|
|
81
|
+
|
|
82
|
+
_orig_retrieve = base.TreatmentIndex.retrieve
|
|
83
|
+
|
|
84
|
+
def _logged_retrieve(self, query, _orig=_orig_retrieve):
|
|
85
|
+
res = _orig(self, query)
|
|
86
|
+
_rlog_write({"kind": "index_retrieve",
|
|
87
|
+
"query": {k: getattr(query, k, "") for k in ("taxon_name", "taxon_family", "body_region",
|
|
88
|
+
"description_mode", "free_text", "k")},
|
|
89
|
+
"index_chunks": len(getattr(self, "chunks", []) or []),
|
|
90
|
+
"results": [{"rank": i + 1, "score": round(float(r.score), 4), "chunk_id": r.chunk.chunk_id,
|
|
91
|
+
"source_type": r.chunk.source_type, "source_id": r.chunk.source_id,
|
|
92
|
+
"source_doi": r.chunk.source_doi, "source_title": r.chunk.source_title,
|
|
93
|
+
"section": r.chunk.description_section, "body_region": r.chunk.body_region,
|
|
94
|
+
"match_reasons": list(r.match_reasons or []),
|
|
95
|
+
"text_preview": (r.chunk.text or "")[:240]} for i, r in enumerate(res)]})
|
|
96
|
+
return res
|
|
97
|
+
base.TreatmentIndex.retrieve = _logged_retrieve
|
|
98
|
+
|
|
99
|
+
_orig_search = base.BioSysLitIngester.search
|
|
100
|
+
|
|
101
|
+
def _logged_search(self, query, *a, _orig=_orig_search, **kw):
|
|
102
|
+
recs = _orig(self, query, *a, **kw)
|
|
103
|
+
_rlog_write({"kind": "biosyslit_search", "query": query,
|
|
104
|
+
"records": [{"id": r.get("id"), "doi": r.get("doi", ""),
|
|
105
|
+
"title": (r.get("metadata", {}) or {}).get("title", "")} for r in (recs or [])]})
|
|
106
|
+
return recs
|
|
107
|
+
base.BioSysLitIngester.search = _logged_search
|
|
108
|
+
|
|
67
109
|
SUBCOMMAND_MODULES = {
|
|
68
110
|
"build-key": "biorag_key_builder_v1",
|
|
69
111
|
"refine-descriptions": "biorag_description_refiner_v1",
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
descriptron_check_cross_image_copies_v1.py - find annotations copied onto images they do not belong to
|
|
4
|
+
=======================================================================================================
|
|
5
|
+
|
|
6
|
+
Before GUI v81, "Load Annotations" drew EVERY annotation of a COCO file on the image on screen and saved
|
|
7
|
+
them under that image. A file that was loaded while another image was showing can therefore carry the
|
|
8
|
+
same outline twice: once under its own image, once under an image it was never drawn on. A copy keeps
|
|
9
|
+
the exact pixel position AND the outline points of the original, so it is found by comparing annotations
|
|
10
|
+
filed under DIFFERENT images: a pair that overlaps almost completely (IoU >= --min_iou, default 0.95) and
|
|
11
|
+
shares most of its outline vertices exactly (>= --min_shared, default 0.9) is a copy. Two people tracing
|
|
12
|
+
similar structures on aligned images can overlap at 0.95 but share only a few percent of their points (1-5%
|
|
13
|
+
on aligned ant heads), so overlap alone is not enough. Where either mask is RLE there are no vertices; such
|
|
14
|
+
a pair is reported on overlap alone and marked "check". Pairs that share 20-90% of their points are listed
|
|
15
|
+
as "partly shared" (an outline copied and then edited, e.g. a second photo of the same specimen).
|
|
16
|
+
|
|
17
|
+
The program cannot tell which of a pair is the original (both may since have been re-labelled), so it
|
|
18
|
+
reports both and changes nothing. To write a cleaned copy, name what to drop: --drop_ids <annotation ids>
|
|
19
|
+
and/or --drop_image <image file name> (every annotation filed under that image). The input file is
|
|
20
|
+
never modified; the cleaned file is written beside it as <name>_cleaned.json unless --output is given.
|
|
21
|
+
|
|
22
|
+
python descriptron_check_cross_image_copies_v1.py --coco annotations.json
|
|
23
|
+
python descriptron_check_cross_image_copies_v1.py --coco annotations.json --drop_image MJ_31308000000.tif
|
|
24
|
+
|
|
25
|
+
Writes <name>_cross_image_copies.tsv beside the input (or --report) and prints a summary.
|
|
26
|
+
"""
|
|
27
|
+
import argparse
|
|
28
|
+
import csv
|
|
29
|
+
import json
|
|
30
|
+
import os
|
|
31
|
+
import sys
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
|
|
34
|
+
import numpy as np
|
|
35
|
+
|
|
36
|
+
STEP = 4 # masks compared at 1/4 resolution
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _image_name(im):
|
|
40
|
+
return os.path.basename(str(im.get("file_name") or im.get("id")))
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _mask(seg, shape, step=STEP):
|
|
44
|
+
"""annotation mask at 1/step resolution on a canvas shared by both images (same pixel coordinates)"""
|
|
45
|
+
import cv2
|
|
46
|
+
H, W = shape
|
|
47
|
+
m = np.zeros((H // step + 1, W // step + 1), np.uint8)
|
|
48
|
+
if isinstance(seg, list):
|
|
49
|
+
for poly in seg:
|
|
50
|
+
p = (np.asarray(poly, float).reshape(-1, 2) / step).round().astype(np.int32)
|
|
51
|
+
if len(p) >= 3:
|
|
52
|
+
cv2.fillPoly(m, [p], 1)
|
|
53
|
+
elif isinstance(seg, dict) and "counts" in seg:
|
|
54
|
+
h, w = seg["size"]
|
|
55
|
+
counts = seg["counts"]
|
|
56
|
+
if isinstance(counts, str):
|
|
57
|
+
from pycocotools import mask as mu
|
|
58
|
+
full = mu.decode(seg)
|
|
59
|
+
else:
|
|
60
|
+
flat = np.zeros(h * w, np.uint8); i = 0; v = 0
|
|
61
|
+
for r in counts:
|
|
62
|
+
flat[i:i + r] = v; i += r; v = 1 - v
|
|
63
|
+
full = flat.reshape((w, h)).T
|
|
64
|
+
small = full[::step, ::step]
|
|
65
|
+
m[:small.shape[0], :small.shape[1]] = small[:m.shape[0], :m.shape[1]]
|
|
66
|
+
return m.astype(bool)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _bbox(a):
|
|
70
|
+
b = a.get("bbox")
|
|
71
|
+
if b and len(b) == 4:
|
|
72
|
+
return [float(x) for x in b]
|
|
73
|
+
seg = a.get("segmentation")
|
|
74
|
+
if isinstance(seg, list) and seg:
|
|
75
|
+
p = np.concatenate([np.asarray(s, float).reshape(-1, 2) for s in seg if len(s) >= 6])
|
|
76
|
+
return [p[:, 0].min(), p[:, 1].min(), np.ptp(p[:, 0]), np.ptp(p[:, 1])]
|
|
77
|
+
return None
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _verts(a):
|
|
81
|
+
s = a.get("segmentation")
|
|
82
|
+
if not isinstance(s, list):
|
|
83
|
+
return None
|
|
84
|
+
v = set()
|
|
85
|
+
for poly in s:
|
|
86
|
+
q = np.round(np.asarray(poly, float).reshape(-1, 2)).astype(int)
|
|
87
|
+
v.update(map(tuple, q.tolist()))
|
|
88
|
+
return v
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def shared_vertices(a, b):
|
|
92
|
+
"""fraction of the smaller outline's points found exactly in the other; None if either is RLE"""
|
|
93
|
+
va, vb = _verts(a), _verts(b)
|
|
94
|
+
if not va or not vb:
|
|
95
|
+
return None
|
|
96
|
+
return len(va & vb) / min(len(va), len(vb))
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def find_copies(d, min_iou=0.95, corner_tol=0.05, min_shared=0.9, partly=0.2):
|
|
100
|
+
"""pairs (a, b, iou) of mask annotations under different images whose masks coincide"""
|
|
101
|
+
imgs = {im["id"]: im for im in d.get("images", [])}
|
|
102
|
+
cats = {c["id"]: c["name"] for c in d.get("categories", [])}
|
|
103
|
+
anns = [a for a in d.get("annotations", []) if a.get("segmentation") and _bbox(a)]
|
|
104
|
+
out = []
|
|
105
|
+
cache = {}
|
|
106
|
+
for i in range(len(anns)):
|
|
107
|
+
a = anns[i]; ba = _bbox(a)
|
|
108
|
+
for j in range(i + 1, len(anns)):
|
|
109
|
+
b = anns[j]
|
|
110
|
+
if a.get("image_id") == b.get("image_id"):
|
|
111
|
+
continue
|
|
112
|
+
bb = _bbox(b)
|
|
113
|
+
tol = corner_tol * max(ba[2], ba[3], bb[2], bb[3], 1.0)
|
|
114
|
+
if abs(ba[0] - bb[0]) > tol or abs(ba[1] - bb[1]) > tol: # copies sit at the same corner
|
|
115
|
+
continue
|
|
116
|
+
H = int(max(ba[1] + ba[3], bb[1] + bb[3])) + 2
|
|
117
|
+
W = int(max(ba[0] + ba[2], bb[0] + bb[2])) + 2
|
|
118
|
+
for k, x in ((i, a), (j, b)):
|
|
119
|
+
if (k, H, W) not in cache:
|
|
120
|
+
cache[(k, H, W)] = _mask(x["segmentation"], (H, W))
|
|
121
|
+
ma, mb = cache[(i, H, W)], cache[(j, H, W)]
|
|
122
|
+
inter = np.logical_and(ma, mb).sum(); union = np.logical_or(ma, mb).sum()
|
|
123
|
+
iou = inter / union if union else 0.0
|
|
124
|
+
if iou >= min_iou:
|
|
125
|
+
sh = shared_vertices(a, b)
|
|
126
|
+
if sh is None:
|
|
127
|
+
verdict = "check (RLE: overlap only)"
|
|
128
|
+
elif sh >= min_shared:
|
|
129
|
+
verdict = "copy"
|
|
130
|
+
elif sh >= partly:
|
|
131
|
+
verdict = "partly shared"
|
|
132
|
+
else:
|
|
133
|
+
continue # similar shapes traced independently
|
|
134
|
+
out.append({"verdict": verdict, "iou": round(float(iou), 4),
|
|
135
|
+
"shared_vertices": "" if sh is None else round(sh, 3),
|
|
136
|
+
"annotation_id_1": a.get("id"), "image_1": _image_name(imgs.get(a.get("image_id"), {"id": a.get("image_id")})),
|
|
137
|
+
"category_1": cats.get(a.get("category_id"), a.get("category_id")),
|
|
138
|
+
"annotation_id_2": b.get("id"), "image_2": _image_name(imgs.get(b.get("image_id"), {"id": b.get("image_id")})),
|
|
139
|
+
"category_2": cats.get(b.get("category_id"), b.get("category_id"))})
|
|
140
|
+
return out
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def drop(d, ids=(), image=None):
|
|
144
|
+
"""a copy of the COCO dict without the named annotations / every annotation of the named image"""
|
|
145
|
+
ids = {str(x) for x in ids}
|
|
146
|
+
gone_imgs = set()
|
|
147
|
+
if image:
|
|
148
|
+
want = os.path.splitext(os.path.basename(image))[0].lower()
|
|
149
|
+
gone_imgs = {im["id"] for im in d.get("images", [])
|
|
150
|
+
if os.path.splitext(_image_name(im))[0].lower() == want
|
|
151
|
+
or os.path.basename(str(im["id"])).lower() == want}
|
|
152
|
+
if not gone_imgs:
|
|
153
|
+
sys.exit(f"--drop_image {image}: no such image in the file")
|
|
154
|
+
out = dict(d)
|
|
155
|
+
out["annotations"] = [a for a in d.get("annotations", [])
|
|
156
|
+
if str(a.get("id")) not in ids and a.get("image_id") not in gone_imgs]
|
|
157
|
+
out["images"] = [im for im in d.get("images", []) if im["id"] not in gone_imgs]
|
|
158
|
+
return out
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def main():
|
|
162
|
+
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
163
|
+
ap.add_argument("--coco", required=True, help="COCO .json or .rlejson")
|
|
164
|
+
ap.add_argument("--min_iou", type=float, default=0.95)
|
|
165
|
+
ap.add_argument("--min_shared", type=float, default=0.9, help="share of identical outline points for a copy")
|
|
166
|
+
ap.add_argument("--report", default=None, help="TSV of the pairs found (default: beside the input)")
|
|
167
|
+
ap.add_argument("--drop_ids", nargs="*", default=[], help="annotation ids to leave out of the cleaned copy")
|
|
168
|
+
ap.add_argument("--drop_image", default=None, help="leave out this image and everything filed under it")
|
|
169
|
+
ap.add_argument("--output", default=None, help="cleaned copy (default: <name>_cleaned<ext> beside the input)")
|
|
170
|
+
a = ap.parse_args()
|
|
171
|
+
src = Path(a.coco)
|
|
172
|
+
d = json.load(open(src))
|
|
173
|
+
pairs = find_copies(d, a.min_iou, min_shared=a.min_shared)
|
|
174
|
+
rep = Path(a.report) if a.report else src.with_name(src.stem + "_cross_image_copies.tsv")
|
|
175
|
+
with open(rep, "w", newline="") as f:
|
|
176
|
+
cols = ["verdict", "iou", "shared_vertices", "annotation_id_1", "image_1", "category_1", "annotation_id_2", "image_2", "category_2"]
|
|
177
|
+
w = csv.DictWriter(f, fieldnames=cols, delimiter="\t"); w.writeheader(); w.writerows(pairs)
|
|
178
|
+
n_img = len(d.get("images", [])); n_ann = len(d.get("annotations", []))
|
|
179
|
+
n_copy = sum(p["verdict"] == "copy" for p in pairs)
|
|
180
|
+
print(f"{src.name}: {n_img} images, {n_ann} annotations; {n_copy} cross-image cop"
|
|
181
|
+
f"{'y' if n_copy == 1 else 'ies'}, {len(pairs) - n_copy} other pair(s) to look at")
|
|
182
|
+
for p in pairs:
|
|
183
|
+
sv = "" if p["shared_vertices"] == "" else f", {100 * p['shared_vertices']:.0f}% points shared"
|
|
184
|
+
print(f" {p['verdict']:24s} IoU {p['iou']:.3f}{sv} {p['image_1']} [{p['category_1']}, id {p['annotation_id_1']}]"
|
|
185
|
+
f" == {p['image_2']} [{p['category_2']}, id {p['annotation_id_2']}]")
|
|
186
|
+
print(f"report -> {rep}")
|
|
187
|
+
if a.drop_ids or a.drop_image:
|
|
188
|
+
out = Path(a.output) if a.output else src.with_name(src.stem + "_cleaned" + src.suffix)
|
|
189
|
+
if out.resolve() == src.resolve():
|
|
190
|
+
sys.exit("--output must not be the input file")
|
|
191
|
+
cleaned = drop(d, a.drop_ids, a.drop_image)
|
|
192
|
+
tmp = out.with_suffix(out.suffix + ".tmp")
|
|
193
|
+
json.dump(cleaned, open(tmp, "w")); os.replace(tmp, out)
|
|
194
|
+
print(f"cleaned copy: {len(d['annotations'])} -> {len(cleaned['annotations'])} annotations -> {out}")
|
|
195
|
+
elif n_copy:
|
|
196
|
+
print("nothing changed; to write a cleaned copy add --drop_ids <id ...> or --drop_image <file name>")
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
if __name__ == "__main__":
|
|
200
|
+
main()
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
descriptron_labelled_plates_v1.py - species plates in the specimens' own colours, with every structure named
|
|
4
|
+
=============================================================================================================
|
|
5
|
+
|
|
6
|
+
The companion to generate_species_plates.py (which is left as it is): the same specimens, but with no mask
|
|
7
|
+
overlay. Each annotated structure gets a thin leader line from the edge of the structure to its name, set in a
|
|
8
|
+
column beside the image, the way a hand-labelled plate in a taxonomic paper looks.
|
|
9
|
+
|
|
10
|
+
Where the line ends: it starts from the deepest point of the part of the structure that no smaller structure
|
|
11
|
+
covers (for a structure with others inside it, such as the whole wing or whole head, from the deepest point of
|
|
12
|
+
its whole outline, since its uncovered remainder may be an unannotated part such as the eyes), heads straight
|
|
13
|
+
for the label, and the dot is put where it leaves the structure's outline. A wing cell's
|
|
14
|
+
dot is therefore on that cell's own edge, facing its label, and the whole wing's dot is on the wing margin.
|
|
15
|
+
With --anchor centre the line runs on to that deepest point instead, so the dot sits in the middle of the
|
|
16
|
+
structure (the deepest point, not the centroid, which can fall outside a curved or U-shaped structure). A
|
|
17
|
+
structure with smaller ones inside it (the whole wing around its cells, the whole head around the vertex) has
|
|
18
|
+
no middle of its own, so its dot goes to the centroid of its whole outline (or its deepest point, if the
|
|
19
|
+
centroid falls outside it).
|
|
20
|
+
Labels sit on the side of the image the structure is on, at the height of the structure, pushed apart only as
|
|
21
|
+
far as needed to stop them overlapping.
|
|
22
|
+
|
|
23
|
+
For every species (from --group_labels) writes into --plates_dir/<species>/:
|
|
24
|
+
<image>_labelled.png one labelled image per specimen photograph
|
|
25
|
+
<species>_labelled_plate.png/pdf those images together (up to --max_images_per_species)
|
|
26
|
+
|
|
27
|
+
python descriptron_labelled_plates_v1.py --coco_json <coco.json> --image_dir <images> \\
|
|
28
|
+
--group_labels <group_labels.csv> --plates_dir <out> [--species sp1 sp4] [--outline] [--anchor centre]
|
|
29
|
+
"""
|
|
30
|
+
import argparse
|
|
31
|
+
import os
|
|
32
|
+
import sys
|
|
33
|
+
from collections import defaultdict
|
|
34
|
+
from pathlib import Path
|
|
35
|
+
|
|
36
|
+
import cv2
|
|
37
|
+
import matplotlib
|
|
38
|
+
matplotlib.use("Agg")
|
|
39
|
+
import matplotlib.patheffects as pe
|
|
40
|
+
import matplotlib.pyplot as plt
|
|
41
|
+
import numpy as np
|
|
42
|
+
|
|
43
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
44
|
+
import generate_species_plates as gsp # noqa: E402 (COCO, mask and group-label readers)
|
|
45
|
+
|
|
46
|
+
INK = "#1d2327"
|
|
47
|
+
HALO = [pe.withStroke(linewidth=1.8, foreground="white")]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def pretty(name):
|
|
51
|
+
return name.replace("_", " ")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def structures_of(anns, id_to_cat, h, w, scale):
|
|
55
|
+
"""(name, mask) at display scale for every annotation with a segmentation (keypoint-only ones skipped)"""
|
|
56
|
+
hs, ws = max(1, round(h * scale)), max(1, round(w * scale))
|
|
57
|
+
out = []
|
|
58
|
+
for a in anns:
|
|
59
|
+
seg = a.get("segmentation")
|
|
60
|
+
if not seg:
|
|
61
|
+
continue
|
|
62
|
+
name = id_to_cat.get(a["category_id"], "?")
|
|
63
|
+
if isinstance(seg, list):
|
|
64
|
+
m = np.zeros((hs, ws), np.uint8)
|
|
65
|
+
for poly in seg:
|
|
66
|
+
pts = (np.asarray(poly, float).reshape(-1, 2) * scale).round().astype(np.int32)
|
|
67
|
+
if len(pts) >= 3:
|
|
68
|
+
cv2.fillPoly(m, [pts], 1)
|
|
69
|
+
m = m.astype(bool)
|
|
70
|
+
else:
|
|
71
|
+
m = cv2.resize(gsp.seg_to_mask(seg, h, w).astype(np.uint8), (ws, hs),
|
|
72
|
+
interpolation=cv2.INTER_NEAREST).astype(bool)
|
|
73
|
+
if m.sum() >= 4:
|
|
74
|
+
out.append((name, m))
|
|
75
|
+
return out
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def own_regions(structs):
|
|
79
|
+
"""each structure minus every SMALLER structure on top of it (falls back to the whole mask if nothing is
|
|
80
|
+
left), so a container such as the whole wing points at its margin, not at a cell inside it"""
|
|
81
|
+
areas = [m.sum() for _, m in structs]
|
|
82
|
+
own = []
|
|
83
|
+
for i, (n, m) in enumerate(structs):
|
|
84
|
+
cover = np.zeros_like(m)
|
|
85
|
+
for j, (_, o) in enumerate(structs):
|
|
86
|
+
if j != i and areas[j] < areas[i]:
|
|
87
|
+
cover |= o
|
|
88
|
+
r = m & ~cover
|
|
89
|
+
own.append(r if r.sum() > 0.03 * m.sum() else m)
|
|
90
|
+
return own
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def is_container(structs, i, inside=0.8):
|
|
94
|
+
"""True when some smaller structure lies (mostly) inside structure i, e.g. the whole wing around its cells"""
|
|
95
|
+
m = structs[i][1]
|
|
96
|
+
a = m.sum()
|
|
97
|
+
return any(o.sum() < a and (o & m).sum() >= inside * o.sum() for j, (_, o) in enumerate(structs) if j != i)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def centroid(mask):
|
|
101
|
+
"""centroid of the whole outline; if it falls outside the structure (a curved one), its deepest point"""
|
|
102
|
+
ys, xs = np.nonzero(mask)
|
|
103
|
+
x, y = int(round(xs.mean())), int(round(ys.mean()))
|
|
104
|
+
if mask[y, x]:
|
|
105
|
+
return x, y
|
|
106
|
+
dt = cv2.distanceTransform(np.pad(mask, 1).astype(np.uint8), cv2.DIST_L2, 3)[1:-1, 1:-1]
|
|
107
|
+
y, x = np.unravel_index(np.argmax(dt), dt.shape)
|
|
108
|
+
return x, y
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def exit_point(mask, start, toward):
|
|
112
|
+
"""walk from `start` (inside the structure) straight toward the label; the last pixel still inside the
|
|
113
|
+
structure is where the leader line meets its outline, on the side facing the label"""
|
|
114
|
+
H, W = mask.shape
|
|
115
|
+
x0, y0 = start
|
|
116
|
+
dx, dy = toward[0] - x0, toward[1] - y0
|
|
117
|
+
n = int(np.hypot(dx, dy)) + 1
|
|
118
|
+
last = (x0, y0)
|
|
119
|
+
for t in np.linspace(0, 1, n):
|
|
120
|
+
x, y = int(round(x0 + t * dx)), int(round(y0 + t * dy))
|
|
121
|
+
if not (0 <= x < W and 0 <= y < H) or not mask[y, x]:
|
|
122
|
+
break
|
|
123
|
+
last = (x, y)
|
|
124
|
+
return last
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def spread(targets, lo, hi, gap):
|
|
128
|
+
"""label heights as close to their targets as possible, at least `gap` apart, inside [lo, hi]"""
|
|
129
|
+
if not targets:
|
|
130
|
+
return []
|
|
131
|
+
order = np.argsort(targets)
|
|
132
|
+
n = len(targets)
|
|
133
|
+
gap = min(gap, (hi - lo) / max(1, n - 1)) if n > 1 else gap
|
|
134
|
+
ys = np.clip(np.asarray(targets, float)[order], lo, hi)
|
|
135
|
+
for _ in range(200): # alternate push-down / push-up until stable
|
|
136
|
+
moved = False
|
|
137
|
+
for k in range(1, n):
|
|
138
|
+
if ys[k] - ys[k - 1] < gap - 1e-9:
|
|
139
|
+
ys[k] = ys[k - 1] + gap; moved = True
|
|
140
|
+
if ys[-1] > hi:
|
|
141
|
+
ys[-1] = hi
|
|
142
|
+
for k in range(n - 2, -1, -1):
|
|
143
|
+
if ys[k + 1] - ys[k] < gap - 1e-9:
|
|
144
|
+
ys[k] = ys[k + 1] - gap; moved = True
|
|
145
|
+
if ys[0] < lo:
|
|
146
|
+
ys += lo - ys[0]; moved = True
|
|
147
|
+
if not moved:
|
|
148
|
+
break
|
|
149
|
+
out = np.empty(n)
|
|
150
|
+
out[order] = ys
|
|
151
|
+
return list(out)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def draw_labelled(img_rgb, structs, out_path, title=None, img_width_in=5.0, fontsize=9, outline=False, dpi=200,
|
|
155
|
+
anchor="edge"):
|
|
156
|
+
H, W = img_rgb.shape[:2]
|
|
157
|
+
own = own_regions(structs)
|
|
158
|
+
poles = []
|
|
159
|
+
for i, r in enumerate(own):
|
|
160
|
+
if is_container(structs, i): # its uncovered remainder can be an unannotated part (a head's eyes),
|
|
161
|
+
r = structs[i][1] # so start from the middle of the whole outline instead
|
|
162
|
+
dt = cv2.distanceTransform(np.pad(r, 1).astype(np.uint8), cv2.DIST_L2, 3)[1:-1, 1:-1]
|
|
163
|
+
y, x = np.unravel_index(np.argmax(dt), dt.shape)
|
|
164
|
+
poles.append((x, y))
|
|
165
|
+
names = [pretty(n) for n, _ in structs]
|
|
166
|
+
left = [i for i in range(len(structs)) if poles[i][0] < W / 2]
|
|
167
|
+
right = [i for i in range(len(structs)) if i not in left]
|
|
168
|
+
per_in = W / img_width_in # data units per inch
|
|
169
|
+
char_in = fontsize * 0.62 / 72
|
|
170
|
+
margin_l = (max((len(names[i]) for i in left), default=0) * char_in + 0.45) * per_in
|
|
171
|
+
margin_r = (max((len(names[i]) for i in right), default=0) * char_in + 0.45) * per_in
|
|
172
|
+
gap = fontsize * 1.45 / 72 * per_in
|
|
173
|
+
fig_w = img_width_in + (margin_l + margin_r) / per_in
|
|
174
|
+
fig_h = H / per_in + (0.35 if title else 0.1)
|
|
175
|
+
fig, ax = plt.subplots(figsize=(fig_w, fig_h))
|
|
176
|
+
ax.imshow(img_rgb)
|
|
177
|
+
if outline:
|
|
178
|
+
for (_, m) in structs:
|
|
179
|
+
cs, _ = cv2.findContours(m.astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
|
|
180
|
+
for c in cs:
|
|
181
|
+
c = c[:, 0, :]
|
|
182
|
+
ax.plot(np.r_[c[:, 0], c[0, 0]], np.r_[c[:, 1], c[0, 1]], color="white", lw=0.5, alpha=0.8)
|
|
183
|
+
for side, idx in (("L", left), ("R", right)):
|
|
184
|
+
ys = spread([poles[i][1] for i in idx], 0.02 * H, 0.98 * H, gap)
|
|
185
|
+
tx = -0.25 * per_in if side == "L" else W + 0.25 * per_in
|
|
186
|
+
for i, ty in zip(idx, ys):
|
|
187
|
+
if anchor == "centre":
|
|
188
|
+
ax_, ay = centroid(structs[i][1]) if is_container(structs, i) else poles[i]
|
|
189
|
+
else:
|
|
190
|
+
ax_, ay = exit_point(structs[i][1], poles[i], (tx, ty))
|
|
191
|
+
ax.plot([tx, ax_], [ty, ay], color=INK, lw=0.6, path_effects=HALO, solid_capstyle="round", zorder=3)
|
|
192
|
+
ax.plot(ax_, ay, "o", ms=2.6, mfc=INK, mec="white", mew=0.6, zorder=4)
|
|
193
|
+
ax.text(tx + (-0.06 if side == "L" else 0.06) * per_in, ty, names[i], fontsize=fontsize, color=INK,
|
|
194
|
+
ha="right" if side == "L" else "left", va="center", zorder=5)
|
|
195
|
+
ax.set_xlim(-margin_l, W + margin_r); ax.set_ylim(H, 0); ax.axis("off")
|
|
196
|
+
if title:
|
|
197
|
+
ax.set_title(title, fontsize=fontsize + 1, color=INK, pad=4)
|
|
198
|
+
fig.subplots_adjust(0, 0, 1, (fig_h - 0.35) / fig_h if title else 1)
|
|
199
|
+
fig.savefig(out_path, dpi=dpi, facecolor="white")
|
|
200
|
+
plt.close(fig)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def make_plate(species, files, out_base, cols=2, dpi=200):
|
|
204
|
+
ims = [plt.imread(str(f)) for f in files]
|
|
205
|
+
rows = int(np.ceil(len(ims) / cols))
|
|
206
|
+
cell_w = 6.5
|
|
207
|
+
heights = [max(im.shape[0] / im.shape[1] * cell_w for im in ims[r * cols:(r + 1) * cols]) for r in range(rows)]
|
|
208
|
+
fig, axs = plt.subplots(rows, cols, figsize=(cell_w * cols, sum(heights) + 0.6),
|
|
209
|
+
gridspec_kw={"height_ratios": heights}, squeeze=False)
|
|
210
|
+
for k, ax in enumerate(axs.ravel()):
|
|
211
|
+
ax.axis("off")
|
|
212
|
+
if k < len(ims):
|
|
213
|
+
ax.imshow(ims[k])
|
|
214
|
+
ax.text(0.0, 1.0, "abcdefghijklmnop"[k], transform=ax.transAxes, fontsize=13, weight="bold",
|
|
215
|
+
va="bottom", ha="left", color=INK)
|
|
216
|
+
fig.suptitle(species, fontsize=15, style="italic" if not species[:2].lower() in ("sp", "cf") else "normal",
|
|
217
|
+
x=0.01, ha="left", color=INK)
|
|
218
|
+
fig.tight_layout(rect=(0, 0, 1, 0.98))
|
|
219
|
+
for ext in ("png", "pdf"):
|
|
220
|
+
fig.savefig(f"{out_base}.{ext}", dpi=dpi, facecolor="white")
|
|
221
|
+
plt.close(fig)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def round_robin(imgs):
|
|
225
|
+
"""one image of each body part in turn (forewing, rostrum, metaleg, ...), so a plate capped at N images
|
|
226
|
+
shows every body part instead of the first N files"""
|
|
227
|
+
groups = defaultdict(list)
|
|
228
|
+
for im in imgs:
|
|
229
|
+
groups[gsp.extract_body_part(im["file_name"])].append(im)
|
|
230
|
+
order = sorted(groups)
|
|
231
|
+
out = []
|
|
232
|
+
while any(groups[g] for g in order):
|
|
233
|
+
for g in order:
|
|
234
|
+
if groups[g]:
|
|
235
|
+
out.append(groups[g].pop(0))
|
|
236
|
+
return out
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def main():
|
|
240
|
+
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
241
|
+
p.add_argument("--coco_json", required=True)
|
|
242
|
+
p.add_argument("--image_dir", required=True)
|
|
243
|
+
p.add_argument("--group_labels", required=True)
|
|
244
|
+
p.add_argument("--plates_dir", required=True)
|
|
245
|
+
p.add_argument("--species", nargs="*", default=None, help="only these species (default: all)")
|
|
246
|
+
p.add_argument("--max_images_per_species", type=int, default=8)
|
|
247
|
+
p.add_argument("--display_px", type=int, default=1400, help="longest image side as drawn")
|
|
248
|
+
p.add_argument("--outline", action="store_true", help="also draw each structure's outline, thin and white")
|
|
249
|
+
p.add_argument("--anchor", default="edge", choices=["edge", "centre", "center"],
|
|
250
|
+
help="where each leader line ends: on the structure's outline, facing its label (default), or "
|
|
251
|
+
"in the middle of the structure (its deepest point)")
|
|
252
|
+
p.add_argument("--dpi", type=int, default=200)
|
|
253
|
+
a = p.parse_args()
|
|
254
|
+
|
|
255
|
+
coco, id_to_img, id_to_cat, img_anns = gsp.load_coco(a.coco_json)
|
|
256
|
+
fn_to_species = gsp.load_group_labels(a.group_labels)
|
|
257
|
+
by_species = defaultdict(list)
|
|
258
|
+
for img in coco["images"]:
|
|
259
|
+
sp = fn_to_species.get(img["file_name"]) or fn_to_species.get(gsp._normalize_fn(img["file_name"]))
|
|
260
|
+
if sp and (not a.species or sp in a.species):
|
|
261
|
+
by_species[sp].append(img)
|
|
262
|
+
out = Path(a.plates_dir); out.mkdir(parents=True, exist_ok=True)
|
|
263
|
+
print(f"Labelled plates for {len(by_species)} species -> {out}")
|
|
264
|
+
for sp in sorted(by_species):
|
|
265
|
+
sd = out / sp; sd.mkdir(exist_ok=True)
|
|
266
|
+
done = []
|
|
267
|
+
for img in round_robin(by_species[sp])[:a.max_images_per_species]:
|
|
268
|
+
path = os.path.join(a.image_dir, img["file_name"])
|
|
269
|
+
bgr = cv2.imread(path) if os.path.exists(path) else None
|
|
270
|
+
if bgr is None:
|
|
271
|
+
continue
|
|
272
|
+
h, w = bgr.shape[:2]
|
|
273
|
+
s = min(1.0, a.display_px / max(h, w))
|
|
274
|
+
rgb = cv2.cvtColor(cv2.resize(bgr, (round(w * s), round(h * s)), interpolation=cv2.INTER_AREA),
|
|
275
|
+
cv2.COLOR_BGR2RGB)
|
|
276
|
+
structs = structures_of(img_anns.get(img["id"], []), id_to_cat, h, w, s)
|
|
277
|
+
if not structs:
|
|
278
|
+
continue
|
|
279
|
+
stem = Path(img["file_name"]).stem
|
|
280
|
+
f = sd / f"{stem}_labelled.png"
|
|
281
|
+
draw_labelled(rgb, structs, f, title=gsp.extract_body_part(img["file_name"]), outline=a.outline,
|
|
282
|
+
dpi=a.dpi, anchor="centre" if a.anchor in ("centre", "center") else "edge")
|
|
283
|
+
done.append(f)
|
|
284
|
+
if done:
|
|
285
|
+
make_plate(sp, done, sd / f"{sp}_labelled_plate", dpi=a.dpi)
|
|
286
|
+
print(f" {sp}: {len(done)} images")
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
if __name__ == "__main__":
|
|
290
|
+
main()
|