descriptron-core 2.0.4__tar.gz → 2.1.0__tar.gz

{descriptron_core-2.0.4 → descriptron_core-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: descriptron-core
-Version: 2.0.4
+Version: 2.1.0
 Summary: Evidence-tiered species descriptions, keys and audits from phenomic matrices
 Project-URL: Homepage, https://github.com/alexrvandam/Descriptron
 Project-URL: Repository, https://github.com/alexrvandam/Descriptron

descriptron_core-2.1.0/pyproject.toml

@@ -0,0 +1,82 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "descriptron-core"
+version = "2.1.0"
+description = "Evidence-tiered species descriptions, keys and audits from phenomic matrices"
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "Apache-2.0" }
+authors = [{ name = "Alex R. Van Dam" }]
+keywords = ["taxonomy", "morphometrics", "phenomics", "species description", "COCO"]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "Topic :: Scientific/Engineering :: Bio-Informatics",
+  "Topic :: Scientific/Engineering :: Image Processing",
+  "Intended Audience :: Science/Research",
+  "License :: OSI Approved :: Apache Software License",
+  "Operating System :: OS Independent",
+]
+
+# Derived from the imports of the 62 programs in this package, not from memory:
+# `python -m descriptron_core.cli --list` names them. No torch, no CUDA, no
+# compiler — this installs in about a minute on Linux, macOS and Windows, and
+# with the deposited COCO files it reproduces every analysis in the methods paper.
+dependencies = [
+  "numpy>=1.23",
+  "pandas>=1.5",
+  "scipy>=1.9",
+  "matplotlib>=3.6",
+  "scikit-learn>=1.2",
+  "scikit-image>=0.20",
+  "opencv-python-headless>=4.7",   # headless: the analysis half never opens a window
+  "pillow>=9",
+  "pycocotools>=2.0.6",
+  "seaborn>=0.12",
+  "statsmodels>=0.14",
+  "scikit-posthocs>=0.7",
+  "umap-learn>=0.5",
+  "networkx>=3",
+  "pyefd>=1.6",
+  "biopython>=1.81",
+  "PyYAML>=6",
+  "lxml>=4.9",
+  "openpyxl>=3.1",
+  "python-docx>=1.0",              # treatment and monograph .docx export
+]
+
+[project.optional-dependencies]
+# Only needed to *write* treatments. The audit, key, matrix, novelty scoring and
+# delimitation are all deterministic and need no model at all.
+llm = ["anthropic>=0.40"]
+# literature retrieval (the RAG in BioRAG): BioSysLit treatments from Zenodo and your
+# own PDFs. Keyword retrieval needs only the first two; sentence-embedding retrieval
+# (`index --embeddings`) also needs sentence-transformers + faiss-cpu, which pull in torch.
+rag = ["requests>=2.28", "pdfplumber>=0.10"]
+rag-embeddings = ["requests>=2.28", "pdfplumber>=0.10", "sentence-transformers>=2.2", "faiss-cpu>=1.7"]
+dev = ["pytest>=7", "build>=1", "twine>=5"]
+
+[project.scripts]
+descriptron        = "descriptron_core.cli:main"
+biorag-pipeline    = "descriptron_core.cli:run_pipeline"
+biorag-key         = "descriptron_core.cli:run_key_builder"
+biorag-audit       = "descriptron_core.cli:run_confabulation_checker"
+biorag-novelty     = "descriptron_core.cli:run_novelty_score"
+biorag-calibrate   = "descriptron_core.cli:run_calibrate"
+descriptron-keys   = "descriptron_core.cli:run_keys"
+
+[project.urls]
+Homepage = "https://github.com/alexrvandam/Descriptron"
+Repository = "https://github.com/alexrvandam/Descriptron"
+Issues = "https://github.com/alexrvandam/Descriptron/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/descriptron_core"]
+
+# tools/ and data/ sit inside the package directory, so `packages` already ships
+# them; a force-include here would add every file a second time and the build
+# fails with "a second file is being added to the wheel archive at the same path".
+[tool.hatch.build]
+artifacts = ["src/descriptron_core/tools/**", "src/descriptron_core/data/**"]

descriptron_core-2.1.0/src/descriptron_core/tools/coco_combiner_V13.py

@@ -0,0 +1,306 @@
+import os
+import sys
+import json
+import argparse
+import logging
+import glob
+import cv2  # Use OpenCV for reading dimensions
+
+def main():
+    # Set up logging
+    logging.basicConfig(
+        level=logging.INFO,
+        format='%(asctime)s [%(levelname)s] %(message)s',
+        handlers=[
+            logging.StreamHandler(sys.stdout)
+        ]
+    )
+
+    # Argument parsing
+    parser = argparse.ArgumentParser(description='Combine multiple COCO JSON files into one, ensuring width/height are included. Tries to append .jpg/.png/.tif if missing.')
+    parser.add_argument('--input-jsons', nargs='+', required=True,
+                        help='List of input COCO JSON files to combine. Can be space-separated or a CSV string.')
+    parser.add_argument('--output-json', required=True, help='Output path for the combined COCO JSON file.')
+    parser.add_argument('--skip-categories', nargs='*', default=[], help='List of category names to skip.')
+    parser.add_argument('--images-folders', nargs='*', default=[],
+                        help='One or more folders where input images are located. Used to look up file extensions if missing and for reading dimensions.')
+
+    args = parser.parse_args()
+
+    # Handle input JSON files (space-separated or CSV)
+    input_json_files = []
+    for item in args.input_jsons:
+        input_json_files.extend(item.split(','))  # Split CSV strings
+    input_json_files = [f.strip() for f in input_json_files if f.strip()]
+
+    output_json_path = args.output_json
+    skip_categories = args.skip_categories
+    images_folders = args.images_folders
+
+    # Check if the output filename has a '.json' extension
+    if not output_json_path.lower().endswith('.json'):
+        # Append '_combined.json' to the output filename (without extension)
+        output_json_path = os.path.splitext(output_json_path)[0] + '_combined.json'
+        logging.info(f"Output filename does not have a '.json' extension. Using '{output_json_path}' as the output file.")
+
+    # Now proceed with combining the files
+    combine_coco_json_files(input_json_files, output_json_path, skip_categories, images_folders)
+
+
+def combine_coco_json_files(input_json_files, output_json_path, skip_categories=[], images_folders=[]):
+    combined_output = {
+        "images": [],
+        "annotations": [],
+        "categories": []
+    }
+
+    # IDs for new images and annotations
+    new_image_id = 1
+    new_annotation_id = 1
+
+    # Mapping for categories: maps category name to new id
+    category_name_to_new_id = {}
+    new_category_id = 1  # Start category IDs from 1
+
+    # Process each input file
+    for json_file in input_json_files:
+        if not os.path.exists(json_file):
+            logging.error(f"File not found: {json_file}")
+            continue
+
+        logging.info(f"Processing file: {json_file}")
+        with open(json_file, 'r') as file:
+            coco_data = json.load(file)
+
+        # Handle categories: map old category IDs to new ones
+        old_category_id_to_new_id = {}
+        
+        # Check if categories exist, if not create a default one for annotations with category_id = -1
+        if not coco_data.get('categories'):
+            logging.warning(f"No categories found in {json_file}. Creating default category for keypoints.")
+            # Check if any annotations have keypoints to determine if we need a keypoint category
+            has_keypoints = any('keypoints' in ann for ann in coco_data.get('annotations', []))
+            if has_keypoints:
+                # Create a default category with keypoints support
+                default_category = {
+                    'id': -1,
+                    'name': 'keypoints',
+                    'supercategory': 'none',
+                    'keypoints': ['point_' + str(i) for i in range(1, 9)],  # 8 keypoints as seen in data
+                    'skeleton': []  # Empty skeleton, can be filled later if needed
+                }
+                coco_data['categories'] = [default_category]
+                logging.info(f"Created default keypoints category")
+        
+        for category in coco_data.get('categories', []):
+            cat_name = category['name'].strip()
+            if cat_name in skip_categories:
+                logging.info(f"Skipping category '{cat_name}' as per skip list.")
+                continue  # Skip this category
+
+            if cat_name not in category_name_to_new_id:
+                new_category = {
+                    "id": new_category_id,
+                    "name": cat_name,
+                    "supercategory": category.get('supercategory', 'none')
+                }
+                # Explicitly copy keypoints and skeleton if present
+                if 'keypoints' in category:
+                    new_category['keypoints'] = category['keypoints']
+                if 'skeleton' in category:
+                    new_category['skeleton'] = category['skeleton']
+                # === new  ADD THIS TO PRESERVE ROSETTA ONTOLOGY ===
+                if 'ontology' in category:
+                    new_category['ontology'] = category['ontology']
+                # =============================================
+                # === ADD THIS TO PRESERVE LINES ===
+                if 'lines' in category:
+                    new_category['lines'] = category['lines']
+                # ==================================
+                # end new #
+                category_name_to_new_id[cat_name] = new_category_id
+                combined_output['categories'].append(new_category)
+                logging.info(f"Assigned new category ID {new_category_id} to category '{cat_name}'.")
+                new_category_id += 1
+
+            # === V13 FIX: First-wins for old_category_id_to_new_id ===
+            # When COCO JSONs have duplicate category entries from multiple
+            # annotation rounds (e.g. id=2 → "left_elytron" then later
+            # id=2 → "pronotum"), the last write silently corrupts the mapping.
+            # First-wins preserves the original annotation intent.
+            old_id = category['id']
+            new_id = category_name_to_new_id[cat_name]
+            if old_id not in old_category_id_to_new_id:
+                old_category_id_to_new_id[old_id] = new_id
+            elif old_category_id_to_new_id[old_id] != new_id:
+                logging.warning(
+                    f"Duplicate category id={old_id} in {json_file}: "
+                    f"keeping first mapping (→new_id={old_category_id_to_new_id[old_id]}), "
+                    f"ignoring \"{cat_name}\" (→new_id={new_id})"
+                )
+
+        # FIXED: Local mapping for images - handle both STRING and INTEGER IDs
+        old_image_id_to_new_id = {}
+        if coco_data.get('images'):
+            for image in coco_data.get('images', []):
+                old_img_id = image['id']  # Can be string or int
+                local_new_id = new_image_id
+                new_image = image.copy()
+                # Ensure file_name has an extension if possible
+                if 'file_name' in new_image:
+                    new_image['file_name'] = get_image_file_name(new_image['file_name'], images_folders)
+                # Make sure we have width/height via OpenCV
+                add_width_and_height_if_missing_cv2(new_image, images_folders)
+                new_image['id'] = new_image_id
+                combined_output['images'].append(new_image)
+                new_image_id += 1
+
+                old_image_id_to_new_id[old_img_id] = local_new_id
+                logging.info(f"Mapped old image ID '{old_img_id}' (type: {type(old_img_id).__name__}) to new ID {local_new_id}")
+
+        # Handle annotations
+        for annotation in coco_data.get('annotations', []):
+            old_cat_id = annotation['category_id']
+            new_cat_id = old_category_id_to_new_id.get(old_cat_id)
+            if new_cat_id is None:
+                logging.warning(f"Annotation ID {annotation.get('id', 'unknown')} has category ID {old_cat_id} which is not mapped. Skipping.")
+                continue  # Skip annotations with categories that were skipped
+
+            old_img_id = annotation['image_id']  # Can be string or int
+            # If the image was not already added, create a new image entry.
+            if old_img_id not in old_image_id_to_new_id:
+                local_new_id = new_image_id
+                new_image_id += 1
+
+                # Handle string IDs for file_name
+                if isinstance(old_img_id, str):
+                    file_name = old_img_id
+                else:
+                    file_name = str(old_img_id)
+                file_name = get_image_file_name(file_name, images_folders)
+
+                new_image = {
+                    "id": local_new_id,
+                    "file_name": file_name
+                }
+                add_width_and_height_if_missing_cv2(new_image, images_folders)
+                combined_output['images'].append(new_image)
+
+                old_image_id_to_new_id[old_img_id] = local_new_id
+                logging.info(f"Created new image entry for old image ID '{old_img_id}' -> new ID {local_new_id}")
+
+            # Create the new annotation entry explicitly preserving keypoints
+            new_annotation = annotation.copy()
+            new_annotation['id'] = new_annotation_id
+            new_annotation_id += 1
+
+            # Update image_id and category_id to the new ones
+            new_annotation['image_id'] = old_image_id_to_new_id[old_img_id]
+            new_annotation['category_id'] = new_cat_id
+
+            # Explicitly preserve keypoints fields if present
+            if 'keypoints' in annotation:
+                new_annotation['keypoints'] = annotation['keypoints']
+                logging.info(f"Preserved {len(annotation['keypoints'])} keypoint values for annotation {new_annotation_id-1}")
+            if 'num_keypoints' in annotation:
+                new_annotation['num_keypoints'] = annotation['num_keypoints']
+
+            combined_output['annotations'].append(new_annotation)
+
+    # Debug prints
+    logging.info(f"Total Combined Categories: {len(combined_output['categories'])}")
+    logging.info(f"Total Combined Images: {len(combined_output['images'])}")
+    logging.info(f"Total Combined Annotations: {len(combined_output['annotations'])}")
+    
+    # Check keypoints in output
+    annotations_with_keypoints = sum(1 for ann in combined_output['annotations'] if 'keypoints' in ann)
+    logging.info(f"Annotations with keypoints: {annotations_with_keypoints}")
+
+    # Save the combined output
+    with open(output_json_path, 'w') as out_file:
+        json.dump(combined_output, out_file, indent=4)
+    logging.info(f"Combined COCO JSON saved to {output_json_path}")
+
+
+def get_image_file_name(file_name, images_folders):
+    """
+    If file_name has an extension, return as-is.
+    Otherwise:
+      1) Try a wildcard search: file_name + '.*' in images_folders.
+      2) If that fails, systematically try .jpg, .jpeg, .png, .tif, .tiff.
+      3) Return the discovered file name or the input as a fallback.
+    """
+    base, ext = os.path.splitext(file_name)
+    if ext:
+        # Already has extension
+        return file_name
+
+    # 1) Try wildcard search: file_name + '.*'
+    for folder in images_folders:
+        pattern = os.path.join(folder, base + ".*")
+        matches = glob.glob(pattern)
+        if matches:
+            found_file = os.path.basename(matches[0])
+            logging.info(f"Found extension for '{file_name}' in folder '{folder}': using '{found_file}'.")
+            return found_file
+    # 2) If still not found, systematically try known extensions
+    possible_exts = [".jpg", ".jpeg", ".png", ".tif", ".tiff"]
+    for folder in images_folders:
+        for e in possible_exts:
+            candidate = os.path.join(folder, base + e)
+            if os.path.exists(candidate):
+                found_file = os.path.basename(candidate)
+                logging.info(f"Found matching file for '{file_name}' in folder '{folder}': using '{found_file}'.")
+                return found_file
+
+    # Also try in current directory if images_folders fails
+    for e in possible_exts:
+        if os.path.exists(base + e):
+            found_file = base + e
+            logging.info(f"Found matching file in current dir for '{file_name}': using '{found_file}'.")
+            return found_file
+
+    # If we got here, no match
+    logging.info(f"Could NOT find any file for '{file_name}' in images_folders or current dir.")
+    return file_name
+
+
+def add_width_and_height_if_missing_cv2(image_dict, images_folders):
+    """
+    Checks if 'width' or 'height' is missing from the image_dict.
+    If so, tries reading with OpenCV (looping over folders + direct).
+    Inserts the discovered w/h, logs success/failure.
+    """
+    if 'width' not in image_dict or 'height' not in image_dict:
+        w, h = get_image_dimensions_cv2(image_dict['file_name'], images_folders)
+        if w is not None and h is not None:
+            image_dict['width'] = w
+            image_dict['height'] = h
+            logging.info(f"Width={w}, Height={h} for image '{image_dict['file_name']}' (ID {image_dict.get('id','?')})")
+        else:
+            logging.info(f"Could NOT determine width/height for image '{image_dict['file_name']}'")
+
+
+def get_image_dimensions_cv2(file_name, images_folders):
+    """
+    Attempts to load the image using OpenCV from the provided folders or the current path,
+    returning (width, height) if successful, else (None, None).
+    """
+    # (1) Check each images_folder
+    for folder in images_folders:
+        candidate = os.path.join(folder, file_name)
+        if os.path.exists(candidate):
+            img = cv2.imread(candidate)
+            if img is not None:
+                h, w = img.shape[:2]
+                return w, h
+    # (2) If not found in the folders, try directly
+    if os.path.exists(file_name):
+        img = cv2.imread(file_name)
+        if img is not None:
+            h, w = img.shape[:2]
+            return w, h
+    return None, None
+
+if __name__ == '__main__':
+    main()