dslighting 1.3.9__py3-none-any.whl → 1.7.6__py3-none-any.whl

dslighting/registry/herbarium-2021-fgvc8/prepare_val.py

@@ -0,0 +1,222 @@
+import json
+import random
+import shutil
+from pathlib import Path
+from typing import Dict, List
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+
+from mlebench.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+def _split_data_by_category(
+    data_by_category: Dict[int, List[dict]],
+) -> (Dict[int, List[dict]], Dict[int, List[dict]]):
+    """
+    Splits data within each category into train and test sets.
+
+    This function replicates the original script's splitting logic to ensure
+    consistency.
+    """
+    train_split_by_category = {}
+    test_split_by_category = {}
+
+    for category_id, annotations_images in data_by_category.items():
+        test_size = 0.2
+        n_samples = len(annotations_images)
+        if n_samples == 1:
+            train_annotations_images = annotations_images
+            test_annotations_images = []
+        elif n_samples < 5:
+            test_size = max(1, int(n_samples * test_size))
+            train_annotations_images = annotations_images[:-test_size]
+            test_annotations_images = annotations_images[-test_size:]
+        else:
+            train_annotations_images, test_annotations_images = train_test_split(
+                annotations_images, test_size=test_size, random_state=0
+            )
+        train_split_by_category[category_id] = train_annotations_images
+        test_split_by_category[category_id] = test_annotations_images
+    return train_split_by_category, test_split_by_category
+
+
+def _process_and_save_split(
+    train_data_by_cat: Dict[int, List[dict]],
+    test_data_by_cat: Dict[int, List[dict]],
+    base_metadata: dict,
+    public_dir: Path,
+    private_dir: Path,
+    raw_data_path: Path,
+    dev_mode: bool,
+    dev_count: int,
+):
+    """
+    Processes and saves a single train/test split to the specified directories.
+
+    This function handles:
+    - Creating training set metadata and copying images.
+    - Creating test set metadata and copying/renaming images.
+    - Creating private ground-truth answers.
+    - Creating a public sample submission file.
+    """
+    # Create required directories
+    public_dir.mkdir(exist_ok=True, parents=True)
+    private_dir.mkdir(exist_ok=True, parents=True)
+    (public_dir / "train/images").mkdir(exist_ok=True, parents=True)
+    (public_dir / "test/images").mkdir(exist_ok=True, parents=True)
+
+    # Process train set
+    new_train_metadata = base_metadata.copy()
+    new_train_metadata.update({"annotations": [], "images": []})
+    train_sample_count = sum(len(v) for v in train_data_by_cat.values())
+
+    with tqdm(
+        desc=f"Creating train set for {public_dir.name}",
+        total=train_sample_count,
+    ) as pbar:
+        for category_id, annotations_images in train_data_by_cat.items():
+            category_subdir = f"{category_id // 100:03d}/{category_id % 100:02d}"
+            (public_dir / "train/images" / category_subdir).mkdir(exist_ok=True, parents=True)
+            for idx, annotation_image in enumerate(annotations_images):
+                new_train_metadata["annotations"].append(annotation_image["annotation"].copy())
+                new_train_metadata["images"].append(annotation_image["image"].copy())
+
+                if not dev_mode or idx < dev_count:
+                    src_path = raw_data_path / "train" / annotation_image["image"]["file_name"]
+                    dst_path = public_dir / "train" / annotation_image["image"]["file_name"]
+                    shutil.copyfile(src=src_path, dst=dst_path)
+                pbar.update(1)
+
+    with open(public_dir / "train/metadata.json", "w") as f:
+        json.dump(new_train_metadata, f, indent=4, sort_keys=True)
+
+    # Process test set
+    new_test_metadata = base_metadata.copy()
+    for key_to_del in ["categories", "institutions"]:
+        if key_to_del in new_test_metadata:
+            del new_test_metadata[key_to_del]
+    new_test_metadata.update({"annotations": [], "images": []})
+
+    test_annotations_images = [
+        item for sublist in test_data_by_cat.values() for item in sublist
+    ]
+    random.Random(0).shuffle(test_annotations_images)
+
+    for idx, annotation_image in tqdm(
+        enumerate(test_annotations_images),
+        desc=f"Creating test set for {public_dir.name}",
+        total=len(test_annotations_images),
+    ):
+        new_image_id = str(idx)
+        new_file_name = f"images/{idx // 1000:03d}/{idx}.jpg"
+
+        new_annotation = annotation_image["annotation"].copy()
+        new_annotation["image_id"] = new_image_id
+        new_test_metadata["annotations"].append(new_annotation)
+
+        new_image = annotation_image["image"].copy()
+        new_image["id"] = new_image_id
+        new_image["file_name"] = new_file_name
+        new_test_metadata["images"].append(new_image)
+
+        if not dev_mode or idx < dev_count:
+            src_path = raw_data_path / "train" / annotation_image["image"]["file_name"]
+            dst_path = public_dir / "test" / new_file_name
+            dst_path.parent.mkdir(exist_ok=True, parents=True)
+            shutil.copyfile(src=src_path, dst=dst_path)
+
+    # Save public test metadata (without answers)
+    public_new_test = new_test_metadata.copy()
+    del public_new_test["annotations"]
+    with open(public_dir / "test/metadata.json", "w") as f:
+        json.dump(public_new_test, f, indent=4, sort_keys=True)
+
+    # Save private test answers
+    answers_rows = [
+        {"Id": img["id"], "Predicted": ann["category_id"]}
+        for img, ann in zip(new_test_metadata["images"], new_test_metadata["annotations"])
+    ]
+    pd.DataFrame(answers_rows).to_csv(private_dir / "answers.csv", index=False)
+
+    # Save public sample submission
+    sample_rows = [{"Id": img["id"], "Predicted": 0} for img in new_test_metadata["images"]]
+    pd.DataFrame(sample_rows).to_csv(public_dir / "sample_submission.csv", index=False)
+
+
+def prepare(raw: Path, public: Path, private: Path):
+    """
+    Splits the raw data into public and private datasets with appropriate test/train splits.
+
+    This script now performs two splits:
+    1.  raw -> train + test (saved to `public`/`private`)
+    2.  train -> train_val + test_val (saved to `public_val`/`private_val`)
+
+    The second split uses the exact same logic as the first, creating a smaller
+    dataset for validation that mirrors the structure of the main one.
+    """
+    dev_mode = False
+    dev_count = 2
+
+    # --- Start: New code for managing validation paths ---
+    # Define and create the new parallel directories for the validation set
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+    public_val.mkdir(exist_ok=True)
+    private_val.mkdir(exist_ok=True)
+    # --- End: New code for managing validation paths ---
+
+    json_path = raw / "train/metadata.json"
+    with open(json_path, "r", encoding="utf-8") as f:
+        old_train_metadata = json.load(f)
+
+    annotations_images_by_category = {}
+    for annotation, image in list(
+        zip(old_train_metadata["annotations"], old_train_metadata["images"])
+    ):
+        category_id = annotation["category_id"]
+        if category_id not in annotations_images_by_category:
+            annotations_images_by_category[category_id] = []
+        annotations_images_by_category[category_id].append(
+            {"annotation": annotation, "image": image}
+        )
+
+    # --- SPLIT 1: Create the original train and test sets ---
+    logger.info("Starting initial split: raw -> train + test")
+    train_annotations_images_by_category, test_annotations_images_by_category = (
+        _split_data_by_category(annotations_images_by_category)
+    )
+
+    _process_and_save_split(
+        train_data_by_cat=train_annotations_images_by_category,
+        test_data_by_cat=test_annotations_images_by_category,
+        base_metadata=old_train_metadata,
+        public_dir=public,
+        private_dir=private,
+        raw_data_path=raw,
+        dev_mode=dev_mode,
+        dev_count=dev_count,
+    )
+    logger.info(f"Original train/test sets saved to {public.name} and {private.name}")
+
+    # --- SPLIT 2: Create the new validation sets from the training set of SPLIT 1 ---
+    logger.info("Starting second split: train -> train_val + test_val")
+    # The input to this split is the *training data* from the first split
+    train_val_by_category, test_val_by_category = _split_data_by_category(
+        train_annotations_images_by_category
+    )
+
+    _process_and_save_split(
+        train_data_by_cat=train_val_by_category,
+        test_data_by_cat=test_val_by_category,
+        base_metadata=old_train_metadata,  # Use original for all top-level keys
+        public_dir=public_val,
+        private_dir=private_val,
+        raw_data_path=raw,
+        dev_mode=dev_mode,
+        dev_count=dev_count,
+    )
+    logger.info(f"Validation sets saved to {public_val.name} and {private_val.name}")

dslighting/registry/herbarium-2022-fgvc9/checksums.yaml

@@ -0,0 +1,7 @@
+private:
+  answers.csv: d7988cc4decb444ed99cbae59381298d
+public:
+  sample_submission.csv: c2b0d6f95036aaa177a4e14a0d627054
+  test_metadata.json: 465f92cf49fb94a793648066989fe817
+  train_metadata.json: 22394926583a35658f6dd26ea0dcc683
+zip: d99e3df24b03dcb2f07fbc4c81b76a5a

dslighting/registry/herbarium-2022-fgvc9/config.yaml

@@ -0,0 +1,16 @@
+id: herbarium-2022-fgvc9
+name: Herbarium 2022 - FGVC9
+competition_type: simple
+awards_medals: false
+prizes: null
+description: mlebench/competitions/herbarium-2022-fgvc9/description.md
+
+dataset:
+  answers: herbarium-2022-fgvc9/prepared/private/answers.csv
+  sample_submission: herbarium-2022-fgvc9/prepared/public/sample_submission.csv
+
+grader:
+  name: macro-f1-score
+  grade_fn: mlebench.competitions.herbarium-2022-fgvc9.grade:grade
+
+preparer: mlebench.competitions.herbarium-2022-fgvc9.prepare:prepare

dslighting/registry/herbarium-2022-fgvc9/description.md

@@ -0,0 +1,154 @@
+# Overview
+
+## Description
+
+[![My-Post.jpg](https://i.postimg.cc/15qZZfvt/My-Post.jpg)](https://postimg.cc/Xp4Pf7SS)
+
+*The Herbarium 2022: Flora of North America* is a part of a project of the [New York Botanical Garden](https://www.nybg.org/) funded by the [National Science Foundation](https://www.nsf.gov/awardsearch/showAward?AWD_ID=2054684&HistoricalAwards=false) to build tools to identify novel plant species around the world. The dataset strives to represent all known vascular plant taxa in North America, using images gathered from 60 different botanical institutions around the world.
+
+In botany, a 'flora' is a complete account of the plants found in a geographic region. The dichotomous keys and detailed descriptions of diagnostic morphological features contained within a flora are used by botanists to determine which names to apply to plant specimens. This year's competition dataset aims to encapsulate the flora of North America so that we can test the capability of artificial intelligence to replicate this traditional tool ---a crucial first step to harnessing AI's potential botanical applications.
+
+*The Herbarium 2022: Flora of North America* dataset comprises 1.05 M images of 15,501 vascular plants, which constitute more than 90% of the taxa documented in North America. Our dataset is constrained to include only vascular land plants (lycophytes, ferns, gymnosperms, and flowering plants).
+
+Our dataset has a long-tail distribution. The number of images per taxon is as few as seven and as many as 100 images. Although more images are available, we capped the maximum number in an attempt to ensure sufficient but manageable training data size for competition participants.
+
+## About
+
+This is an FGVC competition hosted as part of the [FGVC9](https://sites.google.com/view/fgvc9) workshop at [CVPR 2022](http://cvpr2022.thecvf.com/) and sponsored by [NYBG](https://www.nybg.org/).
+
+Details of this competition are mirrored on the [github](https://github.com/visipedia/herbarium_comp) page. Please post in the forum or open an issue if you have any questions or problems with the dataset.
+
+## Acknowledgements
+
+The images are provided by the [New York Botanical Garden](https://www.nybg.org/) and 59 other institutions around the world.\
+![herb22banner](https://i.postimg.cc/g0DJMF52/output.png)
+
+## Evaluation
+
+Submissions are evaluated using the [macro F1 score](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.f1_score.html). The F1 score is given by
+
+$F_1=2 \frac{\text { precision } \cdot \text { recall }}{\text { precision }+ \text { recall }}$
+
+where:
+
+$\begin{gathered}\text { precision }=\frac{T P}{T P+F P}, \\ \text { recall }=\frac{T P}{T P+F N} .\end{gathered}$
+
+In "macro" F1 a separate F1 score is calculated for each `species` value and then averaged. #Submission Format For each image `Id`, you should predict the corresponding image label (`category_id`) in the `Predicted` column. The submission file should have the following format:
+```
+Id,Predicted
+0,1
+1,27
+2,42
+...
+```
+
+## Timeline
+
+-   February 14, 2022 - Competition start date.
+-   May 23, 2022 - Entry deadline. You must accept the competition rules before this date in order to compete.
+-   May 23, 2022 - Team Merger deadline. This is the last day participants may join or merge teams.
+-   May 30, 2022 - Final submission deadline.
+
+All deadlines are at 11:59 PM UTC on the corresponding day unless otherwise noted. The competition organizers reserve the right to update the contest timeline if they deem it necessary.
+
+## CVPR 2022
+
+This competition is part of the Fine-Grained Visual Categorization [FGVC9](https://sites.google.com/view/fgvc9) workshop at the Computer Vision and Pattern Recognition Conference [CVPR 2022](http://cvpr2022.thecvf.com/). A panel will review the top submissions for the competition based on the description of the methods provided. From this, a subset may be invited to present their results at the workshop. Attending the workshop is not required to participate in the competition; however, only teams that are attending the workshop will be considered to present their work.
+
+There is no cash prize for this competition. PLEASE NOTE: CVPR frequently sells out early, we cannot guarantee CVPR registration after the competition's end. If you are interested in attending, please plan ahead.
+
+You can see a list of all of the FGVC9 competitions [here](https://sites.google.com/view/fgvc9/competitions?authuser=0).
+
+## Citation
+
+Brendan Hogan, damon, inversion, John Park, Riccardo de Lutio. (2022). Herbarium 2022 - FGVC9. Kaggle. https://kaggle.com/competitions/herbarium-2022-fgvc9
+
+# Dataset Description
+
+## Data Overview
+
+The training and test sets contain images of herbarium specimens from 15,501 species of vascular plants. Each image contains exactly one specimen. The text labels on the specimen images have been blurred to remove category information in the image.
+
+The data has been approximately split 80%/20% for training/test. Each category has at least 1 instance in both the training and test datasets. Note that the test set distribution is slightly different from the training set distribution. The training set has a number of examples representing species capped at a maximum of 80.
+
+## Dataset Details
+
+### Images
+
+Each image has different image dimensions, with a maximum of 1000 pixels in the larger dimension. These have been resized from the original image resolution. All images are in JPEG format.
+
+### Hierarchical Structure of Classes `category_id`
+
+In addition to the images, we also include a hierarchical taxonomic structure of `category_id`. The `categories` in the `training_metadata.json` contain three levels of hierarchical structure, `family` - `genus` - `species`, from the highest rank to the lowest rank. One can think about this as a directed graph, where families are the root nodes and the species are the leaf nodes. Please note that `species` is only unique under its parent node `genus`, which means that we can find multiple categories with the same `species` name under different `genus` names. This is due to the taxonomic nature that plants are named after, and the `genus`-`species` pair is always unique in our data.
+
+### Phylogenetic Distances Among Genera
+
+What makes this year's data unique is that we include a set of pairwise phylogenetic distances among genera, so that one could test if the difference in morphological features of plant taxa well correspond to their taxonomic distances. You can find the distance data with key name `distances` from `train_metadata.json`.
+
+### Dataset Format
+
+This dataset uses the [COCO dataset format](https://cocodataset.org/#format-data) with additional annotation fields. In addition to the species category labels, we also provide supercategory information.
+
+The training set metadata (`train_metadata.json`) and test set metadata (`test_metadata.json`) are JSON files in the format below. Naturally, the test set metadata file omits the `annotations`, `categories`, and other elements.
+
+```json
+{
+  "annotations" : [annotation],
+  "categories" : [category],
+  "genera" : [genus]
+  "images" : [image],
+  "distances" : [distance],
+  "licenses" : [license],
+  "institutions" : [institution]
+}
+
+annotation {
+  "image_id" : int,
+  "category_id" : int,
+  "genus_id" : int,
+  "institution_id" : int
+}
+
+image {
+  "image_id" : int,
+  "file_name" : str,
+  "license" : int
+}
+
+category {
+  "category_id" : int,
+  "scientificName" : str,
+  # We also provide a super-category for each species.
+  "authors" : str, # correspond to 'authors' field in the wcvp
+  "family" : str, # correspond to 'family' field in the wcvp
+  "genus" : str, # correspond to 'genus' field in the wcvp
+  "species" : str, # correspond to 'species' field in the wcvp
+}
+
+genera {
+  "genus_id" : int,
+  "genus" : str
+}
+
+distance {
+  # We provide the pairwise evolutionary distance between categories (genus_id0 < genus_id1).
+  "genus_id_x" : int,
+  "genus_id_y" : int,
+  "distance" : float
+}
+
+institution {
+  "institution_id" : int
+  "collectionCode" : str
+}
+
+license {
+  "id" : int,
+  "name" : str,
+  "url" : str
+}
+```
+
+The training set images are organized in subfolders `h22-train/images/<subfolder1>/<subfolder2>/<image_id>.jpg`, where `<subfolder1>` and `<subfolder2>` comes from the first three and the last two digits of the image_id. Image_id is a result of combination between `<category_id>` and unique numbers that differentiates images within plant taxa. Please be mindful that `category_id`s are unique, but not complete. Taxa are originally numbered from 1 to 15505, but the competition data has 15501 taxa because we lost four taxa during data cleaning process.
+
+The test set images are organized in subfolders `test/images/<subfolder>/<image id>.jpg`, where `<subfolder>` corresponds to the integer division of the `image_id` by 1000. For example, a test image with and `image_id` of `8005`, can be found at `h22-test/images/008/test-008005.jpg`

dslighting/registry/herbarium-2022-fgvc9/description_obfuscated.md

@@ -0,0 +1,28 @@
+# Task
+
+Categorise plant images.
+
+# Metric
+
+Macro F1 score. In "macro" F1 a separate F1 score is calculated for each `species` value and then averaged. 
+
+# Submission Format 
+
+For each image `Id`, you should predict the corresponding image label (`category_id`) in the `Predicted` column. The submission file should have the following format:
+```
+Id,Predicted
+0,1
+1,27
+2,42
+...
+```
+
+# Dataset
+
+In addition to the images, we also include a hierarchical taxonomic structure of `category_id`. The `categories` in the `training_metadata.json` contain three levels of hierarchical structure, `family` - `genus` - `species`, from the highest rank to the lowest rank. One can think about this as a directed graph, where families are the root nodes and the species are the leaf nodes. Please note that `species` is only unique under its parent node `genus`, which means that we can find multiple categories with the same `species` name under different `genus` names. This is due to the taxonomic nature that plants are named after, and the `genus`-`species` pair is always unique in our data.
+
+Pairwise phylogenetic distances among genera is included in key name `distances` from `train_metadata.json`.
+
+The training set images are organized in subfolders `h22-train/images/<subfolder1>/<subfolder2>/<image_id>.jpg`, where `<subfolder1>` and `<subfolder2>` comes from the first three and the last two digits of the image_id. Image_id is a result of combination between `<category_id>` and unique numbers that differentiates images within plant taxa. Please be mindful that `category_id`s are unique, but not complete. Taxa are originally numbered from 1 to 15505, but the competition data has 15501 taxa because we lost four taxa during data cleaning process.
+
+The test set images are organized in subfolders `test/images/<subfolder>/<image id>.jpg`, where `<subfolder>` corresponds to the integer division of the `image_id` by 1000. For example, a test image with and `image_id` of `8005`, can be found at `h22-test/images/008/test-008005.jpg`

dslighting/registry/herbarium-2022-fgvc9/grade.py

@@ -0,0 +1,31 @@
+import pandas as pd
+from sklearn.metrics import f1_score
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def prepare_for_metric(submission: pd.DataFrame, answers: pd.DataFrame):
+    """
+    `submission` and `answers` are pd.DataFrame with "Id" and "Predicted" columns
+    """
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError(
+            f"Submission length {len(submission)} != answers length {len(answers)}"
+        )
+    if "Id" not in submission.columns or "Predicted" not in submission.columns:
+        raise InvalidSubmissionError("Submission must have 'Id' and 'Predicted' columns")
+
+    # Sort submission and answers by "Id"
+    submission = submission.sort_values("Id")
+    answers = answers.sort_values("Id")
+    if (submission["Id"].values != answers["Id"].values).any():
+        raise InvalidSubmissionError("Submission and answers have mismatched 'Id' columns")
+
+    y_true = [int(y) for y in answers["Predicted"]]
+    y_pred = [int(y) for y in submission["Predicted"]]
+    return y_true, y_pred
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    y_true, y_pred = prepare_for_metric(submission, answers)
+    return f1_score(y_true=y_true, y_pred=y_pred, average="macro")