dslighting 1.3.9__py3-none-any.whl → 1.7.1__py3-none-any.whl

dslighting/registry/imet-2020-fgvc7/description_obfuscated.md

@@ -0,0 +1,29 @@
+# Task
+
+Label artwork images with significant attributes.
+
+# Metric
+
+Micro averaged F1 score.
+
+# Submission Format
+
+```
+id,attribute_ids
+00011f01965f141f5d1eea6592fa9862,0 1 2
+00014abc91ed3e4bf1663fde8136fe80,0 1 2
+0002e2054e303badc1a33463f6fb7973,0 1 2
+```
+
+# Dataset
+
+Multiple modalities can be expected and the camera sources are unknown. The photographs are often centered for objects, and in the case where the museum artifact is an entire room, the images are scenic in nature.
+
+Each object is annotated by a single annotator without a verification step. You should consider these annotations noisy.
+
+The filename of each image is its `id`.
+
+- **train.csv** gives the `attribute_ids` for the train images in **/train**
+- **/test** contains the test images. You must predict the `attribute_ids` for these images.
+- **sample_submission.csv** contains a submission in the correct format
+- **labels.csv** provides descriptions of the attributes

dslighting/registry/imet-2020-fgvc7/grade.py

@@ -0,0 +1,54 @@
+import pandas as pd
+from scipy.sparse import csr_matrix
+from sklearn.metrics import f1_score
+from sklearn.preprocessing import MultiLabelBinarizer
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def prepare_for_metric(
+    submission: pd.DataFrame, answers: pd.DataFrame
+) -> tuple[csr_matrix, csr_matrix]:
+    """Transforms the submission and answers DataFrames into the required format for grading as sparse matrices."""
+
+    # Answers checks
+    assert (
+        "id" in answers.columns and "attribute_ids" in answers.columns
+    ), "Answers DataFrame must have 'id' and 'attribute_ids' columns"
+
+    # Submission checks
+    if "id" not in submission.columns:
+        raise InvalidSubmissionError("Submission DataFrame must have an 'id' column.")
+    if "attribute_ids" not in submission.columns:
+        raise InvalidSubmissionError("Submission DataFrame must have a 'attribute_ids' column.")
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError(
+            "Submission and answers DataFrames must have the same number of rows."
+        )
+
+    # Match order
+    submission = submission.sort_values(by="id").reset_index(drop=True)
+    answers = answers.sort_values(by="id").reset_index(drop=True)
+    if (submission["id"].values != answers["id"].values).any():
+        raise InvalidSubmissionError(
+            "Submission and answers DataFrames must have matching 'id' columns."
+        )
+
+    # pandas reads empty cells as NaNs, which are float. We fill with empty string to match type
+    submission["attribute_ids"] = submission["attribute_ids"].fillna("")
+    answers["attribute_ids"] = answers["attribute_ids"].fillna("")
+
+    # Get classes
+    classes = set(answers["attribute_ids"].str.split().explode().unique())
+
+    # Convert to sparse matrices using MultiLabelBinarizer
+    mlb = MultiLabelBinarizer(classes=sorted(classes), sparse_output=True)
+    y_true = mlb.fit_transform(answers["attribute_ids"].str.split())
+    y_pred = mlb.transform(submission["attribute_ids"].str.split())
+
+    return y_true, y_pred
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    y_true, y_pred = prepare_for_metric(submission, answers)
+    return f1_score(y_true=y_true, y_pred=y_pred, average="micro")

dslighting/registry/imet-2020-fgvc7/prepare.py

@@ -0,0 +1,77 @@
+import shutil
+from pathlib import Path
+
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+
+from mlebench.utils import read_csv
+
+
+def prepare(raw: Path, public: Path, private: Path):
+
+    dev = False
+
+    # Create train, test from train split
+    old_train = read_csv(raw / "train.csv")
+    # 25958/(142119+ 25958) = 0.15 original test split
+    new_train, answers = train_test_split(old_train, test_size=0.15, random_state=0)
+
+    # Sample submission
+    sample_submission_df = answers.copy()
+    sample_submission_df["attribute_ids"] = "0 1 2"
+
+    # Checks
+    assert len(answers) + len(new_train) == len(
+        old_train
+    ), "New train and test should have the same length as the old train"
+    assert len(answers) == len(
+        sample_submission_df
+    ), "Answers and sample submission should have the same length"
+    assert (
+        answers.columns.tolist() == old_train.columns.tolist()
+    ), "Answers and old_train should have the same columns"
+    assert (
+        sample_submission_df.columns.tolist() == old_train.columns.tolist()
+    ), "Sample submission and old train should have the same columns"
+    assert (
+        new_train.columns.tolist() == old_train.columns.tolist()
+    ), "New train and old train should have the same columns"
+    assert set(new_train["id"]).isdisjoint(
+        set(answers["id"])
+    ), "New train and answers should have no overlap of rows"
+
+    # Write CSVs
+    answers.to_csv(private / "answers.csv", index=False)
+    new_train.to_csv(public / "train.csv", index=False)
+    sample_submission_df.to_csv(public / "sample_submission.csv", index=False)
+
+    # Copy over files
+    shutil.copyfile(src=raw / "labels.csv", dst=public / "labels.csv")
+
+    (public / "test").mkdir(exist_ok=True)
+    (public / "train").mkdir(exist_ok=True)
+
+    if dev:
+        # Reduce the number of images to be copied to 10% of the original, for testing
+        new_train = new_train.head(int(len(new_train) * 0.1))
+        answers = answers.head(int(len(answers) * 0.1))
+
+    for file_id in tqdm(new_train["id"], desc="Copying train images"):
+        shutil.copyfile(
+            src=raw / "train" / f"{file_id}.png",
+            dst=public / "train" / f"{file_id}.png",
+        )
+
+    for file_id in tqdm(answers["id"], desc="Copying test images"):
+        shutil.copyfile(
+            src=raw / "train" / f"{file_id}.png",
+            dst=public / "test" / f"{file_id}.png",
+        )
+
+    # File copying checks
+    assert len(list(public.glob("train/*.png"))) == len(
+        new_train
+    ), "Train images should have the same number of images as the train"
+    assert len(list(public.glob("test/*.png"))) == len(
+        answers
+    ), "Test images should have the same number of images as the test"

dslighting/registry/imet-2020-fgvc7/prepare_val.py

@@ -0,0 +1,157 @@
+import shutil
+from pathlib import Path
+
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+
+from mlebench.utils import read_csv
+
+
+def _process_split(
+    train_df,
+    test_df,
+    public_path: Path,
+    private_path: Path,
+    raw_images_path: Path,
+    dev: bool,
+):
+    """
+    Helper function to process a single data split (train/test).
+
+    It handles the creation of directories, writing of CSVs (train, answers,
+    sample_submission), and copying of image files. This ensures a consistent
+    output structure for any given split.
+    """
+    # Create output directories
+    public_path.mkdir(exist_ok=True, parents=True)
+    private_path.mkdir(exist_ok=True, parents=True)
+    (public_path / "test").mkdir(exist_ok=True)
+    (public_path / "train").mkdir(exist_ok=True)
+
+    # The test_df is the ground truth for the test set
+    answers = test_df
+
+    # Create a sample submission from the answers dataframe
+    sample_submission_df = answers.copy()
+    sample_submission_df["attribute_ids"] = "0 1 2"
+
+    # Checks
+    assert len(answers) == len(
+        sample_submission_df
+    ), "Answers and sample submission should have the same length"
+    assert (
+        sample_submission_df.columns.tolist() == answers.columns.tolist()
+    ), "Sample submission and answers should have the same columns"
+
+    # Write CSVs
+    train_df.to_csv(public_path / "train.csv", index=False)
+    answers.to_csv(private_path / "answers.csv", index=False)
+    sample_submission_df.to_csv(public_path / "sample_submission.csv", index=False)
+
+    # If in dev mode, reduce the number of images to copy for faster execution
+    if dev:
+        train_df_to_copy = train_df.head(int(len(train_df) * 0.1))
+        answers_to_copy = answers.head(int(len(answers) * 0.1))
+    else:
+        train_df_to_copy = train_df
+        answers_to_copy = answers
+
+    # Copy train images
+    for file_id in tqdm(
+        train_df_to_copy["id"], desc=f"Copying train images to {public_path.name}"
+    ):
+        shutil.copyfile(
+            src=raw_images_path / f"{file_id}.png",
+            dst=public_path / "train" / f"{file_id}.png",
+        )
+
+    # Copy test images
+    for file_id in tqdm(
+        answers_to_copy["id"], desc=f"Copying test images to {public_path.name}"
+    ):
+        shutil.copyfile(
+            src=raw_images_path / f"{file_id}.png",
+            dst=public_path / "test" / f"{file_id}.png",
+        )
+
+    # File copying checks
+    assert len(list(public_path.glob("train/*.png"))) == len(
+        train_df_to_copy
+    ), f"Train images in {public_path.name} should match dataframe"
+    assert len(list(public_path.glob("test/*.png"))) == len(
+        answers_to_copy
+    ), f"Test images in {public_path.name} should match dataframe"
+
+
+def prepare(raw: Path, public: Path, private: Path):
+
+    dev = False
+
+    # --- 1. Original Split: Create main train and test sets ---
+
+    # Create train, test from train split
+    old_train = read_csv(raw / "train.csv")
+    # 25958/(142119+ 25958) = 0.15 original test split
+    main_train, main_test = train_test_split(old_train, test_size=0.15, random_state=0)
+
+    # Checks
+    assert len(main_test) + len(main_train) == len(
+        old_train
+    ), "New train and test should have the same length as the old train"
+    assert (
+        main_test.columns.tolist() == old_train.columns.tolist()
+    ), "Answers and old_train should have the same columns"
+    assert (
+        main_train.columns.tolist() == old_train.columns.tolist()
+    ), "New train and old train should have the same columns"
+    assert set(main_train["id"]).isdisjoint(
+        set(main_test["id"])
+    ), "New train and answers should have no overlap of rows"
+
+    # Process and save the original split to `public` and `private` directories
+    _process_split(
+        train_df=main_train,
+        test_df=main_test,
+        public_path=public,
+        private_path=private,
+        raw_images_path=raw / "train",
+        dev=dev,
+    )
+
+    # Copy over metadata file for the original split
+    shutil.copyfile(src=raw / "labels.csv", dst=public / "labels.csv")
+
+    # --- 2. New Validation Split: Create validation train and test sets ---
+
+    # Define paths for the new validation split, parallel to the original ones
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+
+    # The test_size for the second split is calculated to yield a validation set
+    # of approximately the same size as the original test set.
+    # Original test set size = 0.15 * N
+    # Original train set size = 0.85 * N
+    # We want: test_size_val * (0.85 * N) ~= 0.15 * N
+    # So, test_size_val ~= 0.15 / 0.85
+    test_size_for_val_split = 0.15 / (1 - 0.15)
+
+    # Split the main_train set again to create a smaller training set and a validation set.
+    # We use the same random_state to ensure this split is also deterministic.
+    train_val, test_val = train_test_split(
+        main_train, test_size=test_size_for_val_split, random_state=0
+    )
+
+    # Process and save the validation split to `public_val` and `private_val`.
+    # The filenames and structure will be identical to the original split, but
+    # will contain the new, smaller training set and the validation set.
+    _process_split(
+        train_df=train_val,
+        test_df=test_val,
+        public_path=public_val,
+        private_path=private_val,
+        raw_images_path=raw / "train",
+        dev=dev,
+    )
+
+    # Copy over metadata file for the validation split
+    shutil.copyfile(src=raw / "labels.csv", dst=public_val / "labels.csv")

dslighting/registry/inaturalist-2019-fgvc6/checksums.yaml

@@ -0,0 +1,8 @@
+private:
+  answers.csv: 6a6ca89c88327fe0071c350e846141d3
+public:
+  kaggle_sample_submission.csv: 7cc29744e99a9f5ed8299c5e2979a2cc
+  test2019.json: c572fca9d6b042b49039d0b87e2a4c98
+  train2019.json: 558b56eb13ad126127de75e379e299f6
+  val2019.json: e5a1d2eb4fd50f2d1ae5ef571be96eb3
+zip: 9d8bb906b3e2740c10b9df89e6a234d1

dslighting/registry/inaturalist-2019-fgvc6/config.yaml

@@ -0,0 +1,16 @@
+id: inaturalist-2019-fgvc6
+name: iNaturalist 2019 at FGVC6
+competition_type: simple
+awards_medals: false
+prizes: null
+description: mlebench/competitions/inaturalist-2019-fgvc6/description.md
+
+dataset:
+  answers: inaturalist-2019-fgvc6/prepared/private/answers.csv
+  sample_submission: inaturalist-2019-fgvc6/prepared/public/kaggle_sample_submission.csv
+
+grader:
+  name: top-1-classification-error
+  grade_fn: mlebench.competitions.inaturalist-2019-fgvc6.grade:grade
+
+preparer: mlebench.competitions.inaturalist-2019-fgvc6.prepare:prepare

dslighting/registry/inaturalist-2019-fgvc6/description.md

@@ -0,0 +1,122 @@
+# Overview
+
+## Description
+
+![iNaturalist banner](https://prod-files-secure.s3.us-west-2.amazonaws.com/667f1cbf-826f-4641-a321-96054292638d/0e1cef9d-568b-4cd0-9076-500fd0c282c4/image.png)
+
+As part of the FGVC6 workshop at CVPR 2019 we are conducting the iNat Challenge 2019 large scale species classification competition, sponsored by Microsoft. It is estimated that the natural world contains several million species of plants and animals. Without expert knowledge, many of these species are extremely difficult to accurately classify due to their visual similarity. The goal  dof this competition is to push the state of the art in automatic image classification for real world data that features a large number of fine-grained categories.
+
+Previous versions of the challenge have focused on classifying large numbers of species. This year features a smaller number of highly similar categories captured in a wide variety of situations, from all over the world. In total, the iNat Challenge 2019 dataset contains 1,010 species, with a combined training and validation set of 268,243 images that have been collected and verified by multiple users from iNaturalist.
+
+Teams with top submissions, at the discretion of the workshop organizers, will be invited to present their work at the FGVC6 workshop. Participants who make a submission that beats the sample submission can fill out this [form](https://www.kaggle.com/GCP-Credits-CVPR2019) to receive $150 in Google Cloud credits.
+
+![Microsoft Logo](https://upload.wikimedia.org/wikipedia/commons/thumb/9/96/Microsoft_logo_%282012%29.svg/200px-Microsoft_logo_%282012%29.svg.png)
+
+*Kaggle is excited to partner with research groups to push forward the frontier of machine learning. Research competitions make use of Kaggle's platform and experience, but are largely organized by the research group's data science team. Any questions or concerns regarding the competition data, quality, or topic will be addressed by them.*
+
+## Evaluation
+
+We use top-1 classification error as the metric for this competition. For each image, an algorithm will produce 1 label. If the predicted label matches the ground truth label then the error for that image is 0, otherwise it is 1. The final score is the error averaged across all images.
+
+### Submission File
+
+For each image in the test set, you must predict 1 category label. However, we encourage you to predict more categories labels (sorted by confidence) so that we can analyze top-3 and top-5 performances. The csv file should contain a header and have the following format:
+
+```
+id,predicted
+268243,71 108 339 341 560
+268244,333 729 838 418 785
+268245,690 347 891 655 755
+```
+
+The `id` column corresponds to the test image id. The `predicted` column corresponds to 1 category id. The first category id will be used to compute the metric. You should have one row for each test image.
+
+## CVPR 2019
+
+This competition is part of the Fine-Grained Visual Categorization [FGVC6](https://sites.google.com/view/fgvc6/home) workshop at the Computer Vision and Pattern Recognition Conference [CVPR 2019](http://cvpr2019.thecvf.com/). A panel will review the top submissions for the competition based on the description of the methods provided. From this, a subset may be selected to present their results at the workshop. Attending the workshop is not required to participate in the competition, however only teams that are attending the workshop will be considered to present their work.
+
+There is no cash prize for this competition. Attendees presenting in person are responsible for all costs associated with travel, expenses, and fees to attend CVPR 2019.
+
+## Timeline
+
+- **June 3, 2019** - Entry deadline. You must accept the competition rules before this date in order to compete.
+- **June 3, 2019** - Team Merger deadline. This is the last day participants may join or merge teams.
+- **June 10, 2019** - Final submission deadline.
+
+All deadlines are at 11:59 PM UTC on the corresponding day unless otherwise noted. The competition organizers reserve the right to update the contest timeline if they deem it necessary.
+
+## Citation
+
+Grant Van Horn, macaodha, Maggie, Wendy Kan. (2019). iNaturalist 2019 at FGVC6. Kaggle. https://kaggle.com/competitions/inaturalist-2019-fgvc6
+
+# Data
+
+## Dataset Description
+
+### File descriptions
+
+- **train_val2019.tar.gz** - Contains the training and validation images in a directory structure following {iconic category name}/{category name}/{image id}.jpg .
+- **train2019.json** - Contains the training annotations.
+- **val2019.json** - Contains the validation annotations.
+- **test2019.tar.gz** - Contains a single directory of test images.
+- **test2019.json** - Contains test image information.
+- **kaggle_sample_submission.csv** - A sample submission file in the correct format.
+
+### Image Format
+
+All images have been saved in the JPEG format and have been resized to have a maximum dimension of 800 pixels.
+
+### Annotation Format
+
+We follow the annotation format of the [COCO dataset][2] and add additional fields. The annotations are stored in the [JSON format][3] and are organized as follows:
+
+```json
+{
+  "info" : info,
+  "images" : [image],
+  "categories" : [category],
+  "annotations" : [annotation],
+  "licenses" : [license]
+}
+
+info{
+  "year" : int,
+  "version" : str,
+  "description" : str,
+  "contributor" : str,
+  "url" : str,
+  "date_created" : datetime,
+}
+
+image{
+  "id" : int,
+  "width" : int,
+  "height" : int,
+  "file_name" : str,
+  "license" : int,
+  "rights_holder" : str
+}
+
+category{
+  "id" : int,
+  "name" : str,
+  "kingdom" : str,
+  "phylum" : str,
+  "class" : str,
+  "order" : str,
+  "family" : str,
+  "genus" : str
+}
+
+annotation{
+  "id" : int,
+  "image_id" : int,
+  "category_id" : int
+}
+
+license{
+  "id" : int,
+  "name" : str,
+  "url" : str
+}
+```

dslighting/registry/inaturalist-2019-fgvc6/description_obfuscated.md

@@ -0,0 +1,31 @@
+# Task
+
+Predict the species of plants and animals from images.
+
+# Metric
+
+Top-1 classification error. 
+
+# Submission Format
+
+For each image in the test set, you must predict 1 category label. However, you may predict more categories labels (sorted by confidence). The csv file should contain a header and have the following format:
+
+```
+id,predicted
+268243,71 108 339 341 560
+268244,333 729 838 418 785
+268245,690 347 891 655 755
+```
+
+The `id` column corresponds to the test image id. The `predicted` column corresponds to 1 category id. The first category id will be used to compute the metric. You should have one row for each test image.
+
+# Dataset
+
+- **train_val2019.tar.gz** - Contains the training and validation images in a directory structure following {iconic category name}/{category name}/{image id}.jpg .
+- **train2019.json** - Contains the training annotations.
+- **val2019.json** - Contains the validation annotations.
+- **test2019.tar.gz** - Contains a single directory of test images.
+- **test2019.json** - Contains test image information.
+
+
+We follow the annotation format of the COCO dataset and add additional fields.

dslighting/registry/inaturalist-2019-fgvc6/grade.py

@@ -0,0 +1,35 @@
+import pandas as pd
+from sklearn.metrics import accuracy_score
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def prepare_for_metric(submission: pd.DataFrame, answers: pd.DataFrame):
+    """
+    `submission` and `answers` are pd.DataFrame with "id" and "predicted" columns
+    """
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError(
+            f"Submission length {len(submission)} != answers length {len(answers)}"
+        )
+    if "id" not in submission.columns or "predicted" not in submission.columns:
+        raise InvalidSubmissionError("Submission must have 'id' and 'predicted' columns")
+
+    assert "id" in answers.columns, "Answers must have 'id' column"
+    assert "predicted" in answers.columns, "Answers must have 'predicted' column"
+
+    # Sort submission and answers by "id"
+    submission = submission.sort_values("id")
+    answers = answers.sort_values("id")
+    if (submission["id"].values != answers["id"].values).any():
+        raise InvalidSubmissionError("Submission and answers have mismatched 'id' columns")
+
+    y_true = [int(y) for y in answers["predicted"]]
+    y_pred = [int(y) if isinstance(y, int) else int(y.split()[0]) for y in submission["predicted"]]
+    return y_true, y_pred
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    y_true, y_pred = prepare_for_metric(submission, answers)
+    accuracy = accuracy_score(y_true=y_true, y_pred=y_pred)
+    return 1 - accuracy