dslighting 1.3.9__py3-none-any.whl → 1.7.1__py3-none-any.whl

dslighting/registry/stanford-covid-vaccine/grade.py

@@ -0,0 +1,65 @@
+import numpy as np
+import pandas as pd
+from sklearn.metrics import root_mean_squared_error
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError(
+            f"Expected submission to be the same length as answers, but got {len(submission)} "
+            f"instead of {len(answers)}."
+        )
+
+    to_predict = ["reactivity", "deg_Mg_pH10", "deg_Mg_50C"]
+    expected_answer_columns = ["id_seqpos"] + to_predict + ["keep"]
+
+    assert set(answers.columns).issuperset(expected_answer_columns), (
+        f"Expected answers to have columns {expected_answer_columns}, but instead it has "
+        f"columns {answers.columns}."
+    )
+
+    # The submission csv contains two columns which aren't used for scoring: `deg_pH10` and
+    # `deg_50C`. These are nonetheless still required to be in the submission as per the
+    # competition rules. See the "Sample Submission" section of the competition overview page for
+    # more information:
+    # https://www.kaggle.com/competitions/stanford-covid-vaccine/overview/evaluation
+    expected_submission_columns = ["id_seqpos"] + to_predict + ["deg_pH10", "deg_50C"]
+
+    if not set(submission.columns).issuperset(expected_submission_columns):
+        raise InvalidSubmissionError(
+            f"Expected the submission to have columns {expected_submission_columns}, but instead "
+            f"it has columns {submission.columns}."
+        )
+
+    filtered_submission = submission[expected_submission_columns]
+
+    # Sort rows by `id_seqpos` and columns alphabetically
+    sorted_submission = filtered_submission.sort_values(by="id_seqpos").sort_index(axis=1)
+    sorted_answers = answers.sort_values(by="id_seqpos").sort_index(axis=1)
+
+    for i, (actual_id, expected_id) in enumerate(
+        zip(sorted_submission["id_seqpos"], sorted_answers["id_seqpos"])
+    ):
+        if actual_id == expected_id:
+            continue
+
+        raise InvalidSubmissionError(
+            f"Expected submission to have the same `id_seqpos` as answers, but got `{actual_id}` "
+            f"instead of `{expected_id}` on row {i} of the submission."
+        )
+
+    mask = sorted_answers["keep"]
+    new_submission = sorted_submission[mask]
+    new_answers = sorted_answers[mask]
+
+    errors = []
+
+    for column in to_predict:
+        y_pred = new_submission[column]
+        y_true = new_answers[column]
+        error = root_mean_squared_error(y_true=y_true, y_pred=y_pred)
+        errors.append(error)
+
+    return np.mean(errors)

dslighting/registry/stanford-covid-vaccine/prepare.py

@@ -0,0 +1,129 @@
+from pathlib import Path
+
+import pandas as pd
+
+
+def prepare(raw: Path, public: Path, private: Path) -> None:
+    old_train = pd.read_json(raw / "train.json", lines=True)
+    old_test = pd.read_json(raw / "test.json", lines=True)
+    old_sample_submission = pd.read_csv(raw / "sample_submission.csv")
+
+    to_predict = ["reactivity", "deg_Mg_pH10", "deg_pH10", "deg_Mg_50C", "deg_50C"]
+    test_size = 0.1
+    n_test_samples = int(len(old_train) * test_size)
+
+    # only put samples that pass the SN filter in the test set, as per comp data desc
+    old_train["test"] = False
+    test_indices = (
+        old_train[old_train["SN_filter"] > 0].sample(n=n_test_samples, random_state=0).index
+    )
+    old_train.loc[test_indices, "test"] = True
+
+    new_train = old_train[~old_train["test"]].copy().drop(columns=["test"], inplace=False)
+    new_test = old_train[old_train["test"]].copy().drop(columns=["test"], inplace=False)
+    old_train = old_train.drop(columns=["test"], inplace=False)
+
+    # Create `test.csv` by exploding each list in the `reactivity` and `deg_*` columns, analogous
+    # to `pd.explode`. Only the first `seq_scored` items are scored out of a possible `seq_length`
+    # items. For each row, we keep track of whether it's scored or not with the `keep` column.
+    records = []
+
+    for _, row in new_test.iterrows():
+        n = row["seq_scored"]
+
+        assert len(row["reactivity"]) == n
+        assert len(row["deg_Mg_pH10"]) == n
+        assert len(row["deg_pH10"]) == n
+        assert len(row["deg_Mg_50C"]) == n
+        assert len(row["deg_50C"]) == n
+
+        for j in range(n):
+            records.append(
+                {
+                    "id_seqpos": f"{row['id']}_{j}",
+                    "reactivity": row["reactivity"][j],
+                    "deg_Mg_pH10": row["deg_Mg_pH10"][j],
+                    "deg_pH10": row["deg_pH10"][j],
+                    "deg_Mg_50C": row["deg_Mg_50C"][j],
+                    "deg_50C": row["deg_50C"][j],
+                    "keep": True,
+                }
+            )
+
+        k = row["seq_length"]
+
+        assert n < k
+
+        for j in range(n, k):
+            records.append(
+                {
+                    "id_seqpos": f"{row['id']}_{j}",
+                    "reactivity": 0.0,
+                    "deg_Mg_pH10": 0.0,
+                    "deg_pH10": 0.0,
+                    "deg_Mg_50C": 0.0,
+                    "deg_50C": 0.0,
+                    "keep": False,
+                }
+            )
+
+    # Write `answers.csv`
+    answers = pd.DataFrame(records)
+    answers.to_csv(private / "test.csv", index=False, float_format="%.10f")
+
+    # Write `train.json`
+    new_train["index"] = range(len(new_train))
+    new_train.to_json(public / "train.json", orient="records", lines=True)
+
+    # Write `test.json`
+    new_test_without_labels = new_test[old_test.columns].copy()
+    new_test_without_labels["index"] = range(len(new_test_without_labels))
+    new_test_without_labels.to_json(public / "test.json", orient="records", lines=True)
+
+    # Write `sample_submission.csv`
+    new_sample_submission = answers[["id_seqpos"] + to_predict].copy()
+    new_sample_submission.loc[:, to_predict] = 0.0
+    new_sample_submission.to_csv(
+        public / "sample_submission.csv", index=False, float_format="%.10f"
+    )
+
+    # Sanity checks
+    assert set(new_train.columns) == set(old_train.columns), (
+        f"Expected the columns of the new train to be the same as the old train, but got "
+        f"{set(new_train.columns)} instead of {set(old_train.columns)}."
+    )
+
+    assert set(new_test_without_labels.columns) == set(old_test.columns), (
+        f"Expected the columns of the new test to be the same as the old test, but got "
+        f"{set(new_test_without_labels.columns)} instead of {set(old_test.columns)}."
+    )
+
+    assert set(to_predict).intersection(set(new_test_without_labels.columns)) == set(), (
+        f"Expected the columns to predict aren't included in the new test, but got "
+        f"{set(to_predict) ^ set(new_test_without_labels.columns)} instead of the empty set."
+    )
+
+    assert set(new_sample_submission.columns) == set(old_sample_submission.columns), (
+        f"Expected the columns of the new sample submission to be the same as the old sample "
+        f"submission, but got {set(new_sample_submission.columns)} instead of "
+        f"{set(old_sample_submission.columns)}."
+    )
+
+    assert len(answers) == len(new_sample_submission), (
+        f"Expected the answers to have the same length as the new sample submission, but got "
+        f"{len(answers)} instead of {len(new_sample_submission)}."
+    )
+
+    # we can use [0] because all sequences have the same length
+    assert len(new_sample_submission) == (
+        len(new_test_without_labels) * new_test_without_labels["seq_length"].iloc[0]
+    ), (
+        "Expected new_sample_submission length to be equal to max seq_length * len(new_test)."
+        f"Got {len(new_sample_submission)} instead of {len(new_test_without_labels) * new_test_without_labels['seq_length']}."
+    )
+
+    assert len(new_train) + len(new_test) == len(old_train), (
+        f"Expected the length of the new train set plus the length of the new test set to be "
+        f"equal to the length of the old train set, but got {len(new_train) + len(new_test)} "
+        f"instead of {len(old_train)}."
+    )

dslighting/registry/stanford-covid-vaccine/prepare_val.py

@@ -0,0 +1,199 @@
+from pathlib import Path
+
+import pandas as pd
+
+
+def _create_competition_files(
+    train_df: pd.DataFrame,
+    test_df: pd.DataFrame,
+    old_test_columns: pd.Index,
+    old_sample_submission: pd.DataFrame,
+    public_dir: Path,
+    private_dir: Path,
+    to_predict: list,
+) -> None:
+    """
+    Helper function to generate the set of competition files for a given train/test split.
+    This function creates the public and private directories and populates them with:
+    - public/train.json
+    - public/test.json
+    - public/sample_submission.csv
+    - private/test.csv (ground truth)
+    """
+    public_dir.mkdir(parents=True, exist_ok=True)
+    private_dir.mkdir(parents=True, exist_ok=True)
+
+    # Create `test.csv` by exploding each list in the `reactivity` and `deg_*` columns, analogous
+    # to `pd.explode`. Only the first `seq_scored` items are scored out of a possible `seq_length`
+    # items. For each row, we keep track of whether it's scored or not with the `keep` column.
+    records = []
+
+    for _, row in test_df.iterrows():
+        n = row["seq_scored"]
+
+        assert len(row["reactivity"]) == n
+        assert len(row["deg_Mg_pH10"]) == n
+        assert len(row["deg_pH10"]) == n
+        assert len(row["deg_Mg_50C"]) == n
+        assert len(row["deg_50C"]) == n
+
+        for j in range(n):
+            records.append(
+                {
+                    "id_seqpos": f"{row['id']}_{j}",
+                    "reactivity": row["reactivity"][j],
+                    "deg_Mg_pH10": row["deg_Mg_pH10"][j],
+                    "deg_pH10": row["deg_pH10"][j],
+                    "deg_Mg_50C": row["deg_Mg_50C"][j],
+                    "deg_50C": row["deg_50C"][j],
+                    "keep": True,
+                }
+            )
+
+        k = row["seq_length"]
+
+        assert n < k
+
+        for j in range(n, k):
+            records.append(
+                {
+                    "id_seqpos": f"{row['id']}_{j}",
+                    "reactivity": 0.0,
+                    "deg_Mg_pH10": 0.0,
+                    "deg_pH10": 0.0,
+                    "deg_Mg_50C": 0.0,
+                    "deg_50C": 0.0,
+                    "keep": False,
+                }
+            )
+
+    # Write `answers.csv`
+    answers = pd.DataFrame(records)
+    answers.to_csv(private_dir / "test.csv", index=False, float_format="%.10f")
+
+    # Write `train.json`
+    train_df["index"] = range(len(train_df))
+    train_df.to_json(public_dir / "train.json", orient="records", lines=True)
+
+    # Write `test.json`
+    test_without_labels = test_df[old_test_columns].copy()
+    test_without_labels["index"] = range(len(test_without_labels))
+    test_without_labels.to_json(public_dir / "test.json", orient="records", lines=True)
+
+    # Write `sample_submission.csv`
+    new_sample_submission = answers[["id_seqpos"] + to_predict].copy()
+    new_sample_submission.loc[:, to_predict] = 0.0
+    new_sample_submission.to_csv(
+        public_dir / "sample_submission.csv", index=False, float_format="%.10f"
+    )
+
+    # Sanity checks
+    assert "test" not in train_df.columns
+    assert "test" not in test_df.columns
+
+    assert set(test_without_labels.columns) == set(old_test_columns), (
+        f"Expected the columns of the new test to be the same as the old test, but got "
+        f"{set(test_without_labels.columns)} instead of {set(old_test_columns)}."
+    )
+
+    assert set(to_predict).intersection(set(test_without_labels.columns)) == set(), (
+        f"Expected the columns to predict aren't included in the new test, but got "
+        f"{set(to_predict) ^ set(test_without_labels.columns)} instead of the empty set."
+    )
+
+    assert set(new_sample_submission.columns) == set(old_sample_submission.columns), (
+        f"Expected the columns of the new sample submission to be the same as the old sample "
+        f"submission, but got {set(new_sample_submission.columns)} instead of "
+        f"{set(old_sample_submission.columns)}."
+    )
+
+    assert len(answers) == len(new_sample_submission), (
+        f"Expected the answers to have the same length as the new sample submission, but got "
+        f"{len(answers)} instead of {len(new_sample_submission)}."
+    )
+
+    # we can use [0] because all sequences have the same length
+    assert len(new_sample_submission) == (
+        len(test_without_labels) * test_without_labels["seq_length"].iloc[0]
+    ), (
+        "Expected new_sample_submission length to be equal to max seq_length * len(new_test)."
+        f"Got {len(new_sample_submission)} instead of {len(test_without_labels) * test_without_labels['seq_length']}."
+    )
+
+
+def prepare(raw: Path, public: Path, private: Path) -> None:
+    old_train = pd.read_json(raw / "train.json", lines=True)
+    old_test = pd.read_json(raw / "test.json", lines=True)
+    old_sample_submission = pd.read_csv(raw / "sample_submission.csv")
+
+    to_predict = ["reactivity", "deg_Mg_pH10", "deg_pH10", "deg_Mg_50C", "deg_50C"]
+    test_size = 0.1
+    n_test_samples = int(len(old_train) * test_size)
+
+    # First split: Create the main train and test sets from the raw data
+    # only put samples that pass the SN filter in the test set, as per comp data desc
+    old_train["test"] = False
+    test_indices = (
+        old_train[old_train["SN_filter"] > 0].sample(n=n_test_samples, random_state=0).index
+    )
+    old_train.loc[test_indices, "test"] = True
+
+    new_train = old_train[~old_train["test"]].copy().drop(columns=["test"])
+    new_test = old_train[old_train["test"]].copy().drop(columns=["test"])
+
+    # Generate the original competition files. This ensures the contents of `public`
+    # and `private` directories remain identical to the original script's output.
+    _create_competition_files(
+        train_df=new_train,
+        test_df=new_test,
+        old_test_columns=old_test.columns,
+        old_sample_submission=old_sample_submission,
+        public_dir=public,
+        private_dir=private,
+        to_predict=to_predict,
+    )
+
+    # --- New: Create a validation set by splitting the training set further ---
+
+    # Define new directories for the validation set, parallel to the original ones
+    public_val = public.parent / "public_val"
+    private_val = private.parent / "private_val"
+
+    # Second split: Create a new, smaller training set and a validation set.
+    # The logic and size are replicated from the first split for consistency.
+    val_split_base = new_train.copy()
+    val_split_base["test"] = False
+    val_test_indices = (
+        val_split_base[val_split_base["SN_filter"] > 0]
+        .sample(n=n_test_samples, random_state=0)  # Use same size and random state
+        .index
+    )
+    val_split_base.loc[val_test_indices, "test"] = True
+
+    train_val = val_split_base[~val_split_base["test"]].copy().drop(columns=["test"])
+    test_val = val_split_base[val_split_base["test"]].copy().drop(columns=["test"])
+
+    # Generate the validation competition files in the new directories.
+    # The file structure and names inside public_val/private_val will mirror
+    # the original public/private directories.
+    _create_competition_files(
+        train_df=train_val,
+        test_df=test_val,
+        old_test_columns=old_test.columns,
+        old_sample_submission=old_sample_submission,
+        public_dir=public_val,
+        private_dir=private_val,
+        to_predict=to_predict,
+    )
+
+    # Final sanity checks
+    assert len(new_train) + len(new_test) == len(old_train), (
+        f"Expected the length of the new train set plus the length of the new test set to be "
+        f"equal to the length of the old train set, but got {len(new_train) + len(new_test)} "
+        f"instead of {len(old_train)}."
+    )
+    assert len(train_val) + len(test_val) == len(new_train), (
+        f"Expected the length of the validation train set plus the length of the validation test set to be "
+        f"equal to the length of the main train set, but got {len(train_val) + len(test_val)} "
+        f"instead of {len(new_train)}."
+    )

dslighting/registry/statoil-iceberg-classifier-challenge/checksums.yaml

@@ -0,0 +1,5 @@
+private:
+  sample_submission.csv: eeade5905d2b08fe7efd353de37ac58b
+  test.csv: 0b3eba8691a3cc72dcb7998baa672446
+public: {}
+zip: 20847b5ba9d0165bac850eb10d8fc234

dslighting/registry/statoil-iceberg-classifier-challenge/config.yaml

@@ -0,0 +1,22 @@
+id: statoil-iceberg-classifier-challenge
+name: Statoil/C-CORE Iceberg Classifier Challenge
+competition_type: simple
+awards_medals: true
+prizes:
+  - position: 1
+    value: 25000
+  - position: 2
+    value: 15000
+  - position: 3
+    value: 10000
+description: mlebench/competitions/statoil-iceberg-classifier-challenge/description.md
+
+dataset:
+  answers: statoil-iceberg-classifier-challenge/prepared/private/test.csv
+  sample_submission: statoil-iceberg-classifier-challenge/prepared/private/sample_submission.csv
+
+grader:
+  name: log-loss
+  grade_fn: mlebench.competitions.statoil-iceberg-classifier-challenge.grade:grade
+
+preparer: mlebench.competitions.statoil-iceberg-classifier-challenge.prepare:prepare

dslighting/registry/statoil-iceberg-classifier-challenge/description.md

@@ -0,0 +1,99 @@
+# Overview
+
+## Description
+![q7uAjTM](https://i.imgur.com/q7uAjTM.jpg)
+
+Drifting icebergs present threats to navigation and activities in areas such as offshore of the East Coast of Canada.
+
+Currently, many institutions and companies use aerial reconnaissance and shore-based support to monitor environmental conditions and assess risks from icebergs. However, in remote areas with particularly harsh weather, these methods are not feasible, and the only viable monitoring option is via satellite.
+
+[Statoil](https://www.statoil.com/), an international energy company operating worldwide, has worked closely with companies like [C-CORE](https://www.c-core.ca/). C-CORE have been using satellite data for over 30 years and have built a computer vision based surveillance system. To keep operations safe and efficient, Statoil is interested in getting a fresh new perspective on how to use machine learning to more accurately detect and discriminate against threatening icebergs as early as possible.
+
+In this competition, you’re challenged to build an algorithm that automatically identifies if a remotely sensed target is a ship or iceberg. Improvements made will help drive the costs down for maintaining safe working conditions.
+
+## Evaluation
+
+Submissions are evaluated on the [log loss](https://www.kaggle.com/wiki/LogLoss) between the predicted values and the ground truth.
+
+## Submission File
+
+For each id in the test set, you must predict the probability that the image contains an iceberg (a number between 0 and 1). The file should contain a header and have the following format:
+
+```
+id,is_iceberg
+809385f7,0.5
+7535f0cd,0.4
+3aa99a38,0.9
+etc.
+```
+
+## Prizes
+
+See the Rules Section labeled "Prizes" for the terms on receiving prize money.
+
+- 1st place - $25,000
+- 2nd place - $15,000
+- 3rd place - $10,000
+
+## Timeline
+
+- **January 16, 2018** - Entry deadline. You must accept the competition rules before this date in order to compete.
+- **January 16, 2018** - Team Merger deadline. This is the last day participants may join or merge teams.
+- **January 23, 2018** - Final submission deadline.
+
+All deadlines are at 11:59 PM UTC on the corresponding day unless otherwise noted. The competition organizers reserve the right to update the contest timeline if they deem it necessary.
+
+## Background
+
+![SO6VDSfm](https://storage.googleapis.com/kaggle-media/competitions/statoil/SO6VDSfm.jpg)
+
+The remote sensing systems used to detect icebergs are housed on satellites over 600 kilometers above the Earth. The Sentinel-1 satellite constellation is used to monitor Land and Ocean. Orbiting 14 times a day, the satellite captures images of the Earth's surface at a given location, at a given instant in time. The C-Band radar operates at a frequency that "sees" through darkness, rain, cloud and even fog. Since it emits it's own energy source it can capture images day or night.
+
+Satellite radar works in much the same way as blips on a ship or aircraft radar. It bounces a signal off an object and records the echo, then that data is translated into an image. An object will appear as a bright spot because it reflects more radar energy than its surroundings, but strong echoes can come from anything solid - land, islands, sea ice, as well as icebergs and ships. The energy reflected back to the radar is referred to as backscatter.
+
+![NM5Eg0Q](https://storage.googleapis.com/kaggle-media/competitions/statoil/NM5Eg0Q.png)
+
+![lhYaHT0](https://storage.googleapis.com/kaggle-media/competitions/statoil/lhYaHT0.png)
+
+When the radar detects a object, it can't tell an iceberg from a ship or any other solid object. The object needs to be analyzed for certain characteristics - shape, size and brightness - to find that out. The area surrounding the object, in this case ocean, can also be analyzed or modeled. Many things affect the backscatter of the ocean or background area. High winds will generate a brighter background. Conversely, low winds will generate a darker background. The Sentinel-1 satellite is a side looking radar, which means it sees the image area at an angle (incidence angle). Generally, the ocean background will be darker at a higher incidence angle. You also need to consider the radar polarization, which is how the radar transmits and receives the energy. More advanced radars like Sentinel-1, can transmit and receive in the horizontal and vertical plane. Using this, you can get what is called a dual-polarization image.
+
+For this contest you will see data with two channels: HH (transmit/receive horizontally) and HV (transmit horizontally and receive vertically). This can play an important role in the object characteristics, since objects tend to reflect energy differently. Easy classification examples are see below. These objects can be visually classified. But in an image with hundreds of objects, this is very time consuming.
+
+![8ZkRcp4](https://storage.googleapis.com/kaggle-media/competitions/statoil/8ZkRcp4.png)
+
+![M8OP2F2](https://storage.googleapis.com/kaggle-media/competitions/statoil/M8OP2F2.png)
+
+Here we see challenging objects to classify. We have given you the answer, but can you automate the answer to the question .... Is it a Ship or is it an Iceberg?
+
+![AR4NDrK](https://storage.googleapis.com/kaggle-media/competitions/statoil/AR4NDrK.png)
+
+![nXK6Vdl](https://storage.googleapis.com/kaggle-media/competitions/statoil/nXK6Vdl.png)
+
+## Citation
+
+Addison Howard, Brad Elliott, Carl Howell, chardy709@gmail.com, David Wade, Francesco S, Hallgeir, Harald W, Jan Richard Sagli, Kelley Dodge, Knut Sebastian, Mark McDonald, richh55, Wendy Kan. (2017). Statoil/C-CORE Iceberg Classifier Challenge. Kaggle. https://kaggle.com/competitions/statoil-iceberg-classifier-challenge
+# Data
+## Dataset Description
+
+In this competition, you will predict whether an image contains a ship or an iceberg. The labels are provided by human experts and geographic knowledge on the target. All the images are 75x75 images with two bands.
+
+### Data fields
+
+#### train.json, test.json
+
+The data (`train.json`, `test.json`) is presented in `json` format.
+
+The files consist of a list of images, and for each image, you can find the following fields:
+
+- **id** - the id of the image
+- **band_1, band_2** - the [flattened](https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.ndarray.flatten.html) image data. Each band has 75x75 pixel values in the list, so the list has 5625 elements. Note that these values are not the normal non-negative integers in image files since they have physical meanings - these are **float** numbers with unit being [dB](https://en.wikipedia.org/wiki/Decibel). Band 1 and Band 2 are signals characterized by radar backscatter produced from different polarizations at a particular incidence angle. The polarizations correspond to HH (transmit/receive horizontally) and HV (transmit horizontally and receive vertically). More background on the satellite imagery can be found [here](https://www.kaggle.com/c/statoil-iceberg-classifier-challenge#Background).
+- **inc_angle** - the incidence angle of which the image was taken. Note that this field has missing data marked as "na", and those images with "na" incidence angles are all in the training data to prevent leakage.
+- **is_iceberg** - the target variable, set to 1 if it is an iceberg, and 0 if it is a ship. This field only exists in `train.json`.
+
+Please note that we have included machine-generated images in the test set to prevent hand labeling. They are excluded in scoring.
+
+#### sample_submission.csv
+The submission file in the correct format:
+
+- **id** - the id of the image
+- **is_iceberg** - your predicted probability that this image is iceberg.

dslighting/registry/statoil-iceberg-classifier-challenge/description_obfuscated.md

@@ -0,0 +1,36 @@
+# Task
+
+Predict whether an image contains a ship or an iceberg.
+
+# Metric
+
+Log loss.
+
+# Submission Format
+
+For each id in the test set, you must predict the probability that the image contains an iceberg (a number between 0 and 1). The file should contain a header and have the following format:
+
+```
+id,is_iceberg
+809385f7,0.5
+7535f0cd,0.4
+3aa99a38,0.9
+etc.
+```
+
+# Dataset
+
+The labels are provided by human experts and geographic knowledge on the target. All the images are 75x75 images with two bands.
+
+The data (`train.json`, `test.json`) is presented in `json` format.
+
+The files consist of a list of images, and for each image, you can find the following fields:
+
+- **id** - the id of the image
+- **band_1, band_2** - the [flattened](https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.ndarray.flatten.html) image data. Each band has 75x75 pixel values in the list, so the list has 5625 elements. Note that these values are not the normal non-negative integers in image files since they have physical meanings - these are **float** numbers with unit being [dB](https://en.wikipedia.org/wiki/Decibel). Band 1 and Band 2 are signals characterized by radar backscatter produced from different polarizations at a particular incidence angle. The polarizations correspond to HH (transmit/receive horizontally) and HV (transmit horizontally and receive vertically).
+- **inc_angle** - the incidence angle of which the image was taken. Note that this field has missing data marked as "na", and those images with "na" incidence angles are all in the training data to prevent leakage.
+- **is_iceberg** - the target variable, set to 1 if it is an iceberg, and 0 if it is a ship. This field only exists in `train.json`.
+
+Please note that we have included machine-generated images in the test set to prevent hand labeling. They are excluded in scoring.
+
+sample_submission.csv: The submission file in the correct format:

dslighting/registry/statoil-iceberg-classifier-challenge/grade.py

@@ -0,0 +1,41 @@
+from typing import Tuple
+
+import numpy as np
+import pandas as pd
+from sklearn.metrics import log_loss
+
+from mlebench.grade_helpers import InvalidSubmissionError
+
+
+def prepare_for_metric(
+    submission: pd.DataFrame, answers: pd.DataFrame
+) -> Tuple[np.ndarray, np.ndarray]:
+    # answers checks
+    assert "id" in answers.columns, f"Answers is missing `id` column"
+    assert "is_iceberg" in answers.columns, f"Answers is missing `is_iceberg` column"
+
+    # submission checks
+    if "id" not in submission.columns:
+        raise InvalidSubmissionError(f"Submission is missing `id` column")
+    if "is_iceberg" not in submission.columns:
+        raise InvalidSubmissionError(f"Submission is missing `is_iceberg` column")
+    if set(submission["id"]) != set(answers["id"]):
+        raise InvalidSubmissionError("Submission and answers have different id's")
+    if not ((submission["is_iceberg"] >= 0) & (submission["is_iceberg"] <= 1)).all():
+        raise InvalidSubmissionError(
+            "All values in submission `is_iceberg` must be between 0 and 1."
+        )
+
+    # sort by id to ensure correct order
+    submission = submission.sort_values("id")
+    answers = answers.sort_values("id")
+
+    y_true = answers["is_iceberg"]
+    y_pred = submission["is_iceberg"]
+    return y_true, y_pred
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    y_true, y_pred = prepare_for_metric(submission, answers)
+    score = log_loss(y_true, y_pred)
+    return score