hafnia 0.2.0__tar.gz → 0.2.1__tar.gz

{hafnia-0.2.0 → hafnia-0.2.1}/.github/workflows/ci_cd.yaml

@@ -21,7 +21,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4.2.2
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.31.0
+        uses: aquasecurity/trivy-action@0.32.0
         with:
           scan-type: 'fs'
           scan-ref: '.'

{hafnia-0.2.0 → hafnia-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hafnia
-Version: 0.2.0
+Version: 0.2.1
 Summary: Python SDK for communication with Hafnia platform.
 Author-email: Milestone Systems <hafniaplatform@milestone.dk>
 License-File: LICENSE
@@ -147,22 +147,20 @@ The `HafniaDataset` object provides a convenient way to interact with the datase
 creating splits, accessing samples, printing statistics, saving to and loading from disk.
 
 In essence, the `HafniaDataset` class contains `dataset.info` with dataset information 
-and `dataset.table` with annotations as a polars DataFrame
+and `dataset.samples` with annotations as a polars DataFrame
 
 ```python
 # Annotations are stored in a polars DataFrame
-print(dataset.table.head(2))
+print(dataset.samples.head(2))
 shape: (2, 14)
-┌──────────┬────────────────────────────────┬────────┬───────┬───┬───────────────────────────────┬──────────┬──────────┬───────────────────────────────┐
-│ image_id ┆ file_name                      ┆ height ┆ width ┆ … ┆ objects                       ┆ bitmasks ┆ polygons ┆ meta                          │
-│ ---      ┆ ---                            ┆ ---    ┆ ---   ┆   ┆ ---                           ┆ ---      ┆ ---      ┆ ---                           │
-│ str      ┆ str                            ┆ i64    ┆ i64   ┆   ┆ list[struct[12]]              ┆ null     ┆ null     ┆ struct[5]                     │
-╞══════════╪════════════════════════════════╪════════╪═══════╪═══╪═══════════════════════════════╪══════════╪══════════╪═══════════════════════════════╡
-│ 7800     ┆ /home/ubuntu/code/hafnia/.data ┆ 1080   ┆ 1920  ┆ … ┆ [{0.0492,0.0357,0.2083,0.23," ┆ null     ┆ null     ┆ {120.0,1.0,"2024-07-10T18:30: │
-│          ┆ …                              ┆        ┆       ┆   ┆ V…                            ┆          ┆          ┆ 0…                            │
-│ 7900     ┆ /home/ubuntu/code/hafnia/.data ┆ 1080   ┆ 1920  ┆ … ┆ [{0.146382,0.078704,0.42963,0 ┆ null     ┆ null     ┆ {120.0,1.0,"2024-07-10T18:30: │
-│          ┆ …                              ┆        ┆       ┆   ┆ .…                            ┆          ┆          ┆ 0…                            │
-└──────────┴────────────────────────────────┴────────┴───────┴───┴───────────────────────────────┴──────────┴──────────┴───────────────────────────────┘
+┌──────────────┬─────────────────────────────────┬────────┬───────┬───┬─────────────────────────────────┬──────────┬──────────┬─────────────────────────────────┐
+│ sample_index ┆ file_name                       ┆ height ┆ width ┆ … ┆ objects                         ┆ bitmasks ┆ polygons ┆ meta                            │
+│ ---          ┆ ---                             ┆ ---    ┆ ---   ┆   ┆ ---                             ┆ ---      ┆ ---      ┆ ---                             │
+│ u32          ┆ str                             ┆ i64    ┆ i64   ┆   ┆ list[struct[11]]                ┆ null     ┆ null     ┆ struct[5]                       │
+╞══════════════╪═════════════════════════════════╪════════╪═══════╪═══╪═════════════════════════════════╪══════════╪══════════╪═════════════════════════════════╡
+│ 0            ┆ /home/ubuntu/code/hafnia/.data… ┆ 1080   ┆ 1920  ┆ … ┆ [{0.0492,0.0357,0.2083,0.23,"V… ┆ null     ┆ null     ┆ {120.0,1.0,"2024-07-10T18:30:0… │
+│ 100          ┆ /home/ubuntu/code/hafnia/.data… ┆ 1080   ┆ 1920  ┆ … ┆ [{0.146382,0.078704,0.42963,0.… ┆ null     ┆ null     ┆ {120.0,1.0,"2024-07-10T18:30:0… │
+└──────────────┴─────────────────────────────────┴────────┴───────┴───┴─────────────────────────────────┴──────────┴──────────┴─────────────────────────────────┘
 ```
 
 ```python
@@ -235,16 +233,15 @@ It also contain annotations as primitive types such as `Bbox`, `Classification`.
 ```python
 rich.print(sample)
 Sample(
-    image_id='7800',
-    file_name='data/video_0026a86b-2f43-49f2-a17c-59244d10a585_1fps_mp4_frame_0
-0000.png',
+    sample_index=120,
+    file_name='/home/ubuntu/code/hafnia/.data/datasets/midwest-vehicle-detection/data/343403325f27e390.png',
     height=1080,
     width=1920,
-    split='test',
+    split='train',
     is_sample=True,
-    frame_number=None,
-    video_name=None,
-    remote_path=None,
+    collection_index=None,
+    collection_id=None,
+    remote_path='s3://mdi-production-midwest-vehicle-detection/sample/data/343403325f27e390.png',
     classifications=[
         Classification(
             class_name='Clear',

{hafnia-0.2.0 → hafnia-0.2.1}/README.md

@@ -121,22 +121,20 @@ The `HafniaDataset` object provides a convenient way to interact with the datase
 creating splits, accessing samples, printing statistics, saving to and loading from disk.
 
 In essence, the `HafniaDataset` class contains `dataset.info` with dataset information 
-and `dataset.table` with annotations as a polars DataFrame
+and `dataset.samples` with annotations as a polars DataFrame
 
 ```python
 # Annotations are stored in a polars DataFrame
-print(dataset.table.head(2))
+print(dataset.samples.head(2))
 shape: (2, 14)
-┌──────────┬────────────────────────────────┬────────┬───────┬───┬───────────────────────────────┬──────────┬──────────┬───────────────────────────────┐
-│ image_id ┆ file_name                      ┆ height ┆ width ┆ … ┆ objects                       ┆ bitmasks ┆ polygons ┆ meta                          │
-│ ---      ┆ ---                            ┆ ---    ┆ ---   ┆   ┆ ---                           ┆ ---      ┆ ---      ┆ ---                           │
-│ str      ┆ str                            ┆ i64    ┆ i64   ┆   ┆ list[struct[12]]              ┆ null     ┆ null     ┆ struct[5]                     │
-╞══════════╪════════════════════════════════╪════════╪═══════╪═══╪═══════════════════════════════╪══════════╪══════════╪═══════════════════════════════╡
-│ 7800     ┆ /home/ubuntu/code/hafnia/.data ┆ 1080   ┆ 1920  ┆ … ┆ [{0.0492,0.0357,0.2083,0.23," ┆ null     ┆ null     ┆ {120.0,1.0,"2024-07-10T18:30: │
-│          ┆ …                              ┆        ┆       ┆   ┆ V…                            ┆          ┆          ┆ 0…                            │
-│ 7900     ┆ /home/ubuntu/code/hafnia/.data ┆ 1080   ┆ 1920  ┆ … ┆ [{0.146382,0.078704,0.42963,0 ┆ null     ┆ null     ┆ {120.0,1.0,"2024-07-10T18:30: │
-│          ┆ …                              ┆        ┆       ┆   ┆ .…                            ┆          ┆          ┆ 0…                            │
-└──────────┴────────────────────────────────┴────────┴───────┴───┴───────────────────────────────┴──────────┴──────────┴───────────────────────────────┘
+┌──────────────┬─────────────────────────────────┬────────┬───────┬───┬─────────────────────────────────┬──────────┬──────────┬─────────────────────────────────┐
+│ sample_index ┆ file_name                       ┆ height ┆ width ┆ … ┆ objects                         ┆ bitmasks ┆ polygons ┆ meta                            │
+│ ---          ┆ ---                             ┆ ---    ┆ ---   ┆   ┆ ---                             ┆ ---      ┆ ---      ┆ ---                             │
+│ u32          ┆ str                             ┆ i64    ┆ i64   ┆   ┆ list[struct[11]]                ┆ null     ┆ null     ┆ struct[5]                       │
+╞══════════════╪═════════════════════════════════╪════════╪═══════╪═══╪═════════════════════════════════╪══════════╪══════════╪═════════════════════════════════╡
+│ 0            ┆ /home/ubuntu/code/hafnia/.data… ┆ 1080   ┆ 1920  ┆ … ┆ [{0.0492,0.0357,0.2083,0.23,"V… ┆ null     ┆ null     ┆ {120.0,1.0,"2024-07-10T18:30:0… │
+│ 100          ┆ /home/ubuntu/code/hafnia/.data… ┆ 1080   ┆ 1920  ┆ … ┆ [{0.146382,0.078704,0.42963,0.… ┆ null     ┆ null     ┆ {120.0,1.0,"2024-07-10T18:30:0… │
+└──────────────┴─────────────────────────────────┴────────┴───────┴───┴─────────────────────────────────┴──────────┴──────────┴─────────────────────────────────┘
 ```
 
 ```python
@@ -209,16 +207,15 @@ It also contain annotations as primitive types such as `Bbox`, `Classification`.
 ```python
 rich.print(sample)
 Sample(
-    image_id='7800',
-    file_name='data/video_0026a86b-2f43-49f2-a17c-59244d10a585_1fps_mp4_frame_0
-0000.png',
+    sample_index=120,
+    file_name='/home/ubuntu/code/hafnia/.data/datasets/midwest-vehicle-detection/data/343403325f27e390.png',
     height=1080,
     width=1920,
-    split='test',
+    split='train',
     is_sample=True,
-    frame_number=None,
-    video_name=None,
-    remote_path=None,
+    collection_index=None,
+    collection_id=None,
+    remote_path='s3://mdi-production-midwest-vehicle-detection/sample/data/343403325f27e390.png',
     classifications=[
         Classification(
             class_name='Clear',

hafnia-0.2.1/examples/example_dataset_recipe.py

@@ -0,0 +1,165 @@
+from pathlib import Path
+
+from rich import print as rprint
+
+from hafnia.data.factory import load_dataset
+from hafnia.dataset.dataset_recipe.dataset_recipe import DatasetRecipe
+from hafnia.dataset.dataset_recipe.recipe_transforms import (
+    SelectSamples,
+    Shuffle,
+    SplitsByRatios,
+)
+from hafnia.dataset.hafnia_dataset import HafniaDataset
+
+### Introducing DatasetRecipe ###
+# A DatasetRecipe is a recipe for the dataset you want to create.
+# The recipe itself is not executed - this is just a specification of the dataset you want!
+
+# A DatasetRecipe is an important concept in Hafnia as it allows you to merge multiple datasets
+# and transformations in a single recipe. This is especially useful for Training as a Service (TaaS)
+# where you need to define the dataset you want as a configuration and load it in the TaaS platform.
+
+# The 'DatasetRecipe' interface is similar to the 'HafniaDataset' interface.
+# To demonstrate, we will first create a dataset with the regular 'HafniaDataset' interface.
+# This line will get the "mnist" dataset, shuffle it, and select 20 samples.
+dataset = HafniaDataset.from_name(name="mnist").shuffle().select_samples(n_samples=20)
+
+# Now the same dataset is created using the 'DatasetRecipe' interface.
+dataset_recipe = DatasetRecipe.from_name(name="mnist").shuffle().select_samples(n_samples=20)
+dataset = dataset_recipe.build()
+# Note that the interface is similar, but to actually create the dataset you need to call `build()` on the recipe.
+
+# An important feature of a 'DatasetRecipe' is that the recipe itself - and not the dataset - can be saved as a file
+# and loaded from file. Meaning you can easily save, share, load and build the dataset later or in a different
+# environment.
+# In programming language, the recipe can be serialized to JSON and deserialized back to the original python object
+# recipe.
+# For TaaS, this is the only way to include multiple datasets during training.
+
+# This is how it looks like in practice:
+# 1) Save the dataset recipe to a file
+path_json = Path(".data/tmp/dataset_recipe.json")
+dataset_recipe.as_json_file(path_json)
+
+# 2) The recipe can be loaded from the file
+dataset_recipe_again = DatasetRecipe.from_json_file(path_json)
+
+# We can verify that the loaded recipe is the same as the original recipe.
+assert dataset_recipe_again == dataset_recipe
+
+# Additionally, you can get the python code for creating the same recipe.
+dataset_recipe.as_python_code()
+
+# Example: DatasetRecipe from Path
+dataset_recipe = DatasetRecipe.from_path(path_folder=Path(".data/datasets/mnist"))
+
+# Example: DatasetRecipe by merging multiple dataset recipes
+dataset_recipe = DatasetRecipe.from_merger(
+    recipes=[
+        DatasetRecipe.from_name(name="mnist"),
+        DatasetRecipe.from_name(name="mnist"),
+    ]
+)
+
+# Example: Recipes can be infinitely nested and combined.
+dataset_recipe = DatasetRecipe.from_merger(
+    recipes=[
+        DatasetRecipe.from_merger(
+            recipes=[
+                DatasetRecipe.from_name(name="mnist"),
+                DatasetRecipe.from_name(name="mnist"),
+            ]
+        ),
+        DatasetRecipe.from_path(path_folder=Path(".data/datasets/mnist"))
+        .select_samples(n_samples=30)
+        .splits_by_ratios(split_ratios={"train": 0.8, "val": 0.1, "test": 0.1}),
+        DatasetRecipe.from_name(name="mnist").select_samples(n_samples=20).shuffle(),
+    ]
+)
+
+# Now you can build the dataset from the recipe.
+dataset: HafniaDataset = dataset_recipe.build()
+assert len(dataset) == 450  # 2x200 + 30 + 20
+
+# Finally, you can print the dataset recipe to see what it contains.
+rprint(dataset_recipe)  # as a python object
+print(dataset_recipe.as_json_str())  # as a JSON string
+
+
+# Example: Using the 'load_dataset' function
+merged_dataset: HafniaDataset = load_dataset(dataset_recipe)
+# You get a few extra things when using `load_dataset`.
+# 1) You get the dataset directly - you don't have to call `build()` on the recipe.
+# 2) The dataset is cached if it already exists, so you don't have to
+#    download or rebuild the dataset on the second run.
+# 3) You can use an implicit form of the recipe. One example of this is that you just specify
+#    the dataset name `load_dataset("mnist")` or path `load_dataset(Path(".data/datasets/mnist"))`
+
+
+### DatasetRecipe Implicit Form ###
+# Below we demonstrate the difference between implicit and explicit forms of dataset recipes.
+# Example: Get dataset by name with implicit and explicit forms
+dataset = load_dataset("mnist")  # Implicit form
+dataset = load_dataset(DatasetRecipe.from_name(name="mnist"))  # Explicit form
+
+# Example: Get dataset from path with implicit and explicit forms:
+dataset = load_dataset(Path(".data/datasets/mnist"))  # Implicit form
+dataset = load_dataset(DatasetRecipe.from_path(path_folder=Path(".data/datasets/mnist")))  # Explicit form
+
+# Example: Merge datasets with implicit and explicit forms
+dataset = load_dataset(("mnist", "mnist"))  # Implicit form
+dataset = load_dataset(  # Explicit form
+    DatasetRecipe.from_merger(
+        recipes=[
+            DatasetRecipe.from_name(name="mnist"),
+            DatasetRecipe.from_name(name="mnist"),
+        ]
+    )
+)
+
+# Example: Define a dataset with transformations using implicit and explicit forms
+dataset = load_dataset(["mnist", SelectSamples(n_samples=20), Shuffle()])  # Implicit form
+dataset = load_dataset(DatasetRecipe.from_name(name="mnist").select_samples(n_samples=20).shuffle())  # Explicit form
+
+
+# Example: Complex nested example with implicit vs explicit forms
+# Implicit form of a complex dataset recipe
+split_ratio = {"train": 0.8, "val": 0.1, "test": 0.1}
+implicit_recipe = (
+    ("mnist", "mnist"),
+    [Path(".data/datasets/mnist"), SelectSamples(n_samples=30), SplitsByRatios(split_ratios=split_ratio)],
+    ["mnist", SelectSamples(n_samples=20), Shuffle()],
+)
+
+# Explicit form of the same complex dataset recipe
+explicit_recipe = DatasetRecipe.from_merger(
+    recipes=[
+        DatasetRecipe.from_merger(
+            recipes=[
+                DatasetRecipe.from_name(name="mnist"),
+                DatasetRecipe.from_name(name="mnist"),
+            ]
+        ),
+        DatasetRecipe.from_path(path_folder=Path(".data/datasets/mnist"))
+        .select_samples(n_samples=30)
+        .splits_by_ratios(split_ratios=split_ratio),
+        DatasetRecipe.from_name(name="mnist").select_samples(n_samples=20).shuffle(),
+    ]
+)
+
+# The implicit form uses the following rules:
+#    str: Will get a dataset by name -> In explicit form it becomes 'DatasetRecipe.from_name'
+#    Path: Will get a dataset from path -> In explicit form it becomes 'DatasetRecipe.from_path'
+#    tuple: Will merge datasets specified in the tuple -> In explicit form it becomes 'DatasetRecipe.from_merger'
+#    list: Will define a dataset followed by a list of transformations -> In explicit form it becomes chained method calls
+# Generally, we recommend using the explicit form over the implicit form when multiple datasets and transformations are involved.
+
+
+# To convert from implicit to explicit recipe form, you can use the `from_implicit_form` method.
+explicit_recipe_from_implicit = DatasetRecipe.from_implicit_form(implicit_recipe)
+rprint("Converted explicit recipe:")
+rprint(explicit_recipe_from_implicit)
+
+# Verify that the conversion produces the same result
+assert explicit_recipe_from_implicit == explicit_recipe
+rprint("✓ Conversion successful - recipes are equivalent!")

{hafnia-0.2.0 → hafnia-0.2.1}/examples/example_hafnia_dataset.py

@@ -21,7 +21,7 @@ from hafnia.dataset.primitives.polygon import Polygon
 
 # Load dataset
 path_dataset = get_dataset_path("midwest-vehicle-detection")
-dataset = HafniaDataset.read_from_path(path_dataset)
+dataset = HafniaDataset.from_path(path_dataset)
 
 # Alternatively, you can use the 'load_dataset' function
 dataset = load_dataset("midwest-vehicle-detection")
@@ -39,14 +39,15 @@ dataset.print_stats()
 # Create a dataset split for training
 dataset_train = dataset.create_split_dataset("train")
 
-# Checkout built-in transformations in 'dataset_transformation' or 'HafniaDataset'
+# Checkout built-in transformations in 'operations/dataset_transformations' or 'HafniaDataset'
 dataset_val = dataset.create_split_dataset(SplitName.VAL)  # Use 'SplitName' to avoid magic strings
 
-small_dataset = dataset.sample(n_samples=10, seed=42)  # Sample 10 samples from the dataset
+
+small_dataset = dataset.select_samples(n_samples=10, seed=42)  # Selects 10 samples from the dataset
 shuffled_dataset = dataset.shuffle(seed=42)  # Shuffle the dataset
 
 split_ratios = {SplitName.TRAIN: 0.8, SplitName.VAL: 0.1, SplitName.TEST: 0.1}
-new_dataset_splits = dataset.split_by_ratios(split_ratios)
+new_dataset_splits = dataset.splits_by_ratios(split_ratios)
 
 # Write dataset to disk
 path_tmp = Path(".data/tmp")
@@ -54,7 +55,7 @@ path_dataset = path_tmp / "hafnia_dataset"
 dataset.write(path_dataset)  # --> Check that data is human readable
 
 # Load dataset from disk
-dataset_again = HafniaDataset.read_from_path(path_dataset)
+dataset_again = HafniaDataset.from_path(path_dataset)
 
 # Access the first sample in the training split - data is stored in a dictionary
 sample_dict = dataset_train[0]

{hafnia-0.2.0 → hafnia-0.2.1}/examples/example_torchvision_dataloader.py

@@ -1,3 +1,5 @@
+from pathlib import Path
+
 import torch
 import torchvision
 import torchvision.transforms.functional
@@ -43,11 +45,14 @@ if __name__ == "__main__":
     image, targets = train_dataset[0]
     visualize_image = torch_helpers.draw_image_and_targets(image=image, targets=targets)
     pil_image = torchvision.transforms.functional.to_pil_image(visualize_image)
-    pil_image.save("visualized_labels.png")
+
+    path_tmp = Path(".data/tmp")
+    path_tmp.mkdir(parents=True, exist_ok=True)
+    pil_image.save(path_tmp / "visualized_labels.png")
 
     # Create DataLoaders - using TorchVisionCollateFn
     collate_fn = torch_helpers.TorchVisionCollateFn()
-    train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True, collate_fn=collate_fn)
+    train_loader = DataLoader(train_dataset, batch_size=5, shuffle=True, collate_fn=collate_fn)
 
     for images, targets in train_loader:
         print(f"Batch of images: {len(images)}")

{hafnia-0.2.0 → hafnia-0.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hafnia"
-version = "0.2.0"
+version = "0.2.1"
 description = "Python SDK for communication with Hafnia platform."
 readme = "README.md"
 authors = [

{hafnia-0.2.0 → hafnia-0.2.1}/src/cli/config.py

@@ -80,7 +80,7 @@ class Config:
     def __init__(self, config_path: Optional[Path] = None) -> None:
         self.config_path = self.resolve_config_path(config_path)
         self.config_path.parent.mkdir(parents=True, exist_ok=True)
-        self.config_data = self.load_config()
+        self.config_data = Config.load_config(self.config_path)
 
     def resolve_config_path(self, path: Optional[Path] = None) -> Path:
         if path:
@@ -111,12 +111,25 @@ class Config:
         endpoint = self.config.platform_url + PLATFORM_API_MAPPING[method]
         return endpoint
 
-    def load_config(self) -> ConfigFileSchema:
+    @staticmethod
+    def load_config(config_path: Path) -> ConfigFileSchema:
         """Load configuration from file."""
-        if not self.config_path.exists():
+
+        # Environment variables has higher priority than config file
+        HAFNIA_API_KEY = os.getenv("HAFNIA_API_KEY")
+        HAFNIA_PLATFORM_URL = os.getenv("HAFNIA_PLATFORM_URL")
+        if HAFNIA_API_KEY and HAFNIA_PLATFORM_URL:
+            HAFNIA_PROFILE_NAME = os.getenv("HAFNIA_PROFILE_NAME", "default").strip()
+            cfg = ConfigFileSchema(
+                active_profile=HAFNIA_PROFILE_NAME,
+                profiles={HAFNIA_PROFILE_NAME: ConfigSchema(platform_url=HAFNIA_PLATFORM_URL, api_key=HAFNIA_API_KEY)},
+            )
+            return cfg
+
+        if not config_path.exists():
             return ConfigFileSchema()
         try:
-            with open(self.config_path.as_posix(), "r") as f:
+            with open(config_path.as_posix(), "r") as f:
                 data = json.load(f)
             return ConfigFileSchema(**data)
         except json.JSONDecodeError:

hafnia-0.2.1/src/hafnia/data/factory.py

@@ -0,0 +1,23 @@
+import os
+from pathlib import Path
+from typing import Any
+
+from hafnia import utils
+from hafnia.dataset.hafnia_dataset import HafniaDataset, get_or_create_dataset_path_from_recipe
+
+
+def load_dataset(recipe: Any, force_redownload: bool = False) -> HafniaDataset:
+    """Load a dataset either from a local path or from the Hafnia platform."""
+
+    path_dataset = get_dataset_path(recipe, force_redownload=force_redownload)
+    dataset = HafniaDataset.from_path(path_dataset)
+    return dataset
+
+
+def get_dataset_path(recipe: Any, force_redownload: bool = False) -> Path:
+    if utils.is_hafnia_cloud_job():
+        return Path(os.getenv("MDI_DATASET_DIR", "/opt/ml/input/data/training"))
+
+    path_dataset = get_or_create_dataset_path_from_recipe(recipe, force_redownload=force_redownload)
+
+    return path_dataset

{hafnia-0.2.0 → hafnia-0.2.1}/src/hafnia/dataset/dataset_names.py

@@ -1,11 +1,12 @@
 from enum import Enum
 from typing import List
 
+FILENAME_RECIPE_JSON = "recipe.json"
 FILENAME_DATASET_INFO = "dataset_info.json"
 FILENAME_ANNOTATIONS_JSONL = "annotations.jsonl"
 FILENAME_ANNOTATIONS_PARQUET = "annotations.parquet"
 
-DATASET_FILENAMES = [
+DATASET_FILENAMES_REQUIRED = [
     FILENAME_DATASET_INFO,
     FILENAME_ANNOTATIONS_JSONL,
     FILENAME_ANNOTATIONS_PARQUET,