datamaestro-text 2025.7.28__tar.gz → 2025.9.11__tar.gz

{datamaestro_text-2025.7.28/src/datamaestro_text.egg-info → datamaestro_text-2025.9.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datamaestro-text
-Version: 2025.7.28
+Version: 2025.9.11
 Summary: Datamaestro module for text-related datasets
 Author-email: Benjamin Piwowarski <benjamin@piwowarski.fr>
 License: GPL-3

{datamaestro_text-2025.7.28 → datamaestro_text-2025.9.11}/src/datamaestro_text/config/com/github/aagohary/canard.py

@@ -37,7 +37,7 @@ def main(train, dev, test):
     Each dataset is an instance of :class:`datamaestro_text.data.conversation.CanardDataset`
     """
     return {
-        "train": CanardDataset(path=train),
-        "validation": CanardDataset(path=dev),
-        "test": CanardDataset(path=test),
+        "train": CanardDataset.C(path=train),
+        "validation": CanardDataset.C(path=dev),
+        "test": CanardDataset.C(path=test),
     }

{datamaestro_text-2025.7.28 → datamaestro_text-2025.9.11}/src/datamaestro_text/config/com/github/apple/ml-qrecc.py

@@ -51,6 +51,7 @@ def main(data: Path) -> Supervised[QReCCDataset, None, QReCCDataset]:
 )
 class Content(LZ4JSONLDocumentStore):
     """QReCC mentionned URLs content"""
+
     @staticmethod
     def __create_dataset__(dataset, options=None):
         ds = reference(reference=main).setup(dataset, options)
@@ -65,7 +66,7 @@ class Content(LZ4JSONLDocumentStore):
             "id",
         ).setup(dataset, options)
 
-        return Content(jsonl_path=store_path)
+        return Content.C(jsonl_path=store_path)
 
     @staticmethod
     def _documents(path: Path):

{datamaestro_text-2025.7.28 → datamaestro_text-2025.9.11}/src/datamaestro_text/config/com/microsoft/msmarco/passage.py

@@ -47,7 +47,7 @@ http://www.msmarco.org/dataset.aspx""",
 @dataset(url="https://github.com/microsoft/MSMARCO-Passage-Ranking")
 def collection_etc(data) -> Folder:
     """Documents and some more files"""
-    return Folder(path=data)
+    return Folder.C(path=data)
 
 
 @lua

{datamaestro_text-2025.7.28 → datamaestro_text-2025.9.11}/src/datamaestro_text/config/com/sentiment140.py

@@ -27,6 +27,6 @@ def english(dir):
     If you use this data, please cite Sentiment140 as your source.
     """
     return Supervised.C(
-        train=Generic(path=dir / "training.1600000.processed.noemoticon.csv"),
-        test=Generic(path=dir / "testdata.manual.2009.06.14.csv"),
+        train=Generic.C(path=dir / "training.1600000.processed.noemoticon.csv"),
+        test=Generic.C(path=dir / "testdata.manual.2009.06.14.csv"),
     )

{datamaestro_text-2025.7.28 → datamaestro_text-2025.9.11}/src/datamaestro_text/config/edu/stanford/aclimdb.py

@@ -11,6 +11,6 @@ def aclimdb(data):
     Paper http://ai.stanford.edu/~amaas/papers/wvSent_acl2011.pdf
     """
     return {
-        "train": FolderBased(path=data / "train", classes=["neg", "pos"]),
-        "test": FolderBased(path=data / "test", classes=["neg", "pos"]),
+        "train": FolderBased.C(path=data / "train", classes=["neg", "pos"]),
+        "test": FolderBased.C(path=data / "test", classes=["neg", "pos"]),
     }

{datamaestro_text-2025.7.28 → datamaestro_text-2025.9.11}/src/datamaestro_text/config/io/github/thunlp/fewrel.py

@@ -32,4 +32,4 @@ def v1(train, validation):
     Only the train and validation dataset are available. The test set is hidden
     for the leaderboard.
     """
-    return {"train": File(path=train), "validation": File(path=validation)}
+    return {"train": File.C(path=train), "validation": File.C(path=validation)}

{datamaestro_text-2025.7.28 → datamaestro_text-2025.9.11}/src/datamaestro_text/config/io/metamind/research/wikitext.py

@@ -30,9 +30,9 @@ def WikiText(data, type):
     https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/
     """
     return {
-        "train": File(path=data / ("wiki.train.%s" % type)),
-        "validation": File(path=data / ("wiki.valid.%s" % type)),
-        "test": File(path=data / ("wiki.test.%s" % type)),
+        "train": File.C(path=data / ("wiki.train.%s" % type)),
+        "validation": File.C(path=data / ("wiki.valid.%s" % type)),
+        "test": File.C(path=data / ("wiki.test.%s" % type)),
     }
 
 

{datamaestro_text-2025.7.28 → datamaestro_text-2025.9.11}/src/datamaestro_text/config/org/grouplens/movielens.py

@@ -31,10 +31,10 @@ def small(ds) -> Movielens:
     100,000 ratings and 3,600 tag applications applied to 9,000 movies by 600 users (as of 9/2018)
     """
     return {
-        "ratings": csv.Generic(path=ds / "ratings.csv", names_row=0),
-        "links": csv.Generic(path=ds / "links.csv", names_row=0),
-        "movies": csv.Generic(path=ds / "movies.csv", names_row=0),
-        "tags": csv.Generic(path=ds / "tags.csv", names_row=0),
+        "ratings": csv.Generic.C(path=ds / "ratings.csv", names_row=0),
+        "links": csv.Generic.C(path=ds / "links.csv", names_row=0),
+        "movies": csv.Generic.C(path=ds / "movies.csv", names_row=0),
+        "tags": csv.Generic.C(path=ds / "tags.csv", names_row=0),
     }
 
 
@@ -46,8 +46,8 @@ def full(ds) -> Movielens:
     27,000,000 ratings and 1,100,000 tag applications applied to 58,000 movies by 280,000 users (as of 9/2018)
     """
     return {
-        "ratings": csv.Generic(path=ds / "ratings.csv", names_row=0),
-        "links": csv.Generic(path=ds / "links.csv", names_row=0),
-        "movies": csv.Generic(path=ds / "movies.csv", names_row=0),
-        "tags": csv.Generic(path=ds / "tags.csv", names_row=0),
+        "ratings": csv.Generic.C(path=ds / "ratings.csv", names_row=0),
+        "links": csv.Generic.C(path=ds / "links.csv", names_row=0),
+        "movies": csv.Generic.C(path=ds / "movies.csv", names_row=0),
+        "tags": csv.Generic.C(path=ds / "tags.csv", names_row=0),
     }

{datamaestro_text-2025.7.28 → datamaestro_text-2025.9.11}/src/datamaestro_text/config/org/universaldependencies/french.py

@@ -34,9 +34,9 @@ def gsd(ds) -> Supervised:
     is updated since 2015 independently from the previous source.
     """
     return {
-        "train": CoNLL_U(path=ds / "fr_gsd-ud-train.conllu"),
-        "test": CoNLL_U(path=ds / "fr_gsd-ud-dev.conllu"),
-        "validation": CoNLL_U(path=ds / "fr_gsd-ud-test.conllu"),
+        "train": CoNLL_U.C(path=ds / "fr_gsd-ud-train.conllu"),
+        "test": CoNLL_U.C(path=ds / "fr_gsd-ud-dev.conllu"),
+        "validation": CoNLL_U.C(path=ds / "fr_gsd-ud-test.conllu"),
     }
 
 

{datamaestro_text-2025.7.28 → datamaestro_text-2025.9.11}/src/datamaestro_text/data/ir/__init__.py

@@ -1,6 +1,7 @@
 """Generic data types for information retrieval"""
 
 from abc import ABC, abstractmethod
+from enum import Enum
 from functools import cached_property
 import logging
 from pathlib import Path
@@ -88,6 +89,19 @@ class Documents(Base):
         ...
 
 
+class FileAccess(Enum):
+    """Defines how to access files (e.g. for document stores)"""
+
+    FILE = 0
+    """Direct file access"""
+
+    MMAP = 1
+    """Use mmap"""
+
+    MEMORY = 2
+    """Use memory"""
+
+
 class DocumentStore(Documents):
     """A document store
 
@@ -97,6 +111,10 @@ class DocumentStore(Documents):
     - return the number of documents
     """
 
+    file_access: Meta[FileAccess] = FileAccess.MMAP
+    """How to access the file collection (might not have any impact, depends on
+    the docstore)"""
+
     def docid_internal2external(self, docid: int):
         """Converts an internal collection ID (integer) to an external ID"""
         raise NotImplementedError(f"For class {self.__class__}")
@@ -327,5 +345,4 @@ class PairwiseSampleDataset(Base, ABC):
     """Datasets where each record is a query with positive and negative samples"""
 
     @abstractmethod
-    def iter(self) -> Iterator[PairwiseSample]:
-        ...
+    def iter(self) -> Iterator[PairwiseSample]: ...

{datamaestro_text-2025.7.28 → datamaestro_text-2025.9.11}/src/datamaestro_text/datasets/irds/data.py

@@ -215,7 +215,19 @@ class Documents(ir.DocumentStore, IRDSId):
 
     @cached_property
     def store(self):
-        return self.dataset.docs_store()
+        kwargs = {}
+        try:
+            # Translate to ir datasets docstore options
+            import ir_datasets.indices as ir_indices
+            file_access = {
+                ir.FileAccess.MMAP: ir_indices.FileAccess.MMAP,
+                ir.FileAccess.FILE: ir_indices.FileAccess.FILE,
+                ir.FileAccess.MEMORY: ir_indices.FileAccess.MEMORY
+            }[self.file_access]
+            kwargs = {"options": ir_indices.DocstoreOptions(file_access=file_access)}
+        except ImportError:
+            logging.warning("This version of ir-datasets cannot handle docstore options")
+        return self.dataset.docs_store(**kwargs)
 
     @cached_property
     def _docs(self):

datamaestro_text-2025.9.11/src/datamaestro_text/version.py

@@ -0,0 +1,34 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
+else:
+    VERSION_TUPLE = object
+    COMMIT_ID = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = '2025.9.11'
+__version_tuple__ = version_tuple = (2025, 9, 11)
+
+__commit_id__ = commit_id = 'gadcc9bd27'

{datamaestro_text-2025.7.28 → datamaestro_text-2025.9.11/src/datamaestro_text.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datamaestro-text
-Version: 2025.7.28
+Version: 2025.9.11
 Summary: Datamaestro module for text-related datasets
 Author-email: Benjamin Piwowarski <benjamin@piwowarski.fr>
 License: GPL-3

datamaestro_text-2025.7.28/src/datamaestro_text/version.py

@@ -1,21 +0,0 @@
-# file generated by setuptools-scm
-# don't change, don't track in version control
-
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
-
-TYPE_CHECKING = False
-if TYPE_CHECKING:
-    from typing import Tuple
-    from typing import Union
-
-    VERSION_TUPLE = Tuple[Union[int, str], ...]
-else:
-    VERSION_TUPLE = object
-
-version: str
-__version__: str
-__version_tuple__: VERSION_TUPLE
-version_tuple: VERSION_TUPLE
-
-__version__ = version = '2025.7.28'
-__version_tuple__ = version_tuple = (2025, 7, 28)