datamaestro-text 2025.7.28__py3-none-any.whl → 2025.9.11__py3-none-any.whl

datamaestro_text/config/com/github/aagohary/canard.py

@@ -37,7 +37,7 @@ def main(train, dev, test):
     Each dataset is an instance of :class:`datamaestro_text.data.conversation.CanardDataset`
     """
     return {
-        "train": CanardDataset(path=train),
-        "validation": CanardDataset(path=dev),
-        "test": CanardDataset(path=test),
+        "train": CanardDataset.C(path=train),
+        "validation": CanardDataset.C(path=dev),
+        "test": CanardDataset.C(path=test),
     }

datamaestro_text/config/com/github/apple/ml-qrecc.py

@@ -51,6 +51,7 @@ def main(data: Path) -> Supervised[QReCCDataset, None, QReCCDataset]:
 )
 class Content(LZ4JSONLDocumentStore):
     """QReCC mentionned URLs content"""
+
     @staticmethod
     def __create_dataset__(dataset, options=None):
         ds = reference(reference=main).setup(dataset, options)
@@ -65,7 +66,7 @@ class Content(LZ4JSONLDocumentStore):
             "id",
         ).setup(dataset, options)
 
-        return Content(jsonl_path=store_path)
+        return Content.C(jsonl_path=store_path)
 
     @staticmethod
     def _documents(path: Path):

datamaestro_text/config/com/microsoft/msmarco/passage.py

@@ -47,7 +47,7 @@ http://www.msmarco.org/dataset.aspx""",
 @dataset(url="https://github.com/microsoft/MSMARCO-Passage-Ranking")
 def collection_etc(data) -> Folder:
     """Documents and some more files"""
-    return Folder(path=data)
+    return Folder.C(path=data)
 
 
 @lua

datamaestro_text/config/com/sentiment140.py

@@ -27,6 +27,6 @@ def english(dir):
     If you use this data, please cite Sentiment140 as your source.
     """
     return Supervised.C(
-        train=Generic(path=dir / "training.1600000.processed.noemoticon.csv"),
-        test=Generic(path=dir / "testdata.manual.2009.06.14.csv"),
+        train=Generic.C(path=dir / "training.1600000.processed.noemoticon.csv"),
+        test=Generic.C(path=dir / "testdata.manual.2009.06.14.csv"),
     )

datamaestro_text/config/edu/stanford/aclimdb.py

@@ -11,6 +11,6 @@ def aclimdb(data):
     Paper http://ai.stanford.edu/~amaas/papers/wvSent_acl2011.pdf
     """
     return {
-        "train": FolderBased(path=data / "train", classes=["neg", "pos"]),
-        "test": FolderBased(path=data / "test", classes=["neg", "pos"]),
+        "train": FolderBased.C(path=data / "train", classes=["neg", "pos"]),
+        "test": FolderBased.C(path=data / "test", classes=["neg", "pos"]),
     }

datamaestro_text/config/io/github/thunlp/fewrel.py

@@ -32,4 +32,4 @@ def v1(train, validation):
     Only the train and validation dataset are available. The test set is hidden
     for the leaderboard.
     """
-    return {"train": File(path=train), "validation": File(path=validation)}
+    return {"train": File.C(path=train), "validation": File.C(path=validation)}

datamaestro_text/config/io/metamind/research/wikitext.py

@@ -30,9 +30,9 @@ def WikiText(data, type):
     https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/
     """
     return {
-        "train": File(path=data / ("wiki.train.%s" % type)),
-        "validation": File(path=data / ("wiki.valid.%s" % type)),
-        "test": File(path=data / ("wiki.test.%s" % type)),
+        "train": File.C(path=data / ("wiki.train.%s" % type)),
+        "validation": File.C(path=data / ("wiki.valid.%s" % type)),
+        "test": File.C(path=data / ("wiki.test.%s" % type)),
     }
 
 

datamaestro_text/config/org/grouplens/movielens.py

@@ -31,10 +31,10 @@ def small(ds) -> Movielens:
     100,000 ratings and 3,600 tag applications applied to 9,000 movies by 600 users (as of 9/2018)
     """
     return {
-        "ratings": csv.Generic(path=ds / "ratings.csv", names_row=0),
-        "links": csv.Generic(path=ds / "links.csv", names_row=0),
-        "movies": csv.Generic(path=ds / "movies.csv", names_row=0),
-        "tags": csv.Generic(path=ds / "tags.csv", names_row=0),
+        "ratings": csv.Generic.C(path=ds / "ratings.csv", names_row=0),
+        "links": csv.Generic.C(path=ds / "links.csv", names_row=0),
+        "movies": csv.Generic.C(path=ds / "movies.csv", names_row=0),
+        "tags": csv.Generic.C(path=ds / "tags.csv", names_row=0),
     }
 
 
@@ -46,8 +46,8 @@ def full(ds) -> Movielens:
     27,000,000 ratings and 1,100,000 tag applications applied to 58,000 movies by 280,000 users (as of 9/2018)
     """
     return {
-        "ratings": csv.Generic(path=ds / "ratings.csv", names_row=0),
-        "links": csv.Generic(path=ds / "links.csv", names_row=0),
-        "movies": csv.Generic(path=ds / "movies.csv", names_row=0),
-        "tags": csv.Generic(path=ds / "tags.csv", names_row=0),
+        "ratings": csv.Generic.C(path=ds / "ratings.csv", names_row=0),
+        "links": csv.Generic.C(path=ds / "links.csv", names_row=0),
+        "movies": csv.Generic.C(path=ds / "movies.csv", names_row=0),
+        "tags": csv.Generic.C(path=ds / "tags.csv", names_row=0),
     }

datamaestro_text/config/org/universaldependencies/french.py

@@ -34,9 +34,9 @@ def gsd(ds) -> Supervised:
     is updated since 2015 independently from the previous source.
     """
     return {
-        "train": CoNLL_U(path=ds / "fr_gsd-ud-train.conllu"),
-        "test": CoNLL_U(path=ds / "fr_gsd-ud-dev.conllu"),
-        "validation": CoNLL_U(path=ds / "fr_gsd-ud-test.conllu"),
+        "train": CoNLL_U.C(path=ds / "fr_gsd-ud-train.conllu"),
+        "test": CoNLL_U.C(path=ds / "fr_gsd-ud-dev.conllu"),
+        "validation": CoNLL_U.C(path=ds / "fr_gsd-ud-test.conllu"),
     }
 
 

datamaestro_text/data/ir/__init__.py

@@ -1,6 +1,7 @@
 """Generic data types for information retrieval"""
 
 from abc import ABC, abstractmethod
+from enum import Enum
 from functools import cached_property
 import logging
 from pathlib import Path
@@ -88,6 +89,19 @@ class Documents(Base):
         ...
 
 
+class FileAccess(Enum):
+    """Defines how to access files (e.g. for document stores)"""
+
+    FILE = 0
+    """Direct file access"""
+
+    MMAP = 1
+    """Use mmap"""
+
+    MEMORY = 2
+    """Use memory"""
+
+
 class DocumentStore(Documents):
     """A document store
 
@@ -97,6 +111,10 @@ class DocumentStore(Documents):
     - return the number of documents
     """
 
+    file_access: Meta[FileAccess] = FileAccess.MMAP
+    """How to access the file collection (might not have any impact, depends on
+    the docstore)"""
+
     def docid_internal2external(self, docid: int):
         """Converts an internal collection ID (integer) to an external ID"""
         raise NotImplementedError(f"For class {self.__class__}")
@@ -327,5 +345,4 @@ class PairwiseSampleDataset(Base, ABC):
     """Datasets where each record is a query with positive and negative samples"""
 
     @abstractmethod
-    def iter(self) -> Iterator[PairwiseSample]:
-        ...
+    def iter(self) -> Iterator[PairwiseSample]: ...

datamaestro_text/datasets/irds/data.py

@@ -215,7 +215,19 @@ class Documents(ir.DocumentStore, IRDSId):
 
     @cached_property
     def store(self):
-        return self.dataset.docs_store()
+        kwargs = {}
+        try:
+            # Translate to ir datasets docstore options
+            import ir_datasets.indices as ir_indices
+            file_access = {
+                ir.FileAccess.MMAP: ir_indices.FileAccess.MMAP,
+                ir.FileAccess.FILE: ir_indices.FileAccess.FILE,
+                ir.FileAccess.MEMORY: ir_indices.FileAccess.MEMORY
+            }[self.file_access]
+            kwargs = {"options": ir_indices.DocstoreOptions(file_access=file_access)}
+        except ImportError:
+            logging.warning("This version of ir-datasets cannot handle docstore options")
+        return self.dataset.docs_store(**kwargs)
 
     @cached_property
     def _docs(self):

datamaestro_text/version.py

@@ -1,7 +1,14 @@
 # file generated by setuptools-scm
 # don't change, don't track in version control
 
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
     from typing import Union
 
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
 
-__version__ = version = '2025.7.28'
-__version_tuple__ = version_tuple = (2025, 7, 28)
+__version__ = version = '2025.9.11'
+__version_tuple__ = version_tuple = (2025, 9, 11)
+
+__commit_id__ = commit_id = 'gadcc9bd27'

{datamaestro_text-2025.7.28.dist-info → datamaestro_text-2025.9.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datamaestro-text
-Version: 2025.7.28
+Version: 2025.9.11
 Summary: Datamaestro module for text-related datasets
 Author-email: Benjamin Piwowarski <benjamin@piwowarski.fr>
 License: GPL-3

{datamaestro_text-2025.7.28.dist-info → datamaestro_text-2025.9.11.dist-info}/RECORD

@@ -1,21 +1,21 @@
 datamaestro_text/__init__.py,sha256=hU8jZpkXl3F74qIfqnJl7v4nJ9YxfoR7IpJpUREFNRI,248
-datamaestro_text/version.py,sha256=rJQHFC3G5XDG0rUPZ6r1msOA_XCbSY-qMukJgu2nA1M,519
+datamaestro_text/version.py,sha256=aHTcqRU_FAq8moUGgwqhCMrhMi8VBbk38TX-uMF8p20,720
 datamaestro_text/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamaestro_text/config/ai/quac.yaml,sha256=h1D7UJo2z1nZ_9MXpDHuQNJG9Ma2oayUdJV6lyumAIg,1103
 datamaestro_text/config/com/oscar-corpus.py,sha256=dDfdy_uhG6esRQQzUOOORwK8KvEq4c4hZbWt22pv-zY,733
-datamaestro_text/config/com/sentiment140.py,sha256=WKKLaD7psbj9fIaTBHDTzOZanO2mukaB1g7aeTN1jdU,1289
+datamaestro_text/config/com/sentiment140.py,sha256=itfBEgcOniECXKOw8I2dhzyS9LOMsltMLfKK6NGRpVY,1293
 datamaestro_text/config/com/fastml/goodbooks-10k.yaml,sha256=5ZABxUnBFs2ZnCXtBH8YoBiPb3SocRRdH1DLSfVWF-Y,1172
 datamaestro_text/config/com/github/ikat.py,sha256=DCayX-t2OBeW5bOJvRxoQgIH3vy-__mYzdmVcnayAkk,4230
-datamaestro_text/config/com/github/aagohary/canard.py,sha256=DREIKU2FFrfwibjU48Y1AsKTUzhwxmD4Kuykd4bDIus,1762
-datamaestro_text/config/com/github/apple/ml-qrecc.py,sha256=oYI0SUxEYzGoL2IbRrnze2cQuWwENwNk4ID9NQuI2Tw,3061
+datamaestro_text/config/com/github/aagohary/canard.py,sha256=5fLwCLNBGM_7--naTCDayAMYLvK3yTD8auaEf-dqrb4,1768
+datamaestro_text/config/com/github/apple/ml-qrecc.py,sha256=zP3w7A9KSvJVCo44OaB1az1pDKWxE6qXS4qFm3hqg3Y,3064
 datamaestro_text/config/com/github/prdwb/orconvqa.py,sha256=MFJYaxJoqJQ6kMDHa2PIE0zoxYTA8Kyl26-vzFoMML0,3032
 datamaestro_text/config/com/github/soskek/bookcorpus.yaml,sha256=qJKs35yeEIilEMgNvU3OEqMp1TSn7mDM2T-uYyA7kTU,1607
 datamaestro_text/config/com/microsoft/wikiqa.yaml,sha256=U7rU-W2Xz1MYv2YXT4jCTj5DsHyM0VssbQPNa3EISaM,540
-datamaestro_text/config/com/microsoft/msmarco/passage.py,sha256=tVldwVh0pQpXXifqcMNEN9cLO1HXkXoEhToSazSx5RE,11643
+datamaestro_text/config/com/microsoft/msmarco/passage.py,sha256=hN2KOdi6ToHlodozqsYAOtxaqiUGkGGtRtb3RFSgnEU,11645
 datamaestro_text/config/com/smashwords/bookcorpus.py,sha256=hCFjZg9t1AHET05qU31uatiAOD5EEzzWG9y_W90jcOE,861
 datamaestro_text/config/edu/cornell/nlvr.yaml,sha256=9Yk5VZMncSmrP7JNuGXqExksgX5nQ_Zfnlps8hWze3Q,921
 datamaestro_text/config/edu/stanford/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamaestro_text/config/edu/stanford/aclimdb.py,sha256=lsPDxnp_rWOCpBte6pZ0_LVaC33w5mmgfGh51rcTgt8,643
+datamaestro_text/config/edu/stanford/aclimdb.py,sha256=QtriReAVsbJlxkgfJWQCZdCeJ9LswYnOR9mFrgghL9c,647
 datamaestro_text/config/edu/stanford/glove.py,sha256=bXxwiJqT8alPs5MwwxVuY0xBO8g1QWhm2L3AKVskTlA,2391
 datamaestro_text/config/edu/stanford/im2p.yaml,sha256=JoToNyEPpmwdyLFedCBot5ypyw7p9rzi12mGXJuZin0,2909
 datamaestro_text/config/edu/upenn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -32,14 +32,14 @@ datamaestro_text/config/gov/nist/trec/index.yaml,sha256=oSUhUmtukf5oXqUbJLhae8xZ
 datamaestro_text/config/gov/nist/trec/tipster.py,sha256=DirpnHpS10e27LcL7v9ksKreKVy7EgfVhyztV49VRds,5364
 datamaestro_text/config/gov/nist/trec/web.yaml,sha256=iNIJ-PCeLwpUaJByJesyCqfFJolWCts_VETACha3hfQ,563
 datamaestro_text/config/io/github/rajpurkar/squad.yaml,sha256=JchkTruEhVd0uOTGgek2tOgBL4UT3G3KVASvZcNigLg,1143
-datamaestro_text/config/io/github/thunlp/fewrel.py,sha256=LJYexk2ssZ5SqKIKLjPdybS9ju4g9fGIFy5eybtsvYQ,1224
+datamaestro_text/config/io/github/thunlp/fewrel.py,sha256=rg_qAnMrXYUZhQYxA12r_Npl0ggyfTLJQjdSCjU0QxM,1228
 datamaestro_text/config/io/metamind/research/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamaestro_text/config/io/metamind/research/wikitext.py,sha256=jWpCadMYJnX2I_Ls230lB8gwBvV51SOhABUznL8_dCA,2301
+datamaestro_text/config/io/metamind/research/wikitext.py,sha256=DjyBmG74JvuMt9RpMwuLAnxzOdByIWsk4VnXgkJp1NM,2307
 datamaestro_text/config/net/mattmahoney/enwiki.yaml,sha256=HCUn3s0AydXX3BjJ6yUXY0vGLGWSBkOCaDhQ4PA2Adg,2452
 datamaestro_text/config/org/acm/recsys/cb2014.yaml,sha256=5SAK3Am1k0HFugSSCIQN5mLPBfr1zZZAkhLrSH5pHQc,1274
 datamaestro_text/config/org/cocodataset/index.yaml,sha256=KISJChMeKwlZbSnHmRcGMsm6jbcFGVe1aA4GhP2fzqw,474
-datamaestro_text/config/org/grouplens/movielens.py,sha256=e78JoiHuwP6gbj7Q84UwPTvjtN_ehfD5AhvuUt8DiH4,1824
-datamaestro_text/config/org/universaldependencies/french.py,sha256=W_gDtfinjlw08qohX_PWvzQlacDwRFB7PeOzO33mRVU,2208
+datamaestro_text/config/org/grouplens/movielens.py,sha256=tV6OSTDdtjll1dQBCsYIlsBbtOO-MCiLles2aj0MgDA,1840
+datamaestro_text/config/org/universaldependencies/french.py,sha256=etedb3_SC-fV5Oa2rM4_smZk6t4CPiNvU4C4keUFZHY,2214
 datamaestro_text/config/uk/ac/ucl/cs/qangaroo.yaml,sha256=IBy82CDNNLjJPNPzues1EgDXu0A5WDvUFeVNSOyrIpI,1137
 datamaestro_text/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamaestro_text/data/embeddings.py,sha256=qdeZ4m6drFNkqW_4pKxJKdJHtOnDRs0P7QVJ7AO6xFE,1127
@@ -52,7 +52,7 @@ datamaestro_text/data/conversation/canard.py,sha256=FI02rPNuzSsA_uXOcgcuQn31d3Qa
 datamaestro_text/data/conversation/ikat.py,sha256=djxTv0vHLXIUPzfmwpB25fyUWJNVPqthTtDbQ6CzmQo,4363
 datamaestro_text/data/conversation/orconvqa.py,sha256=RL9cpK7QK2dsk_j2e4Th8wzGFq-G3ww_EjdM1eKh-8o,3764
 datamaestro_text/data/conversation/qrecc.py,sha256=es4GmqPtE63A7O_GARe8Zy3rQvuLEhAvUA7CfN_nMeA,2562
-datamaestro_text/data/ir/__init__.py,sha256=jHoyIWyl0beUbg1gmkwNFf1cQRawB8p3SGfa17gniGM,9442
+datamaestro_text/data/ir/__init__.py,sha256=ZRJrUeeUyD1ncMN5JINVvFJ2lDr3KsbgiiEBJkczSi0,9814
 datamaestro_text/data/ir/base.py,sha256=uwIiKn0ryK5DWUQsEegeTs6bye0uAOGl0XVm_cvV3ZU,1506
 datamaestro_text/data/ir/cord19.py,sha256=yu1Khgy6AZjH2BPQKdnxDid0vQgQ8zvb8-FQlHH-GOU,1465
 datamaestro_text/data/ir/csv.py,sha256=tnxuqR_MZ3GQhuoXEMYyWLQw8PyD0gRqsnzIP5Gsziw,1212
@@ -63,7 +63,7 @@ datamaestro_text/data/ir/stores.py,sha256=A4Ew0L4P6iLLmDKhxqjjVkHcz797BHf4d76Ygu
 datamaestro_text/data/ir/trec.py,sha256=IOtQRMUz8zx-dYEMR2NIIM6qXEUjsV0eVOhGvKIRJK4,1974
 datamaestro_text/data/ir/utils.py,sha256=6-GhXVtgkBZGhIs2-ODZua_3DmKjSSVydStpHDqbAwE,833
 datamaestro_text/datasets/irds/__init__.py,sha256=Tq0HN1qojnZYLBumM59BuTkz7r0gcu-5OXmDDLgPpAc,707
-datamaestro_text/datasets/irds/data.py,sha256=eUehp_80H1yyh7CVkM7mOWJtB9XHlmI-A7fLewXuaDE,22487
+datamaestro_text/datasets/irds/data.py,sha256=YlDbGFsh6_mCmk49F3bwdsLEbpHVvMv4gvc1H8KZnpo,23096
 datamaestro_text/datasets/irds/datasets.py,sha256=n5mvD2omVI4PkeuJgX13Cio6gLrgYjxq8ZUKKyqocEs,5625
 datamaestro_text/datasets/irds/helpers.py,sha256=fGE-fbuJbXdTzl1qo55247jzn9cvApY-d82GJBgfY1E,3982
 datamaestro_text/datasets/irds/utils.py,sha256=m30JLIrV_HgilN11TvY9dGTyumES6LLzWZDUAMT915M,1425
@@ -80,9 +80,9 @@ datamaestro_text/utils/files.py,sha256=jhcirufgTztEkx1hs2-qgScEcHnIcGF_BjCeuCSsN
 datamaestro_text/utils/iter.py,sha256=QBajeSPLHvkeh6BCTZDSqWlOYNjwUDvgTTZ_YxJntXw,2701
 datamaestro_text/utils/randomstream.py,sha256=_-boH4IIqN8qcl3IktjpNp9vmF4TWRzHUSNVwg7WAr8,973
 datamaestro_text/utils/shuffle.py,sha256=o8JTz3mr0lYWyv0zEh91jEK12ci1etMiUnzh5GkOHCM,3490
-datamaestro_text-2025.7.28.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamaestro_text-2025.7.28.dist-info/METADATA,sha256=M0V-4Q2_EBFIRnP0czVXvZC9t_qhhmVRbWSAry31SW4,1848
-datamaestro_text-2025.7.28.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datamaestro_text-2025.7.28.dist-info/entry_points.txt,sha256=lO1P5hE183L5qEEVHlG8d_ik0HNXnX7Eo87cQLdcl-Y,111
-datamaestro_text-2025.7.28.dist-info/top_level.txt,sha256=gYSeqViE8r7eCxSdqFJL74OwljOwKsGPaIhEcCXqc-o,17
-datamaestro_text-2025.7.28.dist-info/RECORD,,
+datamaestro_text-2025.9.11.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamaestro_text-2025.9.11.dist-info/METADATA,sha256=ChGV_8bnixfGl91eG_3-Qwba8tjMwe2VPCwXdGxG_xM,1848
+datamaestro_text-2025.9.11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datamaestro_text-2025.9.11.dist-info/entry_points.txt,sha256=lO1P5hE183L5qEEVHlG8d_ik0HNXnX7Eo87cQLdcl-Y,111
+datamaestro_text-2025.9.11.dist-info/top_level.txt,sha256=gYSeqViE8r7eCxSdqFJL74OwljOwKsGPaIhEcCXqc-o,17
+datamaestro_text-2025.9.11.dist-info/RECORD,,