datamaestro-text 2026.1.1__py3-none-any.whl → 2026.2.3__py3-none-any.whl

datamaestro_text/__init__.py

@@ -1,6 +1,6 @@
 import datamaestro
 
-from .version import version, version_tuple
+from .version import version as version, version_tuple as version_tuple
 
 
 class Repository(datamaestro.Repository):

datamaestro_text/config/com/github/aagohary/canard.py

@@ -1,5 +1,5 @@
-from datamaestro.definitions import datatasks, datatags, dataset
-from datamaestro.download.single import filedownloader
+from datamaestro.definitions import Dataset, datatasks, datatags, dataset
+from datamaestro.download.single import FileDownloader
 from datamaestro.utils import HashCheck
 
 from datamaestro.data.ml import Supervised
@@ -8,23 +8,8 @@ from datamaestro_text.data.conversation.canard import CanardDataset
 
 @datatags("conversation", "context", "query")
 @datatasks("query rewriting")
-@filedownloader(
-    "train.json",
-    "https://raw.githubusercontent.com/aagohary/canard/refs/heads/master/data/release/train.json",
-    checker=HashCheck("73624ac646fb81e09b0fd7f01370ada3"),
-)
-@filedownloader(
-    "dev.json",
-    "https://raw.githubusercontent.com/aagohary/canard/refs/heads/master/data/release/dev.json",
-    checker=HashCheck("c84525631a83bc771c58ff31f4a9b601"),
-)
-@filedownloader(
-    "test.json",
-    "https://raw.githubusercontent.com/aagohary/canard/refs/heads/master/data/release/test.json",
-    checker=HashCheck("3fc14d0078e7a5056f5da571728f024e"),
-)
-@dataset(Supervised, url="https://sites.google.com/view/qanta/projects/canard", id="")
-def main(train, dev, test):
+@dataset(url="https://sites.google.com/view/qanta/projects/canard", id="")
+class Main(Dataset):
     """Question-in-context rewriting
 
     CANARD is a dataset for question-in-context rewriting that consists of
@@ -36,8 +21,26 @@ def main(train, dev, test):
 
     Each dataset is an instance of :class:`datamaestro_text.data.conversation.CanardDataset`
     """
-    return {
-        "train": CanardDataset.C(path=train),
-        "validation": CanardDataset.C(path=dev),
-        "test": CanardDataset.C(path=test),
-    }
+
+    TRAIN = FileDownloader(
+        "train.json",
+        "https://raw.githubusercontent.com/aagohary/canard/refs/heads/master/data/release/train.json",
+        checker=HashCheck("73624ac646fb81e09b0fd7f01370ada3"),
+    )
+    DEV = FileDownloader(
+        "dev.json",
+        "https://raw.githubusercontent.com/aagohary/canard/refs/heads/master/data/release/dev.json",
+        checker=HashCheck("c84525631a83bc771c58ff31f4a9b601"),
+    )
+    TEST = FileDownloader(
+        "test.json",
+        "https://raw.githubusercontent.com/aagohary/canard/refs/heads/master/data/release/test.json",
+        checker=HashCheck("3fc14d0078e7a5056f5da571728f024e"),
+    )
+
+    def config(self) -> Supervised:
+        return Supervised.C(
+            train=CanardDataset.C(path=self.TRAIN.path),
+            validation=CanardDataset.C(path=self.DEV.path),
+            test=CanardDataset.C(path=self.TEST.path),
+        )

datamaestro_text/config/com/github/apple/ml-qrecc.py

@@ -3,10 +3,10 @@
 import re
 import json
 from pathlib import Path
-from datamaestro.definitions import datatasks, datatags, dataset
+from datamaestro.definitions import Dataset, datatasks, datatags, dataset
 from datamaestro.data.ml import Supervised
 from datamaestro.download import reference
-from datamaestro.download.archive import zipdownloader
+from datamaestro.download.archive import ZipDownloader
 from datamaestro.download.wayback import wayback_documents
 from datamaestro.utils import HashCheck
 from datamaestro_text.data.conversation.qrecc import QReCCDataset
@@ -19,17 +19,12 @@ from datamaestro_text.datasets.irds.helpers import lz4docstore_builder
 
 @datatags("conversation", "context", "query")
 @datatasks("query rewriting")
-@zipdownloader(
-    "data",
-    "https://github.com/apple/ml-qrecc/raw/main/dataset/qrecc_data.zip",
-    checker=HashCheck("f88fcc7ef3678cd6312080389c8abd67"),
-)
 @dataset(
     url="https://github.com/apple/ml-qrecc",
     doi="https://doi.org/10.48550/arXiv.2010.04898",
     id="",
 )
-def main(data: Path) -> Supervised[QReCCDataset, None, QReCCDataset]:
+class Main(Dataset):
     """Open-Domain Question Answering Goes Conversational via Question Rewriting
 
     We introduce QReCC (Question Rewriting in Conversational Context), an
@@ -39,34 +34,44 @@ def main(data: Path) -> Supervised[QReCCDataset, None, QReCCDataset]:
     answering that includes the individual subtasks of question rewriting,
     passage retrieval and reading comprehension
     """
-    return Supervised.C(
-        train=QReCCDataset.C(path=data / "qrecc_train.json"),
-        test=QReCCDataset.C(path=data / "qrecc_test.json"),
+
+    DATA = ZipDownloader(
+        "data",
+        "https://github.com/apple/ml-qrecc/raw/main/dataset/qrecc_data.zip",
+        checker=HashCheck("f88fcc7ef3678cd6312080389c8abd67"),
     )
 
+    def config(self) -> Supervised:
+        return Supervised.C(
+            train=QReCCDataset.C(path=self.DATA.path / "qrecc_train.json"),
+            test=QReCCDataset.C(path=self.DATA.path / "qrecc_test.json"),
+        )
+
 
 @dataset(
     url="https://github.com/apple/ml-qrecc",
     doi="https://doi.org/10.48550/arXiv.2010.04898",
 )
-class Content(LZ4JSONLDocumentStore):
+class Content(Dataset):
     """QReCC mentionned URLs content"""
 
-    @staticmethod
-    def __create_dataset__(dataset, options=None):
-        ds = reference(reference=main).setup(dataset, options)
-        documents_path = wayback_documents(
-            "20191127", lambda: Content._urls(ds), name="wayback.jsonl"
-        ).setup(dataset, options)
+    MAIN = reference(reference=Main)
 
-        store_path = lz4docstore_builder(
-            "store",
-            lambda: Content._documents(documents_path),
-            SimpleJsonDocument,
-            "id",
-        ).setup(dataset, options)
+    WAYBACK_DOCS = wayback_documents(
+        "20191127",
+        lambda: Content._urls(Content.MAIN.prepare()),
+        name="wayback.jsonl",
+    )
+
+    STORE = lz4docstore_builder(
+        "store",
+        lambda: Content._documents(Content.WAYBACK_DOCS.path),
+        SimpleJsonDocument,
+        "id",
+    )
 
-        return Content.C(jsonl_path=store_path)
+    def config(self) -> LZ4JSONLDocumentStore:
+        return LZ4JSONLDocumentStore.C(jsonl_path=self.STORE.path)
 
     @staticmethod
     def _documents(path: Path):

datamaestro_text/config/com/github/ikat.py

@@ -1,14 +1,13 @@
 # See documentation on https://datamaestro.readthedocs.io
 
-import bz2
 from datamaestro.download import reference
-from datamaestro.definitions import datatasks, datatags, dataset
+from datamaestro.definitions import Dataset, datatasks, datatags, dataset
 from datamaestro_text.data.conversation.base import ConversationUserTopics
 from datamaestro_text.data.ir import Adhoc
 
 from datamaestro.utils import HashCheck
 from datamaestro.context import DatafolderPath
-from datamaestro.download.single import filedownloader
+from datamaestro.download.single import FileDownloader
 from datamaestro_text.data.conversation.ikat import IkatConversations
 from datamaestro.download.links import linkfolder
 
@@ -17,105 +16,120 @@ from datamaestro_text.data.ir.trec import TrecAdhocAssessments
 from datamaestro_text.datasets.irds.helpers import lz4docstore_builder
 
 
-@dataset(as_prepare=True)
-def clueweb22(dataset, options=None) -> IKatClueWeb22DocumentStore:
+@dataset()
+class Clueweb22(Dataset):
     # Number of documents in the dataset
     count = 116_838_987
 
-    jsonl_folder = linkfolder(
+    JSONL_FOLDER = linkfolder(
         "documents", [DatafolderPath("gov.nist.trec.ikat.clueweb22", "jsonl")]
-    ).setup(dataset, options)
-    store_path = lz4docstore_builder(
+    )
+
+    STORE_PATH = lz4docstore_builder(
         "store",
         IKatClueWeb22DocumentStore.generator(
-            jsonl_folder,
-            jsonl_folder / "ikat_2023_passages_jsonl.sha256sums",
-            jsonl_folder / "ikat_2023_passages_hashes.tsv.bz2",
+            JSONL_FOLDER,
+            "ikat_2023_passages_jsonl.sha256sums",
+            "ikat_2023_passages_hashes.tsv.bz2",
         ),
         IKatClueWeb22DocumentStore.Document,
         "id",
         count_hint=count,
-    ).setup(dataset, options)
+    )
 
-    return IKatClueWeb22DocumentStore.C(path=store_path, count=count)
+    def config(self) -> IKatClueWeb22DocumentStore:
+        return IKatClueWeb22DocumentStore.C(path=self.STORE_PATH.path, count=self.count)
 
 
 @datatags("conversation", "context", "query")
 @datatasks("conversational search", "query rewriting")
-@reference("documents", clueweb22)
-@filedownloader(
-    "topics.json",
-    "https://raw.githubusercontent.com/irlabamsterdam/iKAT/refs/heads/main/2025/data/2025_test_topics.json",
-    checker=HashCheck("16f8444a8d0a8dfe0090f478f185a63c"),
-)
 @dataset(
-    id="2025",
+    id=".2025",
     url="https://github.com/irlabamsterdam/iKAT/tree/main/2025",
 )
-def test_2025(topics, documents) -> Adhoc.C:
+class Test2025(Dataset):
     """Question-in-context rewriting
 
     iKAT is a test dataset for question-in-context rewriting that consists of
     questions each given in a dialog context together with a context-independent
     rewriting of the question.
     """
-    return Adhoc.C(
-        topics=ConversationUserTopics.C(conversations=IkatConversations.C(path=topics)),
-        # TODO: add when available
-        assessments=TrecAdhocAssessments.C(path="/to/do"),
-        documents=documents,
+
+    DOCUMENTS = reference(varname="documents", reference=Clueweb22)
+    TOPICS = FileDownloader(
+        "topics.json",
+        "https://raw.githubusercontent.com/irlabamsterdam/iKAT/refs/heads/main/2025/data/2025_test_topics.json",
+        checker=HashCheck("16f8444a8d0a8dfe0090f478f185a63c"),
     )
 
+    def config(self) -> Adhoc:
+        return Adhoc.C(
+            topics=ConversationUserTopics.C(
+                conversations=IkatConversations.C(path=self.TOPICS.path)
+            ),
+            # TODO: add when available
+            assessments=TrecAdhocAssessments.C(path="/to/do"),
+            documents=self.DOCUMENTS.prepare(),
+        )
+
 
 @datatags("conversation", "context", "query")
 @datatasks("conversational search", "query rewriting")
-@reference("documents", clueweb22)
-@filedownloader(
-    "qrels",
-    "https://trec.nist.gov/data/ikat/2024-qrels.txt",
-    checker=HashCheck("57f958903ed1c12bbac207f62800814f"),
-)
-@filedownloader(
-    "topics.json",
-    "https://raw.githubusercontent.com/irlabamsterdam/iKAT/refs/heads/main/2024/data/2024_test_topics.json",
-    checker=HashCheck("ad45bc6e7add2081d69ea60a0a4d1203"),
-)
 @dataset(
-    Adhoc,
-    id="2024",
+    id=".2024",
     url="https://github.com/irlabamsterdam/iKAT/tree/main/2024",
 )
-def test_2024(topics, qrels, documents) -> Adhoc.C:
+class Test2024(Dataset):
     """iKAT 2024 dataset"""
-    return Adhoc.C(
-        topics=ConversationUserTopics.C(conversations=IkatConversations.C(path=topics)),
-        assessments=TrecAdhocAssessments.C(path=qrels),
-        documents=documents,
+
+    DOCUMENTS = reference(varname="documents", reference=Clueweb22)
+    QRELS = FileDownloader(
+        "qrels",
+        "https://trec.nist.gov/data/ikat/2024-qrels.txt",
+        checker=HashCheck("57f958903ed1c12bbac207f62800814f"),
+    )
+    TOPICS = FileDownloader(
+        "topics.json",
+        "https://raw.githubusercontent.com/irlabamsterdam/iKAT/refs/heads/main/2024/data/2024_test_topics.json",
+        checker=HashCheck("ad45bc6e7add2081d69ea60a0a4d1203"),
     )
 
+    def config(self) -> Adhoc:
+        return Adhoc.C(
+            topics=ConversationUserTopics.C(
+                conversations=IkatConversations.C(path=self.TOPICS.path)
+            ),
+            assessments=TrecAdhocAssessments.C(path=self.QRELS.path),
+            documents=self.DOCUMENTS.prepare(),
+        )
+
 
 @datatags("conversation", "context", "query")
 @datatasks("conversational search", "query rewriting")
-@reference("documents", clueweb22)
-@filedownloader(
-    "qrels",
-    "https://trec.nist.gov/data/ikat/2023-qrels.all-turns.txt",
-    checker=HashCheck("79dc121bab25b2245e52a53263e5ad1f"),
-)
-@filedownloader(
-    "topics.json",
-    "https://raw.githubusercontent.com/irlabamsterdam/iKAT/refs/heads/main/2023/data/2023_test_topics.json",
-    checker=HashCheck("684fa0197cdec8c3cfb6a2e586ab83f6"),
-)
 @dataset(
-    Adhoc,
-    id="2023",
+    id=".2023",
     url="https://github.com/irlabamsterdam/iKAT/tree/main/2023",
 )
-def test_2023(topics, qrels, documents) -> Adhoc.C:
+class Test2023(Dataset):
     """iKAT 2023 dataset"""
-    return Adhoc.C(
-        topics=ConversationUserTopics.C(conversations=IkatConversations.C(path=topics)),
-        assessments=TrecAdhocAssessments.C(path=qrels),
-        documents=documents,
+
+    DOCUMENTS = reference(varname="documents", reference=Clueweb22)
+    QRELS = FileDownloader(
+        "qrels",
+        "https://trec.nist.gov/data/ikat/2023-qrels.all-turns.txt",
+        checker=HashCheck("79dc121bab25b2245e52a53263e5ad1f"),
     )
+    TOPICS = FileDownloader(
+        "topics.json",
+        "https://raw.githubusercontent.com/irlabamsterdam/iKAT/refs/heads/main/2023/data/2023_test_topics.json",
+        checker=HashCheck("684fa0197cdec8c3cfb6a2e586ab83f6"),
+    )
+
+    def config(self) -> Adhoc:
+        return Adhoc.C(
+            topics=ConversationUserTopics.C(
+                conversations=IkatConversations.C(path=self.TOPICS.path)
+            ),
+            assessments=TrecAdhocAssessments.C(path=self.QRELS.path),
+            documents=self.DOCUMENTS.prepare(),
+        )

datamaestro_text/config/com/github/prdwb/orconvqa.py

@@ -4,8 +4,8 @@ import gzip
 import json
 from pathlib import Path
 from typing import Iterator
-from datamaestro.definitions import datatasks, datatags, dataset
-from datamaestro.download.single import filedownloader
+from datamaestro.definitions import Dataset, datatasks, datatags, dataset
+from datamaestro.download.single import FileDownloader
 from datamaestro.utils import HashCheck
 
 
@@ -18,26 +18,10 @@ from datamaestro_text.datasets.irds.helpers import lz4docstore_downloader
 
 @datatags("conversation", "context", "query")
 @datatasks("query rewriting")
-@filedownloader(
-    "train.jsonl",
-    "https://ciir.cs.umass.edu/downloads/ORConvQA/preprocessed/train.txt",
-    checker=HashCheck("7513a9ef12d8b7a4471166dc4fef77b7"),
-)
-@filedownloader(
-    "dev.jsonl",
-    "https://ciir.cs.umass.edu/downloads/ORConvQA/preprocessed/dev.txt",
-    checker=HashCheck("7765658995cc9ffd5eb39a400d814b20"),
-)
-@filedownloader(
-    "test.jsonl",
-    "https://ciir.cs.umass.edu/downloads/ORConvQA/preprocessed/test.txt",
-    checker=HashCheck("0cf3a755f06297b9c02e7db45f8dc8be"),
-)
 @dataset(
-    Supervised,
     url="https://github.com/prdwb/orconvqa-release",
 )
-def preprocessed(train, dev, test):
+class Preprocessed(Dataset):
     """Open-Retrieval Conversational Question Answering datasets
 
     OrConvQA is an aggregation of three existing datasets:
@@ -48,11 +32,29 @@ def preprocessed(train, dev, test):
 
     Each dataset is an instance of :class:`datamaestro_text.data.conversation.OrConvQADataset`
     """
-    return {
-        "train": OrConvQADataset.C(path=train),
-        "validation": OrConvQADataset.C(path=dev),
-        "test": OrConvQADataset.C(path=test),
-    }
+
+    TRAIN = FileDownloader(
+        "train.jsonl",
+        "https://ciir.cs.umass.edu/downloads/ORConvQA/preprocessed/train.txt",
+        checker=HashCheck("7513a9ef12d8b7a4471166dc4fef77b7"),
+    )
+    DEV = FileDownloader(
+        "dev.jsonl",
+        "https://ciir.cs.umass.edu/downloads/ORConvQA/preprocessed/dev.txt",
+        checker=HashCheck("7765658995cc9ffd5eb39a400d814b20"),
+    )
+    TEST = FileDownloader(
+        "test.jsonl",
+        "https://ciir.cs.umass.edu/downloads/ORConvQA/preprocessed/test.txt",
+        checker=HashCheck("0cf3a755f06297b9c02e7db45f8dc8be"),
+    )
+
+    def config(self) -> Supervised:
+        return Supervised.C(
+            train=OrConvQADataset.C(path=self.TRAIN.path),
+            validation=OrConvQADataset.C(path=self.DEV.path),
+            test=OrConvQADataset.C(path=self.TEST.path),
+        )
 
 
 def orConvQADocumentReader(source: Path) -> Iterator[OrConvQADocumentStore.NAMED_TUPLE]:
@@ -63,21 +65,10 @@ def orConvQADocumentReader(source: Path) -> Iterator[OrConvQADocumentStore.NAMED
             yield OrConvQADocumentStore.NAMED_TUPLE(**data)
 
 
-@lz4docstore_downloader(
-    "all_blocks",
-    "https://ciir.cs.umass.edu/downloads/ORConvQA/all_blocks.txt.gz",
-    orConvQADocumentReader,
-    OrConvQADocumentStore.NAMED_TUPLE,
-    "id",
-    checker=HashCheck("1095a3408690e7bbe4d8a87a2bae6356"),
-    size=5_086_902_800,
-    count_hint=11_377_951,
-)
 @dataset(
-    OrConvQADocumentStore,
     url="https://github.com/prdwb/orconvqa-release",
 )
-def passages(all_blocks):
+class Passages(Dataset):
     """orConvQA wikipedia files
 
     OrConvQA is an aggregation of three existing datasets:
@@ -86,4 +77,17 @@ def passages(all_blocks):
     1. the CANARD dataset that consists of context-independent rewrites of QuAC questions, and
     3. the Wikipedia corpus that serves as the knowledge source of answering questions.
     """
-    return {"path": all_blocks, "count": 11_377_951}
+
+    ALL_BLOCKS = lz4docstore_downloader(
+        "all_blocks",
+        "https://ciir.cs.umass.edu/downloads/ORConvQA/all_blocks.txt.gz",
+        orConvQADocumentReader,
+        OrConvQADocumentStore.NAMED_TUPLE,
+        "id",
+        checker=HashCheck("1095a3408690e7bbe4d8a87a2bae6356"),
+        size=5_086_902_800,
+        count_hint=11_377_951,
+    )
+
+    def config(self) -> OrConvQADocumentStore:
+        return OrConvQADocumentStore.C(path=self.ALL_BLOCKS.path, count=11_377_951)