datamaestro-text 2026.1.1__py3-none-any.whl → 2026.2.3__py3-none-any.whl

datamaestro_text/config/com/microsoft/msmarco/passage.py

@@ -10,10 +10,10 @@ See [https://github.com/microsoft/MSMARCO-Passage-Ranking](https://github.com/mi
 
 from datamaestro.annotations.agreement import useragreement
 from datamaestro.data import Folder
-from datamaestro.download.single import filedownloader
+from datamaestro.download.single import FileDownloader
 from datamaestro.download import reference
-from datamaestro.definitions import datatasks, datatags, dataset
-from datamaestro.download.archive import tardownloader
+from datamaestro.definitions import Dataset, datatasks, datatags, dataset
+from datamaestro.download.archive import TarDownloader
 from datamaestro_text.data.ir import RerankAdhoc, Adhoc, TrainingTripletsLines
 from datamaestro_text.data.ir.csv import (
     Topics,
@@ -39,144 +39,172 @@ http://www.msmarco.org/dataset.aspx""",
 # TODO: Not ideal since it would be better to have small versions right away
 # instead of downloading again the MS Marco Collection
 @lua
-@tardownloader(
-    "data",
-    url="https://msmarco.blob.core.windows.net/msmarcoranking/collectionandqueries.tar.gz",
-    checker=HashCheck("31644046b18952c1386cd4564ba2ae69", md5),
-)
 @dataset(url="https://github.com/microsoft/MSMARCO-Passage-Ranking")
-def collection_etc(data) -> Folder:
+class CollectionEtc(Dataset):
     """Documents and some more files"""
-    return Folder.C(path=data)
+
+    DATA = TarDownloader(
+        "data",
+        url="https://msmarco.blob.core.windows.net/msmarcoranking/collectionandqueries.tar.gz",
+        checker=HashCheck("31644046b18952c1386cd4564ba2ae69", md5),
+    )
+
+    def config(self) -> Folder:
+        return Folder.C(path=self.DATA.path)
 
 
 @lua
-@reference("data", collection_etc)
-@dataset(Documents, size="2.9GB")
-def collection(data):
+@dataset(size="2.9GB")
+class Collection(Dataset):
     """MS-Marco documents
 
     This file contains each passage in the larger MSMARCO dataset.
 
-    Format is TSV (PID \t Passage)"""
-    return {"path": data.path / "collection.tsv"}
+    Format is TSV (PID \\t Passage)"""
+
+    DATA = reference(varname="data", reference=CollectionEtc)
+
+    def config(self) -> Documents:
+        return Documents.C(path=self.DATA.prepare().path / "collection.tsv")
 
 
 # --- Train
 
 
 @lua
-@tardownloader(
-    "run",
-    url="https://msmarco.blob.core.windows.net/msmarcoranking/top1000.train.tar.gz",
-    checker=HashCheck("d99fdbd5b2ea84af8aa23194a3263052", md5),
-)
-@dataset(AdhocRunWithText, size="2.5GB")
-def train_run(run):
+@dataset(size="2.5GB")
+class TrainRun(Dataset):
     """
 
     TSV format: qid, pid, query, passage
     """
-    return {"path": run / "top1000.train.tsv"}
+
+    RUN = TarDownloader(
+        "run",
+        url="https://msmarco.blob.core.windows.net/msmarcoranking/top1000.train.tar.gz",
+        checker=HashCheck("d99fdbd5b2ea84af8aa23194a3263052", md5),
+    )
+
+    def config(self) -> AdhocRunWithText:
+        return AdhocRunWithText.C(path=self.RUN.path / "top1000.train.tsv")
 
 
 @lua
-@tardownloader(
-    "queries",
-    url="https://msmarco.blob.core.windows.net/msmarcoranking/queries.tar.gz",
-    files=["queries.train.tsv"],
-    checker=HashCheck("c177b2795d5f2dcc524cf00fcd973be1", md5),
-)
-@dataset(Topics)
-def train_queries(queries):
-    return {"path": queries / "queries.train.tsv"}
+@dataset()
+class TrainQueries(Dataset):
+    QUERIES = TarDownloader(
+        "queries",
+        url="https://msmarco.blob.core.windows.net/msmarcoranking/queries.tar.gz",
+        files=["queries.train.tsv"],
+        checker=HashCheck("c177b2795d5f2dcc524cf00fcd973be1", md5),
+    )
+
+    def config(self) -> Topics:
+        return Topics.C(path=self.QUERIES.path / "queries.train.tsv")
 
 
 @lua
-@filedownloader(
-    "qrels.tsv",
-    url="https://msmarco.blob.core.windows.net/msmarcoranking/qrels.train.tsv",
-    checker=HashCheck("733fb9fe12d93e497f7289409316eccf", md5),
-)
-@dataset(TrecAdhocAssessments, size="10.1MB")
-def train_qrels(qrels):
-    return {"path": qrels}
+@dataset(size="10.1MB")
+class TrainQrels(Dataset):
+    QRELS = FileDownloader(
+        "qrels.tsv",
+        url="https://msmarco.blob.core.windows.net/msmarcoranking/qrels.train.tsv",
+        checker=HashCheck("733fb9fe12d93e497f7289409316eccf", md5),
+    )
+
+    def config(self) -> TrecAdhocAssessments:
+        return TrecAdhocAssessments.C(path=self.QRELS.path)
 
 
 @lua
-@reference("collection", collection)
-@reference("topics", train_queries)
-@reference("qrels", train_qrels)
 @datatasks("information retrieval", "passage retrieval")
-@dataset(Adhoc, url="https://github.com/microsoft/MSMARCO-Passage-Ranking")
-def train(topics, qrels, collection):
+@dataset(url="https://github.com/microsoft/MSMARCO-Passage-Ranking")
+class Train(Dataset):
     """MS-Marco train dataset"""
-    return {
-        "documents": collection,
-        "topics": topics,
-        "assessments": qrels,
-    }
+
+    COLLECTION = reference(varname="collection", reference=Collection)
+    TOPICS = reference(varname="topics", reference=TrainQueries)
+    QRELS = reference(varname="qrels", reference=TrainQrels)
+
+    def config(self) -> Adhoc:
+        return Adhoc.C(
+            documents=self.COLLECTION.prepare(),
+            topics=self.TOPICS.prepare(),
+            assessments=self.QRELS.prepare(),
+        )
 
 
 @lua
-@reference("train", train)
-@reference("run", train_run)
 @datatasks("information retrieval", "passage retrieval")
-@dataset(RerankAdhoc, url="https://github.com/microsoft/MSMARCO-Passage-Ranking")
-def train_withrun(train, run):
+@dataset(url="https://github.com/microsoft/MSMARCO-Passage-Ranking")
+class TrainWithrun(Dataset):
     """MSMarco train dataset, including the top-1000 to documents to re-rank"""
-    return {**train.__arguments__(), "run": run}
+
+    TRAIN = reference(varname="train", reference=Train)
+    RUN = reference(varname="run", reference=TrainRun)
+
+    def config(self) -> RerankAdhoc:
+        train = self.TRAIN.prepare()
+        return RerankAdhoc.C(**train.__arguments__(), run=self.RUN.prepare())
 
 
 # Training triplets
 
 
-@filedownloader(
-    "triples.tsv",
-    size=1_841_693_309,
-    url="https://msmarco.blob.core.windows.net/msmarcoranking/qidpidtriples.train.full.2.tsv.gz",
-    checker=HashCheck("4e58f45f82f3fe99e3239ecffd8ed371", md5),
-)
 @dataset(
-    TrainingTripletsLines,
     url="https://github.com/microsoft/MSMARCO-Passage-Ranking",
     size="5.7GB",
 )
-def train_idtriples(triples):
+class TrainIdtriples(Dataset):
     """Full training triples (query, positive passage, negative passage) with IDs"""
-    return {"path": triples, "doc_ids": True, "topic_ids": True}
+
+    TRIPLES = FileDownloader(
+        "triples.tsv",
+        size=1_841_693_309,
+        url="https://msmarco.blob.core.windows.net/msmarcoranking/qidpidtriples.train.full.2.tsv.gz",
+        checker=HashCheck("4e58f45f82f3fe99e3239ecffd8ed371", md5),
+    )
+
+    def config(self) -> TrainingTripletsLines:
+        return TrainingTripletsLines.C(
+            path=self.TRIPLES.path, doc_ids=True, topic_ids=True
+        )
 
 
-@filedownloader(
-    "triples.tsv",
-    size=7_930_881_353,
-    url="https://msmarco.blob.core.windows.net/msmarcoranking/triples.train.small.tar.gz",
-    checker=HashCheck("c13bf99ff23ca691105ad12eab837f84", md5),
-)
 @dataset(
-    TrainingTripletsLines,
     url="https://github.com/microsoft/MSMARCO-Passage-Ranking",
     size="27.1GB",
 )
-def train_texttriples_small(triples):
+class TrainTexttriplesSmall(Dataset):
     """Small training triples (query, positive passage, negative passage) with text"""
-    return {"path": triples}
+
+    TRIPLES = FileDownloader(
+        "triples.tsv",
+        size=7_930_881_353,
+        url="https://msmarco.blob.core.windows.net/msmarcoranking/triples.train.small.tar.gz",
+        checker=HashCheck("c13bf99ff23ca691105ad12eab837f84", md5),
+    )
+
+    def config(self) -> TrainingTripletsLines:
+        return TrainingTripletsLines.C(path=self.TRIPLES.path)
 
 
-@filedownloader(
-    "triples.tsv",
-    size=77_877_731_328,
-    url="https://msmarco.blob.core.windows.net/msmarcoranking/triples.train.full.tar.gz",
-    checker=HashCheck("8d509d484ea1971e792b812ae4800c6f", md5),
-)
 @dataset(
-    TrainingTripletsLines,
     url="https://github.com/microsoft/MSMARCO-Passage-Ranking",
     size="272.2GB",
 )
-def train_texttriples_full(triples):
+class TrainTexttripleFull(Dataset):
     """Full training triples (query, positive passage, negative passage) with text"""
-    return {"path": triples}
+
+    TRIPLES = FileDownloader(
+        "triples.tsv",
+        size=77_877_731_328,
+        url="https://msmarco.blob.core.windows.net/msmarcoranking/triples.train.full.tar.gz",
+        checker=HashCheck("8d509d484ea1971e792b812ae4800c6f", md5),
+    )
+
+    def config(self) -> TrainingTripletsLines:
+        return TrainingTripletsLines.C(path=self.TRIPLES.path)
 
 
 # ---
@@ -185,73 +213,88 @@ def train_texttriples_full(triples):
 
 
 @lua
-@tardownloader(
-    "queries",
-    url="https://msmarco.blob.core.windows.net/msmarcoranking/queries.tar.gz",
-    files=["queries.dev.tsv"],
-    checker=HashCheck("c177b2795d5f2dcc524cf00fcd973be1", md5),
-)
-@dataset(Topics)
-def dev_queries(queries):
-    return {"path": queries / "queries.dev.tsv"}
+@dataset()
+class DevQueries(Dataset):
+    QUERIES = TarDownloader(
+        "queries",
+        url="https://msmarco.blob.core.windows.net/msmarcoranking/queries.tar.gz",
+        files=["queries.dev.tsv"],
+        checker=HashCheck("c177b2795d5f2dcc524cf00fcd973be1", md5),
+    )
+
+    def config(self) -> Topics:
+        return Topics.C(path=self.QUERIES.path / "queries.dev.tsv")
 
 
 @lua
-@tardownloader(
-    "run",
-    url="https://msmarco.blob.core.windows.net/msmarcoranking/top1000.dev.tar.gz",
-    checker=HashCheck("8c140662bdf123a98fbfe3bb174c5831", md5),
-)
-@dataset(AdhocRunWithText)
-def dev_run(run):
-    return {"path": run / "top1000.eval.tsv"}
+@dataset()
+class DevRun(Dataset):
+    RUN = TarDownloader(
+        "run",
+        url="https://msmarco.blob.core.windows.net/msmarcoranking/top1000.dev.tar.gz",
+        checker=HashCheck("8c140662bdf123a98fbfe3bb174c5831", md5),
+    )
+
+    def config(self) -> AdhocRunWithText:
+        return AdhocRunWithText.C(path=self.RUN.path / "top1000.eval.tsv")
 
 
 @lua
-@filedownloader(
-    "qrels.tsv",
-    url="https://msmarco.blob.core.windows.net/msmarcoranking/qrels.dev.tsv",
-    checker=HashCheck("9157ccaeaa8227f91722ba5770787b16", md5),
-)
-@dataset(TrecAdhocAssessments)
-def dev_qrels(qrels):
-    return {"path": qrels}
+@dataset()
+class DevQrels(Dataset):
+    QRELS = FileDownloader(
+        "qrels.tsv",
+        url="https://msmarco.blob.core.windows.net/msmarcoranking/qrels.dev.tsv",
+        checker=HashCheck("9157ccaeaa8227f91722ba5770787b16", md5),
+    )
+
+    def config(self) -> TrecAdhocAssessments:
+        return TrecAdhocAssessments.C(path=self.QRELS.path)
 
 
 @lua
-@reference("collection", collection)
-@reference("topics", dev_queries)
-@reference("qrels", dev_qrels)
 @datatasks("information retrieval", "passage retrieval")
-@dataset(Adhoc, url="https://github.com/microsoft/MSMARCO-Passage-Ranking")
-def dev(topics, qrels, collection):
+@dataset(url="https://github.com/microsoft/MSMARCO-Passage-Ranking")
+class Dev(Dataset):
     """MS-Marco dev dataset"""
-    return {
-        "documents": collection,
-        "topics": topics,
-        "assessments": qrels,
-    }
+
+    COLLECTION = reference(varname="collection", reference=Collection)
+    TOPICS = reference(varname="topics", reference=DevQueries)
+    QRELS = reference(varname="qrels", reference=DevQrels)
+
+    def config(self) -> Adhoc:
+        return Adhoc.C(
+            documents=self.COLLECTION.prepare(),
+            topics=self.TOPICS.prepare(),
+            assessments=self.QRELS.prepare(),
+        )
 
 
 @lua
-@reference("dev", dev)
-@reference("run", dev_run)
 @datatasks("information retrieval", "passage retrieval")
-@dataset(RerankAdhoc, url="https://github.com/microsoft/MSMARCO-Passage-Ranking")
-def dev_withrun(dev, run):
+@dataset(url="https://github.com/microsoft/MSMARCO-Passage-Ranking")
+class DevWithrun(Dataset):
     """MSMarco dev dataset, including the top-1000 to documents to re-rank"""
-    return {**dev.__arguments__(), "run": run}
+
+    DEV = reference(varname="dev", reference=Dev)
+    RUN = reference(varname="run", reference=DevRun)
+
+    def config(self) -> RerankAdhoc:
+        dev = self.DEV.prepare()
+        return RerankAdhoc.C(**dev.__arguments__(), run=self.RUN.prepare())
 
 
 @lua
-@tardownloader(
-    "run",
-    url="https://msmarco.blob.core.windows.net/msmarcoranking/top1000.eval.tar.gz",
-    checker=HashCheck("73778cd99f6e0632d12d0b5731b20a02", md5),
-)
-@dataset(AdhocRunWithText)
-def eval_withrun(run):
-    return {"path": run / "top1000.eval.tsv"}
+@dataset()
+class EvalWithrun(Dataset):
+    RUN = TarDownloader(
+        "run",
+        url="https://msmarco.blob.core.windows.net/msmarcoranking/top1000.eval.tar.gz",
+        checker=HashCheck("73778cd99f6e0632d12d0b5731b20a02", md5),
+    )
+
+    def config(self) -> AdhocRunWithText:
+        return AdhocRunWithText.C(path=self.RUN.path / "top1000.eval.tsv")
 
 
 # ---
@@ -260,32 +303,44 @@ def eval_withrun(run):
 # ---
 
 
-@reference("data", collection_etc)
-@dataset(Topics, url="https://github.com/microsoft/MSMARCO-Passage-Ranking")
-def dev_small_queries(data):
-    return {"path": data.path / "queries.dev.small.tsv"}
+@dataset(url="https://github.com/microsoft/MSMARCO-Passage-Ranking")
+class DevSmallQueries(Dataset):
+    DATA = reference(varname="data", reference=CollectionEtc)
 
+    def config(self) -> Topics:
+        return Topics.C(path=self.DATA.prepare().path / "queries.dev.small.tsv")
 
-@reference("data", collection_etc)
-@dataset(
-    TrecAdhocAssessments, url="https://github.com/microsoft/MSMARCO-Passage-Ranking"
-)
-def dev_small_qrels(data):
-    return {"path": data.path / "qrels.dev.small.tsv"}
 
+@dataset(url="https://github.com/microsoft/MSMARCO-Passage-Ranking")
+class DevSmallQrels(Dataset):
+    DATA = reference(varname="data", reference=CollectionEtc)
+
+    def config(self) -> TrecAdhocAssessments:
+        return TrecAdhocAssessments.C(
+            path=self.DATA.prepare().path / "qrels.dev.small.tsv"
+        )
 
-@reference("topics", dev_small_queries)
-@reference("qrels", dev_small_qrels)
-@reference("collection", collection)
-@dataset(Adhoc, url="https://github.com/microsoft/MSMARCO-Passage-Ranking")
-def dev_small(collection, topics, qrels):
-    return {"documents": collection, "topics": topics, "assessments": qrels}
 
+@dataset(url="https://github.com/microsoft/MSMARCO-Passage-Ranking")
+class DevSmall(Dataset):
+    TOPICS = reference(varname="topics", reference=DevSmallQueries)
+    QRELS = reference(varname="qrels", reference=DevSmallQrels)
+    COLLECTION = reference(varname="collection", reference=Collection)
 
-@reference("data", collection_etc)
-@dataset(Topics, url="https://github.com/microsoft/MSMARCO-Passage-Ranking")
-def eval_queries_small(data):
-    return {"path": data.path / "queries.eval.small.tsv"}
+    def config(self) -> Adhoc:
+        return Adhoc.C(
+            documents=self.COLLECTION.prepare(),
+            topics=self.TOPICS.prepare(),
+            assessments=self.QRELS.prepare(),
+        )
+
+
+@dataset(url="https://github.com/microsoft/MSMARCO-Passage-Ranking")
+class EvalQueriesSmall(Dataset):
+    DATA = reference(varname="data", reference=CollectionEtc)
+
+    def config(self) -> Topics:
+        return Topics.C(path=self.DATA.prepare().path / "queries.eval.small.tsv")
 
 
 # ---
@@ -294,63 +349,74 @@ def eval_queries_small(data):
 
 
 @lua
-@filedownloader(
-    "queries.tsv",
-    url="https://msmarco.blob.core.windows.net/msmarcoranking/msmarco-test2019-queries.tsv.gz",
-    checker=HashCheck("756e60d714cee28d3b552289d6272f1d", md5),
-)
-@dataset(Topics)
-def trec2019_test_queries(queries):
-    return {"path": queries}
+@dataset()
+class Trec2019TestQueries(Dataset):
+    QUERIES = FileDownloader(
+        "queries.tsv",
+        url="https://msmarco.blob.core.windows.net/msmarcoranking/msmarco-test2019-queries.tsv.gz",
+        checker=HashCheck("756e60d714cee28d3b552289d6272f1d", md5),
+    )
+
+    def config(self) -> Topics:
+        return Topics.C(path=self.QUERIES.path)
 
 
 @lua
-@filedownloader(
-    "run.tsv",
-    url="https://msmarco.blob.core.windows.net/msmarcoranking/msmarco-passagetest2019-top1000.tsv.gz",
-    checker=HashCheck("ec9e012746aa9763c7ff10b3336a3ce1", md5),
-)
-@dataset(AdhocRunWithText)
-def trec2019_test_run(run):
-    return {"path": run / "top1000.eval.tsv"}
+@dataset()
+class Trec2019TestRun(Dataset):
+    RUN = FileDownloader(
+        "run.tsv",
+        url="https://msmarco.blob.core.windows.net/msmarcoranking/msmarco-passagetest2019-top1000.tsv.gz",
+        checker=HashCheck("ec9e012746aa9763c7ff10b3336a3ce1", md5),
+    )
+
+    def config(self) -> AdhocRunWithText:
+        return AdhocRunWithText.C(path=self.RUN.path / "top1000.eval.tsv")
 
 
 @lua
-@filedownloader(
-    "qrels.tsv",
-    url="https://trec.nist.gov/data/deep/2019qrels-pass.txt",
-    checker=HashCheck("2f4be390198da108f6845c822e5ada14", md5),
-)
-@dataset(TrecAdhocAssessments)
-def trec2019_test_qrels(qrels):
-    return {"path": qrels}
+@dataset()
+class Trec2019TestQrels(Dataset):
+    QRELS = FileDownloader(
+        "qrels.tsv",
+        url="https://trec.nist.gov/data/deep/2019qrels-pass.txt",
+        checker=HashCheck("2f4be390198da108f6845c822e5ada14", md5),
+    )
+
+    def config(self) -> TrecAdhocAssessments:
+        return TrecAdhocAssessments.C(path=self.QRELS.path)
 
 
 @lua
-@reference("collection", collection)
-@reference("topics", trec2019_test_queries)
-@reference("qrels", trec2019_test_qrels)
 @datatasks("information retrieval", "passage retrieval")
-@dataset(Adhoc, url="https://microsoft.github.io/msmarco/TREC-Deep-Learning-2019.html")
-def trec2019_test(topics, qrels, collection):
+@dataset(url="https://microsoft.github.io/msmarco/TREC-Deep-Learning-2019.html")
+class Trec2019Test(Dataset):
     "TREC Deep Learning (2019)"
-    return {
-        "documents": collection,
-        "topics": topics,
-        "assessments": qrels,
-    }
+
+    COLLECTION = reference(varname="collection", reference=Collection)
+    TOPICS = reference(varname="topics", reference=Trec2019TestQueries)
+    QRELS = reference(varname="qrels", reference=Trec2019TestQrels)
+
+    def config(self) -> Adhoc:
+        return Adhoc.C(
+            documents=self.COLLECTION.prepare(),
+            topics=self.TOPICS.prepare(),
+            assessments=self.QRELS.prepare(),
+        )
 
 
 @lua
-@reference("trec2019", trec2019_test)
-@reference("run", trec2019_test_run)
 @datatasks("information retrieval", "passage retrieval")
-@dataset(
-    RerankAdhoc, url="https://microsoft.github.io/msmarco/TREC-Deep-Learning-2019.html"
-)
-def trec2019_test_withrun(trec2019, run):
+@dataset(url="https://microsoft.github.io/msmarco/TREC-Deep-Learning-2019.html")
+class Trec2019TestWithrun(Dataset):
     """TREC Deep Learning (2019), including the top-1000 to documents to re-rank"""
-    return {**trec2019.__arguments__(), "run": run}
+
+    TREC2019 = reference(varname="trec2019", reference=Trec2019Test)
+    RUN = reference(varname="run", reference=Trec2019TestRun)
+
+    def config(self) -> RerankAdhoc:
+        trec2019 = self.TREC2019.prepare()
+        return RerankAdhoc.C(**trec2019.__arguments__(), run=self.RUN.prepare())
 
 
 # ---
@@ -359,33 +425,38 @@ def trec2019_test_withrun(trec2019, run):
 
 
 @lua
-@filedownloader(
-    "queries.tsv",
-    url="https://msmarco.blob.core.windows.net/msmarcoranking/msmarco-test2020-queries.tsv.gz",
-    checker=HashCheck("00a406fb0d14ed3752d70d1e4eb98600", md5),
-)
-@dataset(Topics, size="12K")
-def trec2020_test_queries(queries):
+@dataset(size="12K")
+class Trec2020TestQueries(Dataset):
     """TREC Deep Learning 2019 (topics)
 
     Topics of the TREC 2019 MS-Marco Deep Learning track"""
-    return {"path": queries}
+
+    QUERIES = FileDownloader(
+        "queries.tsv",
+        url="https://msmarco.blob.core.windows.net/msmarcoranking/msmarco-test2020-queries.tsv.gz",
+        checker=HashCheck("00a406fb0d14ed3752d70d1e4eb98600", md5),
+    )
+
+    def config(self) -> Topics:
+        return Topics.C(path=self.QUERIES.path)
 
 
 @lua
 @datatasks("information retrieval", "passage retrieval")
 @datatags("reranking")
-@filedownloader(
-    "run.tsv",
-    url="https://msmarco.blob.core.windows.net/msmarcoranking/msmarco-passagetest2020-top1000.tsv.gz",
-    checker=HashCheck("aa6fbc51d66bd1dc745964c0e140a727", md5),
-)
 @dataset(
-    AdhocRunWithText,
     url="https://microsoft.github.io/msmarco/TREC-Deep-Learning-2020.html",
 )
-def trec2020_test_run(run):
+class Trec2020TestRun(Dataset):
     """TREC Deep Learning (2020)
 
     Set of query/passages for the passage re-ranking task re-rank (TREC 2020)"""
-    return {"path": run / "top1000.eval.tsv"}
+
+    RUN = FileDownloader(
+        "run.tsv",
+        url="https://msmarco.blob.core.windows.net/msmarcoranking/msmarco-passagetest2020-top1000.tsv.gz",
+        checker=HashCheck("aa6fbc51d66bd1dc745964c0e140a727", md5),
+    )
+
+    def config(self) -> AdhocRunWithText:
+        return AdhocRunWithText.C(path=self.RUN.path / "top1000.eval.tsv")

datamaestro_text/config/com/oscar-corpus.py

@@ -1,20 +1,23 @@
-from datamaestro.definitions import datatasks, datatags, dataset
-from datamaestro.download.single import filedownloader
+from datamaestro.definitions import Dataset, dataset
+from datamaestro.download.single import FileDownloader
 from datamaestro_text.data.text import TextFile
 from datamaestro.utils import HashCheck
 
 
-@filedownloader(
-    "file",
-    "https://oscar-public.huma-num.fr/shuffled/en_dedup.txt.gz",
-    checker=HashCheck("5c906ede3c5265f8934b62c275a754bc"),
-)
-@dataset(TextFile, url="https://oscar-corpus.com/", size="2.3T")
-def english(file):
+@dataset(url="https://oscar-corpus.com/", size="2.3T")
+class English(Dataset):
     """Huge French corpus from INRIA
 
     OSCAR or Open Super-large Crawled ALMAnaCH coRpus is a huge multilingual corpus
     obtained by language classification and filtering of the Common Crawl corpus using
     the goclassy architecture.
     """
-    return {"path": file}
+
+    FILE = FileDownloader(
+        "file",
+        "https://oscar-public.huma-num.fr/shuffled/en_dedup.txt.gz",
+        checker=HashCheck("5c906ede3c5265f8934b62c275a754bc"),
+    )
+
+    def config(self) -> TextFile:
+        return TextFile.C(path=self.FILE.path)