datamaestro-text 2026.1.1__tar.gz → 2026.2.3__tar.gz

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datamaestro-text
-Version: 2026.1.1
+Version: 2026.2.3
 Summary: Datamaestro module for text-related datasets
 Project-URL: Homepage, https://github.com/experimaestro/datamaestro_text
 Project-URL: Documentation, https://datamaestro-text.readthedocs.io/en/latest/
@@ -22,15 +22,9 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.10
 Requires-Dist: attrs
-Requires-Dist: datamaestro>=1.6.2
+Requires-Dist: datamaestro>=1.8.0
 Requires-Dist: experimaestro
 Requires-Dist: ir-datasets>=0.5.8
-Provides-Extra: dev
-Requires-Dist: docutils; extra == 'dev'
-Requires-Dist: flake8; extra == 'dev'
-Requires-Dist: pytest; extra == 'dev'
-Requires-Dist: sphinx<8; extra == 'dev'
-Requires-Dist: sphobjinv; extra == 'dev'
 Description-Content-Type: text/markdown
 
  [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) [![PyPI version](https://badge.fury.io/py/datamaestro-text.svg)](https://badge.fury.io/py/datamaestro-text)

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.3}/pyproject.toml

@@ -26,21 +26,12 @@ classifiers = [
 requires-python = ">=3.10"
 dynamic = ["version"]
 dependencies = [
-    "datamaestro>=1.6.2",
+    "datamaestro>=1.8.0",
     "ir_datasets>=0.5.8",
     "attrs",
     "experimaestro",
 ]
 
-[project.optional-dependencies]
-dev = [
-    "pytest",
-    "docutils",
-    "sphobjinv",
-    "flake8",
-    "sphinx<8",
-]
-
 [project.urls]
 Homepage = "https://github.com/experimaestro/datamaestro_text"
 Documentation = "https://datamaestro-text.readthedocs.io/en/latest/"
@@ -79,9 +70,36 @@ norecursedirs = ["node_modules"]
 [dependency-groups]
 dev = [
     "docutils>=0.21.2",
-    "flake8>=7.3.0",
     "git-cliff>=2.11.0",
     "pytest>=8.4.1",
+    "ruff>=0.8",
     "sphinx>=7,<8",
     "sphobjinv>=2.3.1.3",
 ]
+docs = [
+    "datamaestro>=0.8.5",
+    "experimaestro>=2.0.0b29",  # Pre-release needed for EnumType.name() fix
+    "myst-parser>=0.18.0",
+    "sphinx>=6,<8",  # experimaestro.sphinx uses restify removed in Sphinx 8
+    "sphinx-codeautolink>=0.15.0",
+    "sphinx-rtd-theme>=3.1.0",
+    "sphinx-toolbox>=4.1.2",
+]
+
+[tool.ruff]
+line-length = 88
+target-version = "py310"
+
+[tool.ruff.lint]
+select = [
+    "E",      # pycodestyle errors
+    "F",      # pyflakes
+    "W",      # pycodestyle warnings
+    "C90",    # mccabe complexity
+]
+ignore = [
+    "E501",   # line too long (handled by formatter)
+]
+
+[tool.ruff.lint.mccabe]
+max-complexity = 20

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.3}/src/datamaestro_text/__init__.py

@@ -1,6 +1,6 @@
 import datamaestro
 
-from .version import version, version_tuple
+from .version import version as version, version_tuple as version_tuple
 
 
 class Repository(datamaestro.Repository):

datamaestro_text-2026.2.3/src/datamaestro_text/config/com/github/aagohary/canard.py

@@ -0,0 +1,46 @@
+from datamaestro.definitions import Dataset, datatasks, datatags, dataset
+from datamaestro.download.single import FileDownloader
+from datamaestro.utils import HashCheck
+
+from datamaestro.data.ml import Supervised
+from datamaestro_text.data.conversation.canard import CanardDataset
+
+
+@datatags("conversation", "context", "query")
+@datatasks("query rewriting")
+@dataset(url="https://sites.google.com/view/qanta/projects/canard", id="")
+class Main(Dataset):
+    """Question-in-context rewriting
+
+    CANARD is a dataset for question-in-context rewriting that consists of
+    questions each given in a dialog context together with a context-independent
+    rewriting of the question. The context of each question is the dialog
+    utterances that precede the question. CANARD can be used to evaluate
+    question rewriting models that handle important linguistic phenomena such as
+    co-reference and ellipsis resolution.
+
+    Each dataset is an instance of :class:`datamaestro_text.data.conversation.CanardDataset`
+    """
+
+    TRAIN = FileDownloader(
+        "train.json",
+        "https://raw.githubusercontent.com/aagohary/canard/refs/heads/master/data/release/train.json",
+        checker=HashCheck("73624ac646fb81e09b0fd7f01370ada3"),
+    )
+    DEV = FileDownloader(
+        "dev.json",
+        "https://raw.githubusercontent.com/aagohary/canard/refs/heads/master/data/release/dev.json",
+        checker=HashCheck("c84525631a83bc771c58ff31f4a9b601"),
+    )
+    TEST = FileDownloader(
+        "test.json",
+        "https://raw.githubusercontent.com/aagohary/canard/refs/heads/master/data/release/test.json",
+        checker=HashCheck("3fc14d0078e7a5056f5da571728f024e"),
+    )
+
+    def config(self) -> Supervised:
+        return Supervised.C(
+            train=CanardDataset.C(path=self.TRAIN.path),
+            validation=CanardDataset.C(path=self.DEV.path),
+            test=CanardDataset.C(path=self.TEST.path),
+        )

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.3}/src/datamaestro_text/config/com/github/apple/ml-qrecc.py

@@ -3,10 +3,10 @@
 import re
 import json
 from pathlib import Path
-from datamaestro.definitions import datatasks, datatags, dataset
+from datamaestro.definitions import Dataset, datatasks, datatags, dataset
 from datamaestro.data.ml import Supervised
 from datamaestro.download import reference
-from datamaestro.download.archive import zipdownloader
+from datamaestro.download.archive import ZipDownloader
 from datamaestro.download.wayback import wayback_documents
 from datamaestro.utils import HashCheck
 from datamaestro_text.data.conversation.qrecc import QReCCDataset
@@ -19,17 +19,12 @@ from datamaestro_text.datasets.irds.helpers import lz4docstore_builder
 
 @datatags("conversation", "context", "query")
 @datatasks("query rewriting")
-@zipdownloader(
-    "data",
-    "https://github.com/apple/ml-qrecc/raw/main/dataset/qrecc_data.zip",
-    checker=HashCheck("f88fcc7ef3678cd6312080389c8abd67"),
-)
 @dataset(
     url="https://github.com/apple/ml-qrecc",
     doi="https://doi.org/10.48550/arXiv.2010.04898",
     id="",
 )
-def main(data: Path) -> Supervised[QReCCDataset, None, QReCCDataset]:
+class Main(Dataset):
     """Open-Domain Question Answering Goes Conversational via Question Rewriting
 
     We introduce QReCC (Question Rewriting in Conversational Context), an
@@ -39,34 +34,44 @@ def main(data: Path) -> Supervised[QReCCDataset, None, QReCCDataset]:
     answering that includes the individual subtasks of question rewriting,
     passage retrieval and reading comprehension
     """
-    return Supervised.C(
-        train=QReCCDataset.C(path=data / "qrecc_train.json"),
-        test=QReCCDataset.C(path=data / "qrecc_test.json"),
+
+    DATA = ZipDownloader(
+        "data",
+        "https://github.com/apple/ml-qrecc/raw/main/dataset/qrecc_data.zip",
+        checker=HashCheck("f88fcc7ef3678cd6312080389c8abd67"),
     )
 
+    def config(self) -> Supervised:
+        return Supervised.C(
+            train=QReCCDataset.C(path=self.DATA.path / "qrecc_train.json"),
+            test=QReCCDataset.C(path=self.DATA.path / "qrecc_test.json"),
+        )
+
 
 @dataset(
     url="https://github.com/apple/ml-qrecc",
     doi="https://doi.org/10.48550/arXiv.2010.04898",
 )
-class Content(LZ4JSONLDocumentStore):
+class Content(Dataset):
     """QReCC mentionned URLs content"""
 
-    @staticmethod
-    def __create_dataset__(dataset, options=None):
-        ds = reference(reference=main).setup(dataset, options)
-        documents_path = wayback_documents(
-            "20191127", lambda: Content._urls(ds), name="wayback.jsonl"
-        ).setup(dataset, options)
+    MAIN = reference(reference=Main)
 
-        store_path = lz4docstore_builder(
-            "store",
-            lambda: Content._documents(documents_path),
-            SimpleJsonDocument,
-            "id",
-        ).setup(dataset, options)
+    WAYBACK_DOCS = wayback_documents(
+        "20191127",
+        lambda: Content._urls(Content.MAIN.prepare()),
+        name="wayback.jsonl",
+    )
+
+    STORE = lz4docstore_builder(
+        "store",
+        lambda: Content._documents(Content.WAYBACK_DOCS.path),
+        SimpleJsonDocument,
+        "id",
+    )
 
-        return Content.C(jsonl_path=store_path)
+    def config(self) -> LZ4JSONLDocumentStore:
+        return LZ4JSONLDocumentStore.C(jsonl_path=self.STORE.path)
 
     @staticmethod
     def _documents(path: Path):

datamaestro_text-2026.2.3/src/datamaestro_text/config/com/github/ikat.py

@@ -0,0 +1,135 @@
+# See documentation on https://datamaestro.readthedocs.io
+
+from datamaestro.download import reference
+from datamaestro.definitions import Dataset, datatasks, datatags, dataset
+from datamaestro_text.data.conversation.base import ConversationUserTopics
+from datamaestro_text.data.ir import Adhoc
+
+from datamaestro.utils import HashCheck
+from datamaestro.context import DatafolderPath
+from datamaestro.download.single import FileDownloader
+from datamaestro_text.data.conversation.ikat import IkatConversations
+from datamaestro.download.links import linkfolder
+
+from datamaestro_text.data.ir.stores import IKatClueWeb22DocumentStore
+from datamaestro_text.data.ir.trec import TrecAdhocAssessments
+from datamaestro_text.datasets.irds.helpers import lz4docstore_builder
+
+
+@dataset()
+class Clueweb22(Dataset):
+    # Number of documents in the dataset
+    count = 116_838_987
+
+    JSONL_FOLDER = linkfolder(
+        "documents", [DatafolderPath("gov.nist.trec.ikat.clueweb22", "jsonl")]
+    )
+
+    STORE_PATH = lz4docstore_builder(
+        "store",
+        IKatClueWeb22DocumentStore.generator(
+            JSONL_FOLDER,
+            "ikat_2023_passages_jsonl.sha256sums",
+            "ikat_2023_passages_hashes.tsv.bz2",
+        ),
+        IKatClueWeb22DocumentStore.Document,
+        "id",
+        count_hint=count,
+    )
+
+    def config(self) -> IKatClueWeb22DocumentStore:
+        return IKatClueWeb22DocumentStore.C(path=self.STORE_PATH.path, count=self.count)
+
+
+@datatags("conversation", "context", "query")
+@datatasks("conversational search", "query rewriting")
+@dataset(
+    id=".2025",
+    url="https://github.com/irlabamsterdam/iKAT/tree/main/2025",
+)
+class Test2025(Dataset):
+    """Question-in-context rewriting
+
+    iKAT is a test dataset for question-in-context rewriting that consists of
+    questions each given in a dialog context together with a context-independent
+    rewriting of the question.
+    """
+
+    DOCUMENTS = reference(varname="documents", reference=Clueweb22)
+    TOPICS = FileDownloader(
+        "topics.json",
+        "https://raw.githubusercontent.com/irlabamsterdam/iKAT/refs/heads/main/2025/data/2025_test_topics.json",
+        checker=HashCheck("16f8444a8d0a8dfe0090f478f185a63c"),
+    )
+
+    def config(self) -> Adhoc:
+        return Adhoc.C(
+            topics=ConversationUserTopics.C(
+                conversations=IkatConversations.C(path=self.TOPICS.path)
+            ),
+            # TODO: add when available
+            assessments=TrecAdhocAssessments.C(path="/to/do"),
+            documents=self.DOCUMENTS.prepare(),
+        )
+
+
+@datatags("conversation", "context", "query")
+@datatasks("conversational search", "query rewriting")
+@dataset(
+    id=".2024",
+    url="https://github.com/irlabamsterdam/iKAT/tree/main/2024",
+)
+class Test2024(Dataset):
+    """iKAT 2024 dataset"""
+
+    DOCUMENTS = reference(varname="documents", reference=Clueweb22)
+    QRELS = FileDownloader(
+        "qrels",
+        "https://trec.nist.gov/data/ikat/2024-qrels.txt",
+        checker=HashCheck("57f958903ed1c12bbac207f62800814f"),
+    )
+    TOPICS = FileDownloader(
+        "topics.json",
+        "https://raw.githubusercontent.com/irlabamsterdam/iKAT/refs/heads/main/2024/data/2024_test_topics.json",
+        checker=HashCheck("ad45bc6e7add2081d69ea60a0a4d1203"),
+    )
+
+    def config(self) -> Adhoc:
+        return Adhoc.C(
+            topics=ConversationUserTopics.C(
+                conversations=IkatConversations.C(path=self.TOPICS.path)
+            ),
+            assessments=TrecAdhocAssessments.C(path=self.QRELS.path),
+            documents=self.DOCUMENTS.prepare(),
+        )
+
+
+@datatags("conversation", "context", "query")
+@datatasks("conversational search", "query rewriting")
+@dataset(
+    id=".2023",
+    url="https://github.com/irlabamsterdam/iKAT/tree/main/2023",
+)
+class Test2023(Dataset):
+    """iKAT 2023 dataset"""
+
+    DOCUMENTS = reference(varname="documents", reference=Clueweb22)
+    QRELS = FileDownloader(
+        "qrels",
+        "https://trec.nist.gov/data/ikat/2023-qrels.all-turns.txt",
+        checker=HashCheck("79dc121bab25b2245e52a53263e5ad1f"),
+    )
+    TOPICS = FileDownloader(
+        "topics.json",
+        "https://raw.githubusercontent.com/irlabamsterdam/iKAT/refs/heads/main/2023/data/2023_test_topics.json",
+        checker=HashCheck("684fa0197cdec8c3cfb6a2e586ab83f6"),
+    )
+
+    def config(self) -> Adhoc:
+        return Adhoc.C(
+            topics=ConversationUserTopics.C(
+                conversations=IkatConversations.C(path=self.TOPICS.path)
+            ),
+            assessments=TrecAdhocAssessments.C(path=self.QRELS.path),
+            documents=self.DOCUMENTS.prepare(),
+        )

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.3}/src/datamaestro_text/config/com/github/prdwb/orconvqa.py

@@ -4,8 +4,8 @@ import gzip
 import json
 from pathlib import Path
 from typing import Iterator
-from datamaestro.definitions import datatasks, datatags, dataset
-from datamaestro.download.single import filedownloader
+from datamaestro.definitions import Dataset, datatasks, datatags, dataset
+from datamaestro.download.single import FileDownloader
 from datamaestro.utils import HashCheck
 
 
@@ -18,26 +18,10 @@ from datamaestro_text.datasets.irds.helpers import lz4docstore_downloader
 
 @datatags("conversation", "context", "query")
 @datatasks("query rewriting")
-@filedownloader(
-    "train.jsonl",
-    "https://ciir.cs.umass.edu/downloads/ORConvQA/preprocessed/train.txt",
-    checker=HashCheck("7513a9ef12d8b7a4471166dc4fef77b7"),
-)
-@filedownloader(
-    "dev.jsonl",
-    "https://ciir.cs.umass.edu/downloads/ORConvQA/preprocessed/dev.txt",
-    checker=HashCheck("7765658995cc9ffd5eb39a400d814b20"),
-)
-@filedownloader(
-    "test.jsonl",
-    "https://ciir.cs.umass.edu/downloads/ORConvQA/preprocessed/test.txt",
-    checker=HashCheck("0cf3a755f06297b9c02e7db45f8dc8be"),
-)
 @dataset(
-    Supervised,
     url="https://github.com/prdwb/orconvqa-release",
 )
-def preprocessed(train, dev, test):
+class Preprocessed(Dataset):
     """Open-Retrieval Conversational Question Answering datasets
 
     OrConvQA is an aggregation of three existing datasets:
@@ -48,11 +32,29 @@ def preprocessed(train, dev, test):
 
     Each dataset is an instance of :class:`datamaestro_text.data.conversation.OrConvQADataset`
     """
-    return {
-        "train": OrConvQADataset.C(path=train),
-        "validation": OrConvQADataset.C(path=dev),
-        "test": OrConvQADataset.C(path=test),
-    }
+
+    TRAIN = FileDownloader(
+        "train.jsonl",
+        "https://ciir.cs.umass.edu/downloads/ORConvQA/preprocessed/train.txt",
+        checker=HashCheck("7513a9ef12d8b7a4471166dc4fef77b7"),
+    )
+    DEV = FileDownloader(
+        "dev.jsonl",
+        "https://ciir.cs.umass.edu/downloads/ORConvQA/preprocessed/dev.txt",
+        checker=HashCheck("7765658995cc9ffd5eb39a400d814b20"),
+    )
+    TEST = FileDownloader(
+        "test.jsonl",
+        "https://ciir.cs.umass.edu/downloads/ORConvQA/preprocessed/test.txt",
+        checker=HashCheck("0cf3a755f06297b9c02e7db45f8dc8be"),
+    )
+
+    def config(self) -> Supervised:
+        return Supervised.C(
+            train=OrConvQADataset.C(path=self.TRAIN.path),
+            validation=OrConvQADataset.C(path=self.DEV.path),
+            test=OrConvQADataset.C(path=self.TEST.path),
+        )
 
 
 def orConvQADocumentReader(source: Path) -> Iterator[OrConvQADocumentStore.NAMED_TUPLE]:
@@ -63,21 +65,10 @@ def orConvQADocumentReader(source: Path) -> Iterator[OrConvQADocumentStore.NAMED
             yield OrConvQADocumentStore.NAMED_TUPLE(**data)
 
 
-@lz4docstore_downloader(
-    "all_blocks",
-    "https://ciir.cs.umass.edu/downloads/ORConvQA/all_blocks.txt.gz",
-    orConvQADocumentReader,
-    OrConvQADocumentStore.NAMED_TUPLE,
-    "id",
-    checker=HashCheck("1095a3408690e7bbe4d8a87a2bae6356"),
-    size=5_086_902_800,
-    count_hint=11_377_951,
-)
 @dataset(
-    OrConvQADocumentStore,
     url="https://github.com/prdwb/orconvqa-release",
 )
-def passages(all_blocks):
+class Passages(Dataset):
     """orConvQA wikipedia files
 
     OrConvQA is an aggregation of three existing datasets:
@@ -86,4 +77,17 @@ def passages(all_blocks):
     1. the CANARD dataset that consists of context-independent rewrites of QuAC questions, and
     3. the Wikipedia corpus that serves as the knowledge source of answering questions.
     """
-    return {"path": all_blocks, "count": 11_377_951}
+
+    ALL_BLOCKS = lz4docstore_downloader(
+        "all_blocks",
+        "https://ciir.cs.umass.edu/downloads/ORConvQA/all_blocks.txt.gz",
+        orConvQADocumentReader,
+        OrConvQADocumentStore.NAMED_TUPLE,
+        "id",
+        checker=HashCheck("1095a3408690e7bbe4d8a87a2bae6356"),
+        size=5_086_902_800,
+        count_hint=11_377_951,
+    )
+
+    def config(self) -> OrConvQADocumentStore:
+        return OrConvQADocumentStore.C(path=self.ALL_BLOCKS.path, count=11_377_951)