datamaestro-text 2025.6.11__tar.gz → 2025.7.28__tar.gz

{datamaestro_text-2025.6.11 → datamaestro_text-2025.7.28}/.github/workflows/pytest.yml

@@ -25,7 +25,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
+        python -m pip install --upgrade pip setuptools
         SETUPTOOLS_SCM_PRETEND_VERSION="0.1-dev" pip install -e '.[dev]'
     - name: Lint with flake8
       run: |

{datamaestro_text-2025.6.11 → datamaestro_text-2025.7.28}/.readthedocs.yml

@@ -11,7 +11,7 @@ sphinx:
 build:
   os: ubuntu-20.04
   tools:
-    python: "3.9"
+    python: "3.10"
 
 
 # Install the package

{datamaestro_text-2025.6.11/src/datamaestro_text.egg-info → datamaestro_text-2025.7.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datamaestro-text
-Version: 2025.6.11
+Version: 2025.7.28
 Summary: Datamaestro module for text-related datasets
 Author-email: Benjamin Piwowarski <benjamin@piwowarski.fr>
 License: GPL-3
@@ -15,10 +15,10 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Requires-Python: >=3.8
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: datamaestro>=1.4.2
+Requires-Dist: datamaestro>=1.5.0
 Requires-Dist: ir_datasets>=0.5.8
 Requires-Dist: attrs
 Provides-Extra: dev

{datamaestro_text-2025.6.11 → datamaestro_text-2025.7.28}/docs/source/api/conversation.rst

@@ -26,6 +26,12 @@ Data classes
 
 .. autoclass:: ConversationTopic
 
+Conversational IR
+-----------------
+
+.. autoxpmconfig:: datamaestro_text.data.conversation.base.ConversationUserTopics
+
+
 Contextual query reformulation
 ------------------------------
 
@@ -34,9 +40,13 @@ Contextual query reformulation
 .. autoclass:: ContextualizedRewrittenQuery
     :members:
 
+CANARD Dataset
+
 .. autoxpmconfig:: datamaestro_text.data.conversation.canard.CanardDataset
     :members: iter
 
+OrConvQA Dataset
+
 .. autoxpmconfig:: datamaestro_text.data.conversation.orconvqa.OrConvQADataset
     :members: iter
 
@@ -46,10 +56,22 @@ Contextual query reformulation
 .. autoclass:: datamaestro_text.data.conversation.orconvqa.OrConvQADatasetHistoryEntry
     :members:
 
+QReCC Dataset
 
-
-.. autoclass:: datamaestro_text.data.conversation.orconvqa.QReCCDatasetEntry
+.. autoclass:: datamaestro_text.data.conversation.qrecc.QReCCDatasetEntry
     :members:
 
 .. autoxpmconfig:: datamaestro_text.data.conversation.qrecc.QReCCDataset
     :members: iter
+
+
+iKAT Dataset
+
+.. autoclass:: datamaestro_text.data.conversation.ikat.IkatConversationTopic
+    :members:
+
+.. autoclass:: datamaestro_text.data.conversation.ikat.IkatConversationEntry
+    :members:
+
+.. autoxpmconfig:: datamaestro_text.data.conversation.ikat.IkatConversations
+    :members: iter

{datamaestro_text-2025.6.11 → datamaestro_text-2025.7.28}/docs/source/api/ir.rst

@@ -36,11 +36,17 @@ Documents
 
 .. autoxpmconfig:: datamaestro_text.data.ir.Documents
     :members: iter_documents, iter_ids, documentcount
+.. autoxpmconfig:: datamaestro_text.data.ir.csv.Documents
+.. autoxpmconfig:: datamaestro_text.datasets.irds.data.LZ4DocumentStore
+
+
+Dataset-specific documents
+**************************
+
 .. autoxpmconfig:: datamaestro_text.data.ir.cord19.Documents
 .. autoxpmconfig:: datamaestro_text.data.ir.trec.TipsterCollection
-.. autoxpmconfig:: datamaestro_text.data.ir.csv.Documents
 .. autoxpmconfig:: datamaestro_text.data.ir.stores.OrConvQADocumentStore
-.. autoxpmconfig:: datamaestro_text.datasets.irds.data.LZ4DocumentStore
+.. autoxpmconfig:: datamaestro_text.data.ir.stores.IKatClueWeb22DocumentStore
 
 Assessments
 -----------

{datamaestro_text-2025.6.11 → datamaestro_text-2025.7.28}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datamaestro-text"
-requires-python = ">=3.8"
+requires-python = ">=3.10"
 keywords = ["dataset manager", "information retrieval", "experiments"]
 description = "Datamaestro module for text-related datasets"
 dynamic = ["version", "readme", "dependencies"]
@@ -44,6 +44,14 @@ fallback_version = "0.0.0-dev"
 [build-system]
 requires = ["setuptools", "setuptools-scm", "wheel"]  # PEP 508 specifications.
 
+[dependency-groups]
+dev = [
+    "docutils>=0.21.2",
+    "pytest>=8.4.1",
+    "sphinx>=8.1.3",
+    "sphobjinv>=2.3.1.3",
+]
+
 [project.entry-points."datamaestro.repositories"]
 text = "datamaestro_text:Repository"
 irds = "datamaestro_text.datasets.irds:Repository"

{datamaestro_text-2025.6.11 → datamaestro_text-2025.7.28}/requirements.txt

@@ -1,3 +1,3 @@
-datamaestro>=1.4.2
+datamaestro>=1.5.0
 ir_datasets>=0.5.8
 attrs

{datamaestro_text-2025.6.11 → datamaestro_text-2025.7.28}/src/datamaestro_text/config/com/github/apple/ml-qrecc.py

@@ -39,9 +39,9 @@ def main(data: Path) -> Supervised[QReCCDataset, None, QReCCDataset]:
     answering that includes the individual subtasks of question rewriting,
     passage retrieval and reading comprehension
     """
-    return Supervised(
-        train=QReCCDataset(path=data / "qrecc_train.json"),
-        test=QReCCDataset(path=data / "qrecc_test.json"),
+    return Supervised.C(
+        train=QReCCDataset.C(path=data / "qrecc_train.json"),
+        test=QReCCDataset.C(path=data / "qrecc_test.json"),
     )
 
 

datamaestro_text-2025.7.28/src/datamaestro_text/config/com/github/ikat.py

@@ -0,0 +1,121 @@
+# See documentation on https://datamaestro.readthedocs.io
+
+import bz2
+from datamaestro.download import reference
+from datamaestro.definitions import datatasks, datatags, dataset
+from datamaestro_text.data.conversation.base import ConversationUserTopics
+from datamaestro_text.data.ir import Adhoc
+
+from datamaestro.utils import HashCheck
+from datamaestro.context import DatafolderPath
+from datamaestro.download.single import filedownloader
+from datamaestro_text.data.conversation.ikat import IkatConversations
+from datamaestro.download.links import linkfolder
+
+from datamaestro_text.data.ir.stores import IKatClueWeb22DocumentStore
+from datamaestro_text.data.ir.trec import TrecAdhocAssessments
+from datamaestro_text.datasets.irds.helpers import lz4docstore_builder
+
+
+@dataset(as_prepare=True)
+def clueweb22(dataset, options=None) -> IKatClueWeb22DocumentStore:
+    # Number of documents in the dataset
+    count = 116_838_987
+
+    jsonl_folder = linkfolder(
+        "documents", [DatafolderPath("gov.nist.trec.ikat.clueweb22", "jsonl")]
+    ).setup(dataset, options)
+    store_path = lz4docstore_builder(
+        "store",
+        IKatClueWeb22DocumentStore.generator(
+            jsonl_folder,
+            jsonl_folder / "ikat_2023_passages_jsonl.sha256sums",
+            jsonl_folder / "ikat_2023_passages_hashes.tsv.bz2",
+        ),
+        IKatClueWeb22DocumentStore.Document,
+        "id",
+        count_hint=count,
+    ).setup(dataset, options)
+
+    return IKatClueWeb22DocumentStore.C(path=store_path, count=count)
+
+
+@datatags("conversation", "context", "query")
+@datatasks("conversational search", "query rewriting")
+@reference("documents", clueweb22)
+@filedownloader(
+    "topics.json",
+    "https://raw.githubusercontent.com/irlabamsterdam/iKAT/refs/heads/main/2025/data/2025_test_topics.json",
+    checker=HashCheck("16f8444a8d0a8dfe0090f478f185a63c"),
+)
+@dataset(
+    id="2025",
+    url="https://github.com/irlabamsterdam/iKAT/tree/main/2025",
+)
+def test_2025(topics, documents) -> Adhoc.C:
+    """Question-in-context rewriting
+
+    iKAT is a test dataset for question-in-context rewriting that consists of
+    questions each given in a dialog context together with a context-independent
+    rewriting of the question.
+    """
+    return Adhoc.C(
+        topics=ConversationUserTopics.C(conversations=IkatConversations.C(path=topics)),
+        # TODO: add when available
+        assessments=TrecAdhocAssessments.C(path="/to/do"),
+        documents=documents,
+    )
+
+
+@datatags("conversation", "context", "query")
+@datatasks("conversational search", "query rewriting")
+@reference("documents", clueweb22)
+@filedownloader(
+    "qrels",
+    "https://trec.nist.gov/data/ikat/2024-qrels.txt",
+    checker=HashCheck("57f958903ed1c12bbac207f62800814f"),
+)
+@filedownloader(
+    "topics.json",
+    "https://raw.githubusercontent.com/irlabamsterdam/iKAT/refs/heads/main/2024/data/2024_test_topics.json",
+    checker=HashCheck("ad45bc6e7add2081d69ea60a0a4d1203"),
+)
+@dataset(
+    Adhoc,
+    id="2024",
+    url="https://github.com/irlabamsterdam/iKAT/tree/main/2024",
+)
+def test_2024(topics, qrels, documents) -> Adhoc.C:
+    """iKAT 2024 dataset"""
+    return Adhoc.C(
+        topics=ConversationUserTopics.C(conversations=IkatConversations.C(path=topics)),
+        assessments=TrecAdhocAssessments.C(path=qrels),
+        documents=documents,
+    )
+
+
+@datatags("conversation", "context", "query")
+@datatasks("conversational search", "query rewriting")
+@reference("documents", clueweb22)
+@filedownloader(
+    "qrels",
+    "https://trec.nist.gov/data/ikat/2023-qrels.all-turns.txt",
+    checker=HashCheck("79dc121bab25b2245e52a53263e5ad1f"),
+)
+@filedownloader(
+    "topics.json",
+    "https://raw.githubusercontent.com/irlabamsterdam/iKAT/refs/heads/main/2023/data/2023_test_topics.json",
+    checker=HashCheck("684fa0197cdec8c3cfb6a2e586ab83f6"),
+)
+@dataset(
+    Adhoc,
+    id="2023",
+    url="https://github.com/irlabamsterdam/iKAT/tree/main/2023",
+)
+def test_2023(topics, qrels, documents) -> Adhoc.C:
+    """iKAT 2023 dataset"""
+    return Adhoc.C(
+        topics=ConversationUserTopics.C(conversations=IkatConversations.C(path=topics)),
+        assessments=TrecAdhocAssessments.C(path=qrels),
+        documents=documents,
+    )

{datamaestro_text-2025.6.11 → datamaestro_text-2025.7.28}/src/datamaestro_text/config/com/sentiment140.py

@@ -26,7 +26,7 @@ def english(dir):
 
     If you use this data, please cite Sentiment140 as your source.
     """
-    return {
-        "train": Generic(path=dir / "training.1600000.processed.noemoticon.csv"),
-        "test": Generic(path=dir / "testdata.manual.2009.06.14.csv"),
-    }
+    return Supervised.C(
+        train=Generic(path=dir / "training.1600000.processed.noemoticon.csv"),
+        test=Generic(path=dir / "testdata.manual.2009.06.14.csv"),
+    )

{datamaestro_text-2025.6.11 → datamaestro_text-2025.7.28}/src/datamaestro_text/config/gov/nist/trec/tipster.py

@@ -17,7 +17,6 @@ of three CD-ROMs of SGML encoded documents distributed by LDC plus queries and a
 See also https://trec.nist.gov/data/docs_eng.html and https://trec.nist.gov/data/intro_eng.html
 """
 
-from datamaestro.data import Base
 from datamaestro_text.data.ir.trec import TipsterCollection
 from datamaestro.download.links import linkfolder
 from datamaestro.definitions import (

{datamaestro_text-2025.6.11 → datamaestro_text-2025.7.28}/src/datamaestro_text/data/conversation/base.py

@@ -1,10 +1,13 @@
 from abc import ABC, abstractmethod
 from enum import Enum
+from datamaestro_text.data.ir.base import IDItem, SimpleTextItem
+from experimaestro import Param
 from typing import Dict, Generic, Iterator, List, Optional, Sequence, Tuple
 from attr import define
+from datamaestro.record import record_type
 from datamaestro.data import Base
 from datamaestro.record import Record, Item
-from datamaestro_text.data.ir import TopicRecord
+from datamaestro_text.data.ir import TopicRecord, Topics
 from datamaestro_text.utils.iter import FactoryIterable, LazyList, RangeView
 
 # ---- Basic types
@@ -120,20 +123,17 @@ class ConversationNode:
         ...
 
     @abstractmethod
-    def parent(self) -> Optional["ConversationNode"]:
-        ...
+    def parent(self) -> Optional["ConversationNode"]: ...
 
     @abstractmethod
-    def children(self) -> List["ConversationNode"]:
-        ...
+    def children(self) -> List["ConversationNode"]: ...
 
 
 class ConversationTree(ABC):
     """Represents a conversation tree"""
 
     @abstractmethod
-    def root(self) -> ConversationNode:
-        ...
+    def root(self) -> ConversationNode: ...
 
     @abstractmethod
     def __iter__(self) -> Iterator[ConversationNode]:
@@ -214,8 +214,8 @@ class ConversationTreeNode(ConversationNode, ConversationTree):
 
     def __init__(self, entry):
         self.entry = entry
-        self.parent = None
-        self.children = []
+        self._parent = None
+        self._children = []
 
     def add(self, node: "ConversationTreeNode") -> "ConversationTreeNode":
         self._children.append(node)
@@ -224,10 +224,10 @@ class ConversationTreeNode(ConversationNode, ConversationTree):
 
     def conversation(self, skip_self: bool) -> ConversationHistory:
         def iterator():
-            current = self.parent if skip_self else self
+            current = self.parent() if skip_self else self
             while current is not None:
                 yield current.entry
-                current = current.parent
+                current = current.parent()
 
         return LazyList(FactoryIterable(iterator))
 
@@ -253,5 +253,30 @@ class ConversationDataset(Base, ABC):
     @abstractmethod
     def __iter__(self) -> Iterator[ConversationTree]:
         """Return an iterator over conversations"""
-        for i in range(len(self)):
-            yield self.get(i)
+        ...
+
+
+class ConversationUserTopics(Topics):
+    """Extract user topics from conversations"""
+
+    conversations: Param[ConversationDataset]
+
+    topic_recordtype = record_type(IDItem, SimpleTextItem)
+
+    def iter(self) -> Iterator[TopicRecord]:
+        """Returns an iterator over topics"""
+        # Extracts topics from conversations, Each user query is a topic (can perform retrieval on it)
+        # TODO: merge with xpmir.learning.DatasetConversationBase -> same logic
+        
+        records: List[TopicRecord] = []
+        for conversation in self.conversations.__iter__():
+            nodes = [
+                node
+                for node in conversation
+                if node.entry[EntryType] == EntryType.USER_QUERY
+            ]
+            for node in nodes:
+                records.append(
+                    node.entry.update(ConversationHistoryItem(node.history()))
+                )
+        return iter(records)

datamaestro_text-2025.7.28/src/datamaestro_text/data/conversation/ikat.py

@@ -0,0 +1,145 @@
+from typing import Iterator, List
+from attr import define, field
+import json
+import logging
+from datamaestro.data import File
+from datamaestro.record import Record
+
+from datamaestro_text.data.ir import Topics
+from datamaestro_text.data.ir.base import (
+    IDItem,
+    SimpleTextItem,
+)
+
+
+from .base import (
+    AnswerEntry,
+    ConversationTree,
+    EntryType,
+    SimpleDecontextualizedItem,
+    SingleConversationTree,
+)
+from . import ConversationDataset
+
+# Keys to change in the dataset entries for compatibility across different years
+
+KEY_MAPPINGS = {
+    # Keys to replace: Target Key
+    "turns": "responses",
+    "utterance": "user_utterance",
+    "ptkb_provenance": "relevant_ptkbs",
+    "response_provenance": "citations",
+}
+
+
+def norm_dict(entry: dict) -> dict:
+    """Convert keys in the entry to match the expected format."""
+    normalized = {}
+    for k, v in entry.items():
+        # Check for direct mapping, then try lowercase mapping
+        new_key = KEY_MAPPINGS.get(k) or KEY_MAPPINGS.get(k.lower()) or k.lower()
+        normalized[new_key] = v
+    return normalized
+
+
+@define(kw_only=True)
+class IkatConversationEntry:
+    """A query with past history"""
+
+    turn_id: int
+    """Turn number in the conversation"""
+
+    user_utterance: str
+    """The last issued query"""
+
+    resolved_utterance: str
+    """Manually rewritten query"""
+
+    response: str
+    """The system response to the query"""
+
+    relevant_ptkbs: List[str]
+    """The list of relevant personal knowledge bases for the query"""
+
+    citations: List[str]
+    """The list of citations for the response"""
+
+
+@define(kw_only=True)
+class IkatConversationTopic:
+    """A query with past history"""
+
+    number: str
+    """Conversation ID"""
+
+    title: str
+    """Title of the conversation"""
+
+    ptkb: str
+    """The personal knowledge base associated with the user"""
+
+    responses: List[IkatConversationEntry] = field(
+        converter=lambda items: [
+            IkatConversationEntry(**item) if isinstance(item, dict) else item
+            for item in map(norm_dict, items)
+        ]
+    )
+    """The list of responses to the query"""
+
+
+class IkatConversations(ConversationDataset, File):
+    """A dataset containing conversations from the IKAT project"""
+
+    """Keys to change in the dataset entries for compatibility across different years"""
+
+    def entries(self) -> Iterator[IkatConversationTopic]:
+        """Reads all conversation entries from the dataset file."""
+        with self.path.open("rt") as fp:
+            raw_data = json.load(fp)
+
+        logging.debug("Loaded %d entries from %s", len(raw_data), self.path)
+        logging.debug(f"raw data has keys {raw_data[0].keys()}")
+
+        for entry in raw_data:
+            try:
+                normalized_entry = norm_dict(entry)
+                yield IkatConversationTopic(**normalized_entry)
+            except Exception as e:
+                logging.warning(f"Failed to parse entry: {e}")
+                raise e
+
+    def __iter__(self) -> Iterator[ConversationTree]:
+        for entry in self.entries():
+            history: List[Record] = []
+
+            for turn in entry.responses:
+                turn: IkatConversationEntry = turn  # Ensure type is correct
+                query_id = f"{entry.number}_{turn.turn_id}"
+
+                # USER QUERY record
+                history.append(
+                    Record(
+                        IDItem(query_id),
+                        SimpleTextItem(turn.user_utterance),
+                        SimpleDecontextualizedItem(turn.resolved_utterance),
+                        EntryType.USER_QUERY,
+                    )
+                )
+
+                # Build citation info (stubbed relevance to match format)
+                relevances = {}
+                if turn.relevant_ptkbs:
+                    # Example: just use first as relevant (can be improved)
+                    relevances[0] = (0, None)  # No position info in this structure
+
+                # SYSTEM ANSWER record
+                history.append(
+                    Record(
+                        AnswerEntry(turn.response),
+                        EntryType.SYSTEM_ANSWER,
+                    )
+                )
+
+            # Ensure reverse if needed for compatibility (optional)
+            history.reverse()
+            yield SingleConversationTree(entry.number, history)

{datamaestro_text-2025.6.11 → datamaestro_text-2025.7.28}/src/datamaestro_text/data/ir/__init__.py

@@ -2,9 +2,10 @@
 
 from abc import ABC, abstractmethod
 from functools import cached_property
+import logging
 from pathlib import Path
 from attrs import define
-from typing import Callable, Dict, Iterator, List, Optional, Tuple, Type
+from typing import Callable, Dict, Iterator, List, Optional, Tuple, Type, TYPE_CHECKING
 import random
 from experimaestro import Config
 from datamaestro.definitions import datatasks, Param, Meta
@@ -28,6 +29,9 @@ from .base import (  # noqa: F401
     AdhocAssessedTopic,
 )
 
+#: A adhoc run dictionary (query id -> doc id -> score)
+AdhocRunDict = dict[str, dict[str, float]]
+
 
 class Documents(Base):
     """A set of documents with identifiers
@@ -45,6 +49,22 @@ class Documents(Base):
     def iter_documents(self) -> Iterator[DocumentRecord]:
         return self.iter()
 
+    def iter_documents_from(self, start=0) -> Iterator[DocumentRecord]:
+        """Iterate over a range of documents
+
+        Can be specialized in a subclass for faster access
+
+        :param start: The starting document, defaults to 0
+        :return: An iterator
+        """
+        iter = self.iter()
+        if start > 0:
+            logging.info("skipping %d documents", start + 1)
+            for _ in range(start + 1):
+                next(iter)
+
+        return iter
+
     def iter_ids(self) -> Iterator[str]:
         """Iterates over document ids
 
@@ -168,7 +188,10 @@ class AdhocAssessments(Base, ABC):
 class AdhocRun(Base):
     """IR adhoc run"""
 
-    pass
+    @abstractmethod
+    def get_dict(self) -> "AdhocRunDict":
+        """Get the run as a dictionary query ID -> doc ID -> score"""
+        ...
 
 
 class AdhocResults(Base):

{datamaestro_text-2025.6.11 → datamaestro_text-2025.7.28}/src/datamaestro_text/data/ir/base.py

@@ -43,6 +43,7 @@ class IDItem(Item, ABC):
 
     id: str
 
+
 @define
 class UrlItem(Item):
     """An url item"""
@@ -70,7 +71,7 @@ class AdhocAssessedTopic:
     """List of assessments for this topic"""
 
 
-def create_record(*items: Item, id: str = None, text: str = None):
+def create_record(*items: Item, id: str = None, text: str = None) -> Record:
     """Easy creation of a text/id item"""
     extra_items = []
     if id is not None:

{datamaestro_text-2025.6.11 → datamaestro_text-2025.7.28}/src/datamaestro_text/data/ir/formats.py

@@ -99,7 +99,7 @@ class WapoDocument(TextItem):
     body_media: Tuple[WapoDocMedia, ...]
 
     @cached_property
-    def text(self):        
+    def text(self):
         return f"{self.title} {self.body_paras_html}"
 
 
@@ -132,11 +132,15 @@ class DprW100Doc(TextItem):
     text: str
     title: str
 
-@define 
+
+@define
 class MsMarcoV2Passage(TextItem):
     text: str
     spans: Tuple[Tuple[int, int], ...]
     msmarco_document_id: str
+
+
+@define
 class Touche2020(TextItem):
     text: str
     title: str
@@ -204,11 +208,13 @@ class TrecTopic(SimpleTextItem):
 
 TrecTopicRecord = record_type(IDItem, TrecTopic)
 
+
 @define
 class DprW100Query(TextItem):
     text: str
     answers: Tuple[str]
 
+
 @define
 class TrecBackgroundLinkingQuery(IDItem):
     query_id: str