datamaestro-text 2025.5.13__tar.gz → 2025.6.30__tar.gz

{datamaestro_text-2025.5.13 → datamaestro_text-2025.6.30}/.gitignore

@@ -1,4 +1,5 @@
 *.egg-info
+.DS_Store
 .vscode
 __pycache__
 site

{datamaestro_text-2025.5.13/src/datamaestro_text.egg-info → datamaestro_text-2025.6.30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datamaestro-text
-Version: 2025.5.13
+Version: 2025.6.30
 Summary: Datamaestro module for text-related datasets
 Author-email: Benjamin Piwowarski <benjamin@piwowarski.fr>
 License: GPL-3

{datamaestro_text-2025.5.13 → datamaestro_text-2025.6.30}/docs/source/api/conversation.rst

@@ -34,9 +34,13 @@ Contextual query reformulation
 .. autoclass:: ContextualizedRewrittenQuery
     :members:
 
+CANARD Dataset 
+
 .. autoxpmconfig:: datamaestro_text.data.conversation.canard.CanardDataset
     :members: iter
 
+OrConvQA Dataset
+
 .. autoxpmconfig:: datamaestro_text.data.conversation.orconvqa.OrConvQADataset
     :members: iter
 
@@ -46,10 +50,22 @@ Contextual query reformulation
 .. autoclass:: datamaestro_text.data.conversation.orconvqa.OrConvQADatasetHistoryEntry
     :members:
 
+QReCC Dataset 
 
-
-.. autoclass:: datamaestro_text.data.conversation.orconvqa.QReCCDatasetEntry
+.. autoclass:: datamaestro_text.data.conversation.qrecc.QReCCDatasetEntry
     :members:
 
 .. autoxpmconfig:: datamaestro_text.data.conversation.qrecc.QReCCDataset
     :members: iter
+
+
+iKAT Dataset
+
+.. autoclass:: datamaestro_text.data.conversation.ikat.IkatDatasetEntry
+    :members:
+
+.. autoclass:: datamaestro_text.data.conversation.ikat.IkatConversationEntry
+    :members:
+
+.. autoxpmconfig:: datamaestro_text.data.conversation.ikat.IkatDataset
+    :members: iter

{datamaestro_text-2025.5.13 → datamaestro_text-2025.6.30}/src/datamaestro_text/config/com/github/apple/ml-qrecc.py

@@ -39,9 +39,9 @@ def main(data: Path) -> Supervised[QReCCDataset, None, QReCCDataset]:
     answering that includes the individual subtasks of question rewriting,
     passage retrieval and reading comprehension
     """
-    return Supervised(
-        train=QReCCDataset(path=data / "qrecc_train.json"),
-        test=QReCCDataset(path=data / "qrecc_test.json"),
+    return Supervised.C(
+        train=QReCCDataset.C(path=data / "qrecc_train.json"),
+        test=QReCCDataset.C(path=data / "qrecc_test.json"),
     )
 
 

datamaestro_text-2025.6.30/src/datamaestro_text/config/com/github/ikat.py

@@ -0,0 +1,38 @@
+# See documentation on https://datamaestro.readthedocs.io
+
+from datamaestro.definitions import datatasks, datatags, dataset
+from datamaestro.data.ml import Supervised
+from datamaestro.data import Base
+
+from datamaestro.utils import HashCheck
+from datamaestro.download.single import filedownloader
+from datamaestro_text.data.conversation.ikat import IkatDatasetEntry, IkatDataset
+from datamaestro_text.datasets.irds.data import (
+    SimpleJsonDocument,
+    LZ4JSONLDocumentStore,
+)
+import logging
+
+@datatags("conversation", "context", "query")
+@datatasks("query rewriting")
+@filedownloader(
+    "test.json",
+    "https://raw.githubusercontent.com/irlabamsterdam/iKAT/refs/heads/main/2025/data/2025_test_topics.json",
+    checker=HashCheck("16f8444a8d0a8dfe0090f478f185a63c"),
+)
+
+@dataset(
+    Base,
+    url="https://github.com/irlabamsterdam/iKAT/tree/main/2025",
+)
+
+def main(test) -> Supervised[IkatDataset, None, IkatDataset]:
+    """Question-in-context rewriting
+
+    iKAT is a test dataset for question-in-context rewriting that consists of 
+    questions each given in a dialog context together with a context-independent
+    rewriting of the question. 
+    One of the special features of iKAT is that it includes a Personal PKTB', 
+    """
+    logging.info("Creating iKAT dataset from %s", test)
+    return IkatDataset.C(path=test)

{datamaestro_text-2025.5.13 → datamaestro_text-2025.6.30}/src/datamaestro_text/config/com/github/prdwb/orconvqa.py

@@ -49,9 +49,9 @@ def preprocessed(train, dev, test):
     Each dataset is an instance of :class:`datamaestro_text.data.conversation.OrConvQADataset`
     """
     return {
-        "train": OrConvQADataset(path=train),
-        "validation": OrConvQADataset(path=dev),
-        "test": OrConvQADataset(path=test),
+        "train": OrConvQADataset.C(path=train),
+        "validation": OrConvQADataset.C(path=dev),
+        "test": OrConvQADataset.C(path=test),
     }
 
 

{datamaestro_text-2025.5.13 → datamaestro_text-2025.6.30}/src/datamaestro_text/data/conversation/base.py

@@ -214,8 +214,8 @@ class ConversationTreeNode(ConversationNode, ConversationTree):
 
     def __init__(self, entry):
         self.entry = entry
-        self.parent = None
-        self.children = []
+        self._parent = None
+        self._children = []
 
     def add(self, node: "ConversationTreeNode") -> "ConversationTreeNode":
         self._children.append(node)
@@ -224,10 +224,10 @@ class ConversationTreeNode(ConversationNode, ConversationTree):
 
     def conversation(self, skip_self: bool) -> ConversationHistory:
         def iterator():
-            current = self.parent if skip_self else self
+            current = self.parent() if skip_self else self
             while current is not None:
                 yield current.entry
-                current = current.parent
+                current = current.parent()
 
         return LazyList(FactoryIterable(iterator))
 

datamaestro_text-2025.6.30/src/datamaestro_text/data/conversation/ikat.py

@@ -0,0 +1,120 @@
+from typing import Iterator, List, Optional
+from attr import define, field
+import json
+import logging
+from datamaestro.data import File
+from datamaestro.record import Record
+
+from datamaestro_text.data.ir.base import (
+    IDItem,
+    SimpleTextItem,
+)
+
+
+from .base import (
+    AnswerDocumentURL,
+    AnswerEntry,
+    ConversationTree,
+    EntryType,
+    SimpleDecontextualizedItem,
+    SingleConversationTree,
+)
+from . import ConversationDataset
+
+
+
+@define(kw_only=True)
+class IkatConversationEntry:
+    """A query with past history"""
+
+    turn_id: int
+    """Turn number in the conversation"""
+
+    user_utterance: str
+    """The last issued query"""
+
+    resolved_utterance: str
+    """Manually rewritten query"""
+
+    response: str
+    """The system response to the query"""
+
+    relevant_ptkbs: List[str]
+    """The list of relevant personal knowledge bases for the query"""
+
+    citations: List[str]
+    """The list of citations for the response"""
+
+
+@define(kw_only=True)
+class IkatDatasetEntry:
+    """A query with past history"""
+
+    number: str
+    """Conversation ID"""
+
+    title: str
+    """Title of the conversation"""
+
+    ptkb: str
+    """The personal knowledge base associated with the user"""
+
+    responses: List[IkatConversationEntry] = field(
+        converter=lambda items: [IkatConversationEntry(**item) if isinstance(item, dict) else item for item in items]
+    )
+    """The list of responses to the query"""
+
+
+class IkatDataset(ConversationDataset, File):
+
+    def entries(self) -> Iterator[IkatDatasetEntry]:
+        """Reads all conversation entries from the dataset file."""
+        with self.path.open("rt") as fp:
+            raw_data = json.load(fp)
+
+        logging.debug("Loaded %d entries from %s", len(raw_data), self.path)
+        logging.debug(f"raw data has keys {raw_data[0].keys()}")
+
+        processed_data = []
+        for entry in raw_data:
+            processed_data.append(IkatDatasetEntry(**{key.lower(): value for key, value in entry.items()}))
+
+        logging.debug(f"First parsed data sample: {processed_data[0]}")
+        return iter(processed_data)
+
+    def __iter__(self) -> Iterator[ConversationTree]:
+        for entry in self.entries():
+            history: List[Record] = []
+
+            for turn in entry.responses:
+                turn: IkatConversationEntry = turn  # Ensure type is correct
+                query_id = f"{entry.number}#{turn.turn_id}"
+
+                # USER QUERY record
+                history.append(
+                    Record(
+                        IDItem(query_id),
+                        SimpleTextItem(turn.user_utterance),
+                        SimpleDecontextualizedItem(turn.resolved_utterance),
+                        EntryType.USER_QUERY,
+                    )
+                )
+
+                # Build citation info (stubbed relevance to match format)
+                relevances = {}
+                if turn.relevant_ptkbs:
+                    # Example: just use first as relevant (can be improved)
+                    relevances[0] = (0, None)  # No position info in this structure
+
+                # SYSTEM ANSWER record
+                history.append(
+                    Record(
+                        AnswerEntry(turn.response),
+                        EntryType.SYSTEM_ANSWER,
+                    )
+                )
+
+            # Ensure reverse if needed for compatibility (optional)
+            history.reverse()
+            yield SingleConversationTree(entry.number, history)
+

{datamaestro_text-2025.5.13 → datamaestro_text-2025.6.30}/src/datamaestro_text/datasets/irds/datasets.py

@@ -65,7 +65,7 @@ class QrelsDataset(Dataset):
         return True
 
     def _prepare(self, download=False) -> Documents:
-        return AdhocAssessments(id=self.fullid)
+        return AdhocAssessments.C(id=self.fullid)
 
 
 class QueriesDataset(Dataset):
@@ -78,7 +78,7 @@ class QueriesDataset(Dataset):
         return True
 
     def _prepare(self, download=False) -> Documents:
-        return Topics(id=self.fullid)
+        return Topics.C(id=self.fullid)
 
 
 # class ScoredDocuments(Dataset):
@@ -96,7 +96,7 @@ class DocumentsDataset(Dataset):
         return True
 
     def _prepare(self, download=False) -> Documents:
-        return Documents(id=self.fullid)
+        return Documents.C(id=self.fullid)
 
 
 class TrainingTripletsDataset(Dataset):
@@ -123,7 +123,7 @@ class Collection(Dataset):
     topics: QueriesDataset
 
     def _prepare(self, download=False) -> Documents:
-        return Adhoc(
+        return Adhoc.C(
             id=self.fullid,
             topics=self.topics.prepare(download),
             assessments=self.assessments.prepare(download),

{datamaestro_text-2025.5.13 → datamaestro_text-2025.6.30}/src/datamaestro_text/version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2025.5.13'
-__version_tuple__ = version_tuple = (2025, 5, 13)
+__version__ = version = '2025.6.30'
+__version_tuple__ = version_tuple = (2025, 6, 30)

{datamaestro_text-2025.5.13 → datamaestro_text-2025.6.30/src/datamaestro_text.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datamaestro-text
-Version: 2025.5.13
+Version: 2025.6.30
 Summary: Datamaestro module for text-related datasets
 Author-email: Benjamin Piwowarski <benjamin@piwowarski.fr>
 License: GPL-3

{datamaestro_text-2025.5.13 → datamaestro_text-2025.6.30}/src/datamaestro_text.egg-info/SOURCES.txt

@@ -46,6 +46,7 @@ src/datamaestro_text/config/ai/quac.yaml
 src/datamaestro_text/config/com/oscar-corpus.py
 src/datamaestro_text/config/com/sentiment140.py
 src/datamaestro_text/config/com/fastml/goodbooks-10k.yaml
+src/datamaestro_text/config/com/github/ikat.py
 src/datamaestro_text/config/com/github/aagohary/canard.py
 src/datamaestro_text/config/com/github/apple/ml-qrecc.py
 src/datamaestro_text/config/com/github/prdwb/orconvqa.py
@@ -89,6 +90,7 @@ src/datamaestro_text/data/text.py
 src/datamaestro_text/data/conversation/__init__.py
 src/datamaestro_text/data/conversation/base.py
 src/datamaestro_text/data/conversation/canard.py
+src/datamaestro_text/data/conversation/ikat.py
 src/datamaestro_text/data/conversation/orconvqa.py
 src/datamaestro_text/data/conversation/qrecc.py
 src/datamaestro_text/data/ir/__init__.py