datamaestro-text 2024.3.10__tar.gz → 2024.5.31__tar.gz

datamaestro_text-2024.5.31/.pre-commit-config.yaml

@@ -0,0 +1,19 @@
+repos:
+- hooks:
+  - id: check-yaml
+  - id: end-of-file-fixer
+  - id: trailing-whitespace
+  repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.4.0
+- hooks:
+  - exclude: ^src/experimaestro/server/data
+    id: black
+  repo: https://github.com/psf/black
+  rev: 23.1.0
+- hooks:
+  - additional_dependencies:
+    - flake8-print
+    - flake8-fixme
+    id: flake8
+  repo: https://github.com/pycqa/flake8
+  rev: 6.0.0

{datamaestro-text-2024.3.10/src/datamaestro_text.egg-info → datamaestro_text-2024.5.31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamaestro-text
-Version: 2024.3.10
+Version: 2024.5.31
 Summary: Datamaestro module for text-related datasets
 Author-email: Benjamin Piwowarski <benjamin@piwowarski.fr>
 License: GPL-3
@@ -18,7 +18,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: datamaestro>=1.1.0
+Requires-Dist: datamaestro>=1.1.1
 Requires-Dist: ir_datasets
 Requires-Dist: attrs
 

{datamaestro-text-2024.3.10 → datamaestro_text-2024.5.31}/docs/source/api/conversation.rst

@@ -45,3 +45,11 @@ Contextual query reformulation
 
 .. autoclass:: datamaestro_text.data.conversation.orconvqa.OrConvQADatasetHistoryEntry
     :members:
+
+
+
+.. autoclass:: datamaestro_text.data.conversation.orconvqa.QReCCDatasetEntry
+    :members:
+
+.. autoxpmconfig:: datamaestro_text.data.conversation.qrecc.QReCCDataset
+    :members: iter

{datamaestro-text-2024.3.10 → datamaestro_text-2024.5.31}/docs/source/datasets/conversation.rst

@@ -8,3 +8,5 @@ Contextualized Query Rewriting
 .. dm:datasets:: com.github.aagohary.canard text
 
 .. dm:datasets:: com.github.prdwb.orconvqa text
+
+.. dm:datasets:: com.github.apple.ml-qrecc text

{datamaestro-text-2024.3.10 → datamaestro_text-2024.5.31}/docs/source/datasets/irds.rst

@@ -8,9 +8,10 @@ version of `ir-datasets` is more ancient or newer than the one used at generatio
 Data types
 ----------
 
-.. autoxpmconfig:: xpmir.datasets.irds.data.Topics
-.. autoxpmconfig:: xpmir.datasets.irds.data.Documents
-.. autoxpmconfig:: xpmir.datasets.irds.data.AdhocAssessments
+.. autoxpmconfig:: datamaestro_text.datasets.irds.data.Topics
+.. autoxpmconfig:: datamaestro_text.datasets.irds.data.Documents
+.. autoxpmconfig:: datamaestro_text.datasets.irds.data.AdhocAssessments
+.. autoxpmconfig:: datamaestro_text.datasets.irds.data.LZ4DocumentStore
 
 
 List of datasets

datamaestro_text-2024.5.31/requirements.txt

@@ -0,0 +1,3 @@
+datamaestro>=1.1.1
+ir_datasets
+attrs

datamaestro_text-2024.5.31/src/datamaestro_text/config/com/github/apple/ml-qrecc.py

@@ -0,0 +1,37 @@
+# See documentation on https://datamaestro.readthedocs.io
+
+from pathlib import Path
+from datamaestro.definitions import datatasks, datatags, dataset
+from datamaestro.data.ml import Supervised
+from datamaestro.download.archive import zipdownloader
+from datamaestro.utils import HashCheck
+from datamaestro_text.data.conversation.qrecc import QReCCDataset
+
+
+@datatags("conversation", "context", "query")
+@datatasks("query rewriting")
+@zipdownloader(
+    "data",
+    "https://github.com/apple/ml-qrecc/raw/main/dataset/qrecc_data.zip",
+    checker=HashCheck("f88fcc7ef3678cd6312080389c8abd67"),
+)
+@dataset(
+    Supervised[QReCCDataset, None, QReCCDataset],
+    url="https://github.com/apple/ml-qrecc",
+    doi="https://doi.org/10.48550/arXiv.2010.04898",
+    id="",
+)
+def main(data: Path):
+    """Open-Domain Question Answering Goes Conversational via Question Rewriting
+
+    We introduce QReCC (Question Rewriting in Conversational Context), an
+    end-to-end open-domain question answering dataset comprising of 14K
+    conversations with 81K question-answer pairs. The goal of this dataset is to
+    provide a challenging benchmark for end-to-end conversational question
+    answering that includes the individual subtasks of question rewriting,
+    passage retrieval and reading comprehension
+    """
+    return {
+        "train": QReCCDataset(path=data / "qrecc_train.json"),
+        "test": QReCCDataset(path=data / "qrecc_test.json"),
+    }

{datamaestro-text-2024.3.10 → datamaestro_text-2024.5.31}/src/datamaestro_text/config/com/github/prdwb/orconvqa.py

@@ -1,11 +1,9 @@
 # See documentation on https://datamaestro.readthedocs.io
 
-from collections import namedtuple
 import gzip
 import json
 from pathlib import Path
-from typing import Iterator, NamedTuple
-import attrs
+from typing import Iterator
 from datamaestro.definitions import datatasks, datatags, dataset
 from datamaestro.download.single import filedownloader
 from datamaestro.utils import HashCheck
@@ -14,10 +12,7 @@ from datamaestro.utils import HashCheck
 from datamaestro_text.data.conversation.orconvqa import OrConvQADataset
 from datamaestro.data.ml import Supervised
 
-from datamaestro_text.data.ir import DocumentStore
-from datamaestro_text.data.ir.formats import OrConvQADocument
 from datamaestro_text.data.ir.stores import OrConvQADocumentStore
-from datamaestro_text.datasets.irds.data import LZ4DocumentStore
 from datamaestro_text.datasets.irds.helpers import lz4docstore_downloader
 
 
@@ -63,7 +58,9 @@ def preprocessed(train, dev, test):
 def orConvQADocumentReader(source: Path) -> Iterator[OrConvQADocumentStore.NAMED_TUPLE]:
     with gzip.open(source, "rt") as fp:
         for line in fp:
-            yield OrConvQADocumentStore.NAMED_TUPLE(**json.loads(line))
+            data = json.loads(line)
+            data["body"] = data.pop("text")
+            yield OrConvQADocumentStore.NAMED_TUPLE(**data)
 
 
 @lz4docstore_downloader(

{datamaestro-text-2024.3.10 → datamaestro_text-2024.5.31}/src/datamaestro_text/data/conversation/base.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from enum import Enum
-from typing import Dict, Generic, Iterator, List, Optional, Sequence
+from typing import Dict, Generic, Iterator, List, Optional, Sequence, Tuple
 from attr import define
 from datamaestro.data import Base
 from datamaestro.record import Record, Item
@@ -61,6 +61,20 @@ class AnswerEntry(Item):
     """The system answer"""
 
 
+@define
+class AnswerDocumentID(Item):
+    """An answer as a document ID"""
+
+    document_id: str
+
+
+@define
+class AnswerDocumentURL(Item):
+    """An answer as a document ID"""
+
+    url: str
+
+
 @define
 class RetrievedEntry(Item):
     """List of system-retrieved documents and their relevance"""
@@ -68,8 +82,8 @@ class RetrievedEntry(Item):
     documents: List[str]
     """List of retrieved documents"""
 
-    document_relevances: Optional[List[str]] = None
-    """List of retrieved documents and their relevance status"""
+    relevant_documents: Optional[Dict[int, Tuple[Optional[int], Optional[int]]]] = None
+    """List of relevance status (optional), with start/stop position"""
 
 
 @define
@@ -95,56 +109,99 @@ class ConversationHistoryItem(Item):
 
 
 class ConversationNode:
+    @abstractmethod
     def entry(self) -> Record:
         """The current conversation entry"""
         ...
 
+    @abstractmethod
     def history(self) -> ConversationHistory:
         """Preceding conversation entries, from most recent to more ancient"""
         ...
 
+    @abstractmethod
+    def parent(self) -> Optional["ConversationNode"]:
+        ...
+
+    @abstractmethod
+    def children(self) -> List["ConversationNode"]:
+        ...
+
 
-class ConversationTree:
+class ConversationTree(ABC):
+    @abstractmethod
+    def root(self) -> ConversationNode:
+        ...
+
+    @abstractmethod
     def __iter__(self) -> Iterator[ConversationNode]:
         """Iterates over conversation nodes"""
-        pass
+        ...
 
 
 # ---- A conversation tree
 
 
-class SingleConversationTree(ConversationTree):
+class SingleConversationTree(ConversationTree, ABC):
     """Simple conversations, based on a sequence of entries"""
 
     id: str
-    history: Sequence[Record]
+    history: List[Record]
 
     def __init__(self, id: Optional[str], history: List[Record]):
         """Create a simple conversation
 
-        :param history: The entries, in reverse order (i.e. more ancient first)
+        :param history: The entries, in **reverse** order (i.e. more ancient first)
         """
         self.history = history or []
+        self.id = id
 
     def add(self, entry: Record):
         self.history.insert(0, entry)
 
     def __iter__(self) -> Iterator[ConversationNode]:
-        for ix in range(len(self.history)):
+        """Iterates over the conversation (starting with the beginning)"""
+        for ix in reversed(range(len(self.history))):
             yield SingleConversationTreeNode(self, ix)
 
+    def root(self):
+        return SingleConversationTreeNode(self, len(self.history) - 1)
+
 
 @define
 class SingleConversationTreeNode(ConversationNode):
     tree: SingleConversationTree
     index: int
 
+    @property
     def entry(self) -> Record:
         return self.tree.history[self.index]
 
+    @entry.setter
+    def entry(self, record: Record):
+        try:
+            self.tree.history[self.index] = record
+        except Exception as e:
+            print(e)
+            raise
+
     def history(self) -> Sequence[Record]:
         return self.tree.history[self.index + 1 :]
 
+    def parent(self) -> ConversationNode | None:
+        return (
+            SingleConversationTreeNode(self.tree, self.index + 1)
+            if self.index < len(self.tree.history) - 1
+            else []
+        )
+
+    def children(self) -> List[ConversationNode]:
+        return (
+            [SingleConversationTreeNode(self.tree, self.index - 1)]
+            if self.index > 0
+            else []
+        )
+
 
 class ConversationTreeNode(ConversationNode, ConversationTree):
     """A conversation tree node"""
@@ -178,6 +235,15 @@ class ConversationTreeNode(ConversationNode, ConversationTree):
         for child in self.children:
             yield from child
 
+    def parent(self) -> ConversationNode | None:
+        return self.parent
+
+    def children(self) -> List[ConversationNode]:
+        return self.children
+
+    def root(self):
+        return self
+
 
 class ConversationDataset(Base, ABC):
     """A dataset made of conversations"""
@@ -186,4 +252,4 @@ class ConversationDataset(Base, ABC):
     def __iter__(self) -> Iterator[ConversationTree]:
         """Return an iterator over conversations"""
         for i in range(len(self)):
-            return self.get(i)
+            yield self.get(i)

{datamaestro-text-2024.3.10 → datamaestro_text-2024.5.31}/src/datamaestro_text/data/conversation/orconvqa.py

@@ -102,16 +102,26 @@ class OrConvQADataset(ConversationDataset, File):
             # Add to current
             history.append(
                 Record(
-                    IDItem(query_no),
+                    IDItem(entry.query_id),
                     SimpleTextItem(entry.query),
                     SimpleDecontextualizedItem(entry.rewrite),
                     EntryType.USER_QUERY,
                 )
             )
+
+            relevances = {}
+            for rank, relevance in enumerate(entry.retrieval_labels):
+                if relevance > 0:
+                    relevances[rank] = (entry.answer.answer_start, None)
+
+            assert (
+                len(relevances) <= 1
+            ), f"Too many relevance labels ({len(relevances)}) for {entry.query_id}"
+
             history.append(
                 Record(
                     AnswerEntry(entry.answer.text),
-                    RetrievedEntry(entry.evidences, entry.retrieval_labels),
+                    RetrievedEntry(entry.evidences, relevances),
                     EntryType.SYSTEM_ANSWER,
                 )
             )

datamaestro_text-2024.5.31/src/datamaestro_text/data/conversation/qrecc.py

@@ -0,0 +1,99 @@
+from typing import Iterator, List, Optional
+from attr import define
+import json
+from datamaestro.data import File
+from datamaestro.record import Record
+
+from datamaestro_text.data.ir.base import (
+    IDItem,
+    SimpleTextItem,
+)
+
+
+from .base import (
+    AnswerDocumentURL,
+    AnswerEntry,
+    ConversationTree,
+    EntryType,
+    SimpleDecontextualizedItem,
+    SingleConversationTree,
+)
+from . import ConversationDataset
+
+
+@define(kw_only=True)
+class QReCCDatasetEntry:
+    """A query with past history"""
+
+    conversation_no: int
+    """Conversation ID"""
+
+    turn_no: int
+    """The turn in the conversation"""
+
+    conversation_source: str
+    """Conversation source"""
+
+    question: str
+    """The last issued query"""
+
+    rewrite: str
+    """Manually rewritten query"""
+
+    context: List[str]
+    """The list of queries asked by the user"""
+
+    answer: str
+    """The answer"""
+
+    answer_url: str
+    """The URL containing the answer"""
+
+
+class QReCCDataset(ConversationDataset, File):
+    def entries(self) -> Iterator[QReCCDatasetEntry]:
+        """Iterates over re-written query with their context"""
+        with self.path.open("rt") as fp:
+            data = json.load(fp)
+
+        data = [
+            QReCCDatasetEntry(**{key.lower(): value for key, value in entry.items()})
+            for entry in data
+        ]
+        return iter(data)
+
+    def __iter__(self) -> Iterator[ConversationTree]:
+        history: List[Record] = []
+        current_id: Optional[str] = None
+
+        for entry in self.entries():
+            # Creates a new conversation if needed
+            if entry.conversation_no != current_id:
+                if current_id is not None:
+                    history.reverse()
+                    yield SingleConversationTree(current_id, history)
+
+                current_id = entry.conversation_no
+                history = []
+
+            # Add to current
+            history.append(
+                Record(
+                    IDItem(f"{entry.conversation_no}#{entry.turn_no}"),
+                    SimpleTextItem(entry.question),
+                    AnswerDocumentURL(entry.answer_url),
+                    SimpleDecontextualizedItem(entry.rewrite),
+                    EntryType.USER_QUERY,
+                )
+            )
+
+            history.append(
+                Record(
+                    AnswerEntry(entry.answer),
+                    EntryType.SYSTEM_ANSWER,
+                )
+            )
+
+        # Yields the last one
+        history.reverse()
+        yield SingleConversationTree(current_id, history)

{datamaestro-text-2024.3.10 → datamaestro_text-2024.5.31}/src/datamaestro_text/data/ir/__init__.py

@@ -25,6 +25,7 @@ from .base import (  # noqa: F401
     create_record,
     # Other things
     AdhocAssessment,
+    AdhocAssessedTopic,
 )
 
 
@@ -83,7 +84,7 @@ class DocumentStore(Documents):
     def document_int(self, internal_docid: int) -> DocumentRecord:
         """Returns a document given its internal ID"""
         docid = self.docid_internal2external(internal_docid)
-        return self.document(docid)
+        return self.document_ext(docid)
 
     def document_ext(self, docid: str) -> DocumentRecord:
         """Returns a document given its external ID"""
@@ -159,7 +160,7 @@ class TopicsStore(Topics):
 class AdhocAssessments(Base, ABC):
     """Ad-hoc assessments (qrels)"""
 
-    def iter(self) -> Iterator[AdhocAssessment]:
+    def iter(self) -> Iterator[AdhocAssessedTopic]:
         """Returns an iterator over assessments"""
         raise NotImplementedError(f"For class {self.__class__}")
 

{datamaestro-text-2024.3.10 → datamaestro_text-2024.5.31}/src/datamaestro_text/data/ir/stores.py

@@ -3,6 +3,7 @@ from typing import List
 from experimaestro import Constant
 import attrs
 
+from datamaestro.record import Record
 from datamaestro_text.datasets.irds.data import LZ4DocumentStore
 from datamaestro_text.data.ir.formats import OrConvQADocument
 
@@ -19,4 +20,4 @@ class OrConvQADocumentStore(LZ4DocumentStore):
     data_cls = NAMED_TUPLE
 
     def converter(self, data: NAMED_TUPLE) -> OrConvQADocument:
-        return OrConvQADocument(**data._asdict())
+        return Record(OrConvQADocument(**data._asdict()))

{datamaestro-text-2024.3.10 → datamaestro_text-2024.5.31}/src/datamaestro_text/datasets/irds/data.py

@@ -2,7 +2,7 @@ from abc import ABC, abstractmethod
 from functools import partial
 import logging
 from pathlib import Path
-from typing import Iterator, Tuple, Type, List
+from typing import Dict, Iterator, Tuple, Type, List
 import ir_datasets
 from ir_datasets.indices import PickleLz4FullStore
 from ir_datasets.formats import (
@@ -17,6 +17,7 @@ from experimaestro import Config, Param
 from experimaestro.compat import cached_property
 from experimaestro import Option
 from datamaestro.record import RecordType, record_type
+from datamaestro_text.data.conversation.base import AnswerEntry
 import datamaestro_text.data.ir as ir
 from datamaestro_text.data.ir.base import (
     Record,
@@ -254,15 +255,12 @@ class LZ4DocumentStore(ir.DocumentStore):
         return getattr(self._docs[ix], self.store._id_field)
 
     def document_ext(self, docid: str) -> DocumentRecord:
-        return self.converter(self.document_recordtype, self.store.get(docid))
+        return self.converter(self.store.get(docid))
 
     def documents_ext(self, docids: List[str]) -> DocumentRecord:
         """Returns documents given their external IDs (optimized for batch)"""
         retrieved = self.store.get_many(docids)
-        return [
-            self.converter(self.document_recordtype, retrieved[docid])
-            for docid in docids
-        ]
+        return [self.converter(retrieved[docid]) for docid in docids]
 
     def converter(self, data):
         """Converts a document from LZ4 tuples to any other format"""
@@ -271,10 +269,9 @@ class LZ4DocumentStore(ir.DocumentStore):
 
     def iter(self) -> Iterator[DocumentRecord]:
         """Returns an iterator over documents"""
-        return map(
-            partial(self.converter, self.document_recordtype), self.store.__iter__()
-        )
+        return map(self.converter, self.store.__iter__())
 
+    @cached_property
     def documentcount(self):
         if self.count:
             return self.count
@@ -399,8 +396,9 @@ if hasattr(_irds.trec_cast, "Cast2022Query"):
     from datamaestro_text.data.conversation.base import (
         ConversationTreeNode,
         DecontextualizedDictItem,
-        RetrievedEntry,
+        AnswerDocumentID,
         ConversationHistoryItem,
+        EntryType,
     )
 
     class CastTopicsHandler(TopicsHandler):
@@ -428,7 +426,6 @@ if hasattr(_irds.trec_cast, "Cast2022Query"):
             """Returns an iterator over topics"""
             return iter(self.records)
 
-    class Cast2020TopicsHandler(CastTopicsHandler):
         @cached_property
         def records(self):
             try:
@@ -437,11 +434,7 @@ if hasattr(_irds.trec_cast, "Cast2022Query"):
                 conversation = []
                 records = []
 
-                for (
-                    query
-                ) in (
-                    self.dataset.dataset.queries_iter()
-                ):  # type: _irds.trec_cast.Cast2020Query
+                for query in self.dataset.dataset.queries_iter():
                     decontextualized = DecontextualizedDictItem(
                         "manual",
                         {
@@ -449,28 +442,35 @@ if hasattr(_irds.trec_cast, "Cast2022Query"):
                             "auto": query.automatic_rewritten_utterance,
                         },
                     )
+
+                    is_new_conversation = topic_number != query.topic_number
+
                     topic = Record(
                         IDItem(query.query_id),
                         SimpleTextItem(query.raw_utterance),
                         decontextualized,
                         ConversationHistoryItem(
-                            node.conversation(False) if node else []
+                            [] if is_new_conversation else node.conversation(False)
                         ),
+                        EntryType.USER_QUERY,
                     )
 
-                    if topic_number == query.topic_number:
-                        node = node.add(ConversationTreeNode(topic))
-                    else:
+                    if is_new_conversation:
                         conversation = []
                         node = ConversationTreeNode(topic)
                         topic_number = query.topic_number
+                    else:
+                        node = node.add(ConversationTreeNode(topic))
 
                     records.append(topic)
 
                     conversation.append(node)
                     node = node.add(
                         ConversationTreeNode(
-                            Record(RetrievedEntry(query.manual_canonical_result_id))
+                            Record(
+                                AnswerDocumentID(self.get_canonical_result_id(query)),
+                                EntryType.SYSTEM_ANSWER,
+                            )
                         )
                     )
                     conversation.append(node)
@@ -480,15 +480,110 @@ if hasattr(_irds.trec_cast, "Cast2022Query"):
 
             return records
 
+        @staticmethod
+        def get_canonical_result_id():
+            return None
+
+    class Cast2020TopicsHandler(CastTopicsHandler):
+        @staticmethod
+        def get_canonical_result_id(query: _irds.trec_cast.Cast2020Query):
+            return query.manual_canonical_result_id
+
+    class Cast2021TopicsHandler(CastTopicsHandler):
+        @staticmethod
+        def get_canonical_result_id(query: _irds.trec_cast.Cast2021Query):
+            return query.canonical_result_id
+
+    class Cast2022TopicsHandler(CastTopicsHandler):
+        def __init__(self, dataset):
+            self.dataset = dataset
+
+        @cached_property
+        def records(self):
+            try:
+                records = []
+                nodes: Dict[str, ConversationTreeNode] = {}
+
+                for (
+                    query
+                ) in (
+                    self.dataset.dataset.queries_iter()
+                ):  # type: _irds.trec_cast.Cast2022Query
+                    parent = nodes[query.parent_id] if query.parent_id else None
+
+                    if query.participant == "User":
+                        topic = Record(
+                            IDItem(query.query_id),
+                            SimpleTextItem(query.raw_utterance),
+                            DecontextualizedDictItem(
+                                "manual",
+                                {
+                                    "manual": query.manual_rewritten_utterance,
+                                },
+                            ),
+                            ConversationHistoryItem(
+                                parent.conversation(False) if parent else []
+                            ),
+                            EntryType.USER_QUERY,
+                        )
+                        node = ConversationTreeNode(topic)
+                        records.append(topic)
+                    else:
+                        node = ConversationTreeNode(
+                            Record(
+                                AnswerEntry(query.response),
+                                EntryType.SYSTEM_ANSWER,
+                            )
+                        )
+
+                    nodes[query.query_id] = node
+                    if parent:
+                        parent.add(node)
+            except Exception:
+                logging.exception("Error while computing topic records")
+                raise
+
+            return records
+
     Topics.HANDLERS.update(
         {
             # _irds.trec_cast.Cast2019Query: Cast2019TopicsHandler,
             _irds.trec_cast.Cast2020Query: Cast2020TopicsHandler,
-            # _irds.trec_cast.Cast2021Query: Cast2021TopicsHandler,
-            # _irds.trec_cast.Cast2022Query: Cast2022TopicsHandler
+            _irds.trec_cast.Cast2021Query: Cast2021TopicsHandler,
+            _irds.trec_cast.Cast2022Query: Cast2022TopicsHandler,
         }
     )
 
+    class CastDocHandler:
+        def check(self, cls):
+            assert issubclass(cls, _irds.trec_cast.CastDoc)
+
+        @cached_property
+        def target_cls(self):
+            return formats.TitleUrlDocument
+
+        def __call__(self, _, doc: _irds.trec_cast.CastDoc):
+            return Record(
+                IDItem(doc.doc_id), formats.SimpleTextItem(" ".join(doc.passages))
+            )
+
+    class CastPassageDocHandler:
+        def check(self, cls):
+            assert issubclass(cls, _irds.trec_cast.CastPassageDoc)
+
+        @cached_property
+        def target_cls(self):
+            return formats.TitleUrlDocument
+
+        def __call__(self, _, doc: _irds.trec_cast.CastPassageDoc):
+            return Record(
+                IDItem(doc.doc_id),
+                formats.TitleUrlDocument(doc.text, doc.title, doc.url),
+            )
+
+    Documents.CONVERTERS[_irds.trec_cast.CastDoc] = CastDocHandler()
+    Documents.CONVERTERS[_irds.trec_cast.CastPassageDoc] = CastPassageDocHandler()
+
 
 class Adhoc(ir.Adhoc, IRDSId):
     pass

{datamaestro-text-2024.3.10 → datamaestro_text-2024.5.31}/src/datamaestro_text/version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2024.3.10'
-__version_tuple__ = version_tuple = (2024, 3, 10)
+__version__ = version = '2024.5.31'
+__version_tuple__ = version_tuple = (2024, 5, 31)

{datamaestro-text-2024.3.10 → datamaestro_text-2024.5.31/src/datamaestro_text.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamaestro-text
-Version: 2024.3.10
+Version: 2024.5.31
 Summary: Datamaestro module for text-related datasets
 Author-email: Benjamin Piwowarski <benjamin@piwowarski.fr>
 License: GPL-3
@@ -18,7 +18,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: datamaestro>=1.1.0
+Requires-Dist: datamaestro>=1.1.1
 Requires-Dist: ir_datasets
 Requires-Dist: attrs
 

{datamaestro-text-2024.3.10 → datamaestro_text-2024.5.31}/src/datamaestro_text.egg-info/SOURCES.txt

@@ -47,6 +47,7 @@ src/datamaestro_text/config/com/oscar-corpus.py
 src/datamaestro_text/config/com/sentiment140.py
 src/datamaestro_text/config/com/fastml/goodbooks-10k.yaml
 src/datamaestro_text/config/com/github/aagohary/canard.py
+src/datamaestro_text/config/com/github/apple/ml-qrecc.py
 src/datamaestro_text/config/com/github/prdwb/orconvqa.py
 src/datamaestro_text/config/com/github/soskek/bookcorpus.yaml
 src/datamaestro_text/config/com/microsoft/wikiqa.yaml
@@ -89,6 +90,7 @@ src/datamaestro_text/data/conversation/__init__.py
 src/datamaestro_text/data/conversation/base.py
 src/datamaestro_text/data/conversation/canard.py
 src/datamaestro_text/data/conversation/orconvqa.py
+src/datamaestro_text/data/conversation/qrecc.py
 src/datamaestro_text/data/ir/__init__.py
 src/datamaestro_text/data/ir/base.py
 src/datamaestro_text/data/ir/cord19.py

datamaestro_text-2024.5.31/src/datamaestro_text.egg-info/requires.txt

@@ -0,0 +1,3 @@
+datamaestro>=1.1.1
+ir_datasets
+attrs

datamaestro-text-2024.3.10/.pre-commit-config.yaml

@@ -1,11 +0,0 @@
-repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.2.0
-    hooks:
-    -   id: check-yaml
-    -   id: end-of-file-fixer
-    -   id: trailing-whitespace
--   repo: https://github.com/psf/black
-    rev: 22.3.0
-    hooks:
-    -   id: black

datamaestro-text-2024.3.10/requirements.txt

@@ -1,3 +0,0 @@
-datamaestro>=1.1.0
-ir_datasets
-attrs

datamaestro-text-2024.3.10/src/datamaestro_text.egg-info/requires.txt

@@ -1,3 +0,0 @@
-datamaestro>=1.1.0
-ir_datasets
-attrs