datamaestro-text 2024.3.10__tar.gz → 2025.1.7__tar.gz

datamaestro_text-2025.1.7/.pre-commit-config.yaml

@@ -0,0 +1,19 @@
+repos:
+- hooks:
+  - id: check-yaml
+  - id: end-of-file-fixer
+  - id: trailing-whitespace
+  repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.4.0
+- hooks:
+  - exclude: ^src/experimaestro/server/data
+    id: black
+  repo: https://github.com/psf/black
+  rev: 23.1.0
+- hooks:
+  - additional_dependencies:
+    - flake8-print
+    - flake8-fixme
+    id: flake8
+  repo: https://github.com/pycqa/flake8
+  rev: 6.0.0

{datamaestro-text-2024.3.10/src/datamaestro_text.egg-info → datamaestro_text-2025.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamaestro-text
-Version: 2024.3.10
+Version: 2025.1.7
 Summary: Datamaestro module for text-related datasets
 Author-email: Benjamin Piwowarski <benjamin@piwowarski.fr>
 License: GPL-3
@@ -18,8 +18,8 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: datamaestro>=1.1.0
-Requires-Dist: ir_datasets
+Requires-Dist: datamaestro>=1.2.1
+Requires-Dist: ir_datasets>=0.5.8
 Requires-Dist: attrs
 
  [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) [![PyPI version](https://badge.fury.io/py/datamaestro-text.svg)](https://badge.fury.io/py/datamaestro-text)

{datamaestro-text-2024.3.10 → datamaestro_text-2025.1.7}/docs/source/api/conversation.rst

@@ -45,3 +45,11 @@ Contextual query reformulation
 
 .. autoclass:: datamaestro_text.data.conversation.orconvqa.OrConvQADatasetHistoryEntry
     :members:
+
+
+
+.. autoclass:: datamaestro_text.data.conversation.orconvqa.QReCCDatasetEntry
+    :members:
+
+.. autoxpmconfig:: datamaestro_text.data.conversation.qrecc.QReCCDataset
+    :members: iter

{datamaestro-text-2024.3.10 → datamaestro_text-2025.1.7}/docs/source/datasets/conversation.rst

@@ -8,3 +8,5 @@ Contextualized Query Rewriting
 .. dm:datasets:: com.github.aagohary.canard text
 
 .. dm:datasets:: com.github.prdwb.orconvqa text
+
+.. dm:datasets:: com.github.apple.ml-qrecc text

{datamaestro-text-2024.3.10 → datamaestro_text-2025.1.7}/docs/source/datasets/irds.rst

@@ -8,9 +8,10 @@ version of `ir-datasets` is more ancient or newer than the one used at generatio
 Data types
 ----------
 
-.. autoxpmconfig:: xpmir.datasets.irds.data.Topics
-.. autoxpmconfig:: xpmir.datasets.irds.data.Documents
-.. autoxpmconfig:: xpmir.datasets.irds.data.AdhocAssessments
+.. autoxpmconfig:: datamaestro_text.datasets.irds.data.Topics
+.. autoxpmconfig:: datamaestro_text.datasets.irds.data.Documents
+.. autoxpmconfig:: datamaestro_text.datasets.irds.data.AdhocAssessments
+.. autoxpmconfig:: datamaestro_text.datasets.irds.data.LZ4DocumentStore
 
 
 List of datasets

datamaestro_text-2025.1.7/requirements.txt

@@ -0,0 +1,3 @@
+datamaestro>=1.2.1
+ir_datasets>=0.5.8
+attrs

datamaestro_text-2025.1.7/src/datamaestro_text/config/com/github/apple/ml-qrecc.py

@@ -0,0 +1,87 @@
+# See documentation on https://datamaestro.readthedocs.io
+
+import re
+import json
+from pathlib import Path
+from datamaestro.definitions import datatasks, datatags, dataset
+from datamaestro.data.ml import Supervised
+from datamaestro.download import reference
+from datamaestro.download.archive import zipdownloader
+from datamaestro.download.wayback import wayback_documents
+from datamaestro.utils import HashCheck
+from datamaestro_text.data.conversation.qrecc import QReCCDataset
+from datamaestro_text.datasets.irds.data import (
+    LZ4JSONLDocumentStore,
+    SimpleJsonDocument,
+)
+from datamaestro_text.datasets.irds.helpers import lz4docstore_builder
+
+
+@datatags("conversation", "context", "query")
+@datatasks("query rewriting")
+@zipdownloader(
+    "data",
+    "https://github.com/apple/ml-qrecc/raw/main/dataset/qrecc_data.zip",
+    checker=HashCheck("f88fcc7ef3678cd6312080389c8abd67"),
+)
+@dataset(
+    Supervised[QReCCDataset, None, QReCCDataset],
+    url="https://github.com/apple/ml-qrecc",
+    doi="https://doi.org/10.48550/arXiv.2010.04898",
+    id="",
+)
+def main(data: Path):
+    """Open-Domain Question Answering Goes Conversational via Question Rewriting
+
+    We introduce QReCC (Question Rewriting in Conversational Context), an
+    end-to-end open-domain question answering dataset comprising of 14K
+    conversations with 81K question-answer pairs. The goal of this dataset is to
+    provide a challenging benchmark for end-to-end conversational question
+    answering that includes the individual subtasks of question rewriting,
+    passage retrieval and reading comprehension
+    """
+    return {
+        "train": QReCCDataset(path=data / "qrecc_train.json"),
+        "test": QReCCDataset(path=data / "qrecc_test.json"),
+    }
+
+
+@dataset(
+    url="https://github.com/apple/ml-qrecc",
+    doi="https://doi.org/10.48550/arXiv.2010.04898",
+)
+class Content(LZ4JSONLDocumentStore):
+    """QReCC mentionned URLs content"""
+
+    @staticmethod
+    def __create_dataset__(dataset, options=None):
+        ds = reference(reference=main).setup(dataset, options)
+        documents_path = wayback_documents(
+            "20191127", lambda: Content._urls(ds), name="wayback.jsonl"
+        ).setup(dataset, options)
+
+        store_path = lz4docstore_builder(
+            "store",
+            lambda: Content._documents(documents_path),
+            SimpleJsonDocument,
+            "id",
+        ).setup(dataset, options)
+
+        return LZ4JSONLDocumentStore(jsonl_path=store_path)
+
+    @staticmethod
+    def _documents(path: Path):
+        """Iterates over documents from wayback"""
+        with path.open("rt") as fp:
+            for line in fp:
+                yield SimpleJsonDocument(**json.loads(line))
+
+    @staticmethod
+    def _urls(supervised: Supervised[QReCCDataset, None, QReCCDataset]):
+        urls = set()
+        for ds in [supervised.train, supervised.test]:
+            for entry in ds.entries():
+                if entry.answer_url:
+                    url = re.sub("#.*$", "", entry.answer_url)
+                    urls.add(url)
+        return urls

{datamaestro-text-2024.3.10 → datamaestro_text-2025.1.7}/src/datamaestro_text/config/com/github/prdwb/orconvqa.py

@@ -1,11 +1,9 @@
 # See documentation on https://datamaestro.readthedocs.io
 
-from collections import namedtuple
 import gzip
 import json
 from pathlib import Path
-from typing import Iterator, NamedTuple
-import attrs
+from typing import Iterator
 from datamaestro.definitions import datatasks, datatags, dataset
 from datamaestro.download.single import filedownloader
 from datamaestro.utils import HashCheck
@@ -14,10 +12,7 @@ from datamaestro.utils import HashCheck
 from datamaestro_text.data.conversation.orconvqa import OrConvQADataset
 from datamaestro.data.ml import Supervised
 
-from datamaestro_text.data.ir import DocumentStore
-from datamaestro_text.data.ir.formats import OrConvQADocument
 from datamaestro_text.data.ir.stores import OrConvQADocumentStore
-from datamaestro_text.datasets.irds.data import LZ4DocumentStore
 from datamaestro_text.datasets.irds.helpers import lz4docstore_downloader
 
 
@@ -63,7 +58,9 @@ def preprocessed(train, dev, test):
 def orConvQADocumentReader(source: Path) -> Iterator[OrConvQADocumentStore.NAMED_TUPLE]:
     with gzip.open(source, "rt") as fp:
         for line in fp:
-            yield OrConvQADocumentStore.NAMED_TUPLE(**json.loads(line))
+            data = json.loads(line)
+            data["body"] = data.pop("text")
+            yield OrConvQADocumentStore.NAMED_TUPLE(**data)
 
 
 @lz4docstore_downloader(

{datamaestro-text-2024.3.10 → datamaestro_text-2025.1.7}/src/datamaestro_text/data/conversation/base.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from enum import Enum
-from typing import Dict, Generic, Iterator, List, Optional, Sequence
+from typing import Dict, Generic, Iterator, List, Optional, Sequence, Tuple
 from attr import define
 from datamaestro.data import Base
 from datamaestro.record import Record, Item
@@ -61,6 +61,20 @@ class AnswerEntry(Item):
     """The system answer"""
 
 
+@define
+class AnswerDocumentID(Item):
+    """An answer as a document ID"""
+
+    document_id: str
+
+
+@define
+class AnswerDocumentURL(Item):
+    """An answer as a document ID"""
+
+    url: str
+
+
 @define
 class RetrievedEntry(Item):
     """List of system-retrieved documents and their relevance"""
@@ -68,8 +82,8 @@ class RetrievedEntry(Item):
     documents: List[str]
     """List of retrieved documents"""
 
-    document_relevances: Optional[List[str]] = None
-    """List of retrieved documents and their relevance status"""
+    relevant_documents: Optional[Dict[int, Tuple[Optional[int], Optional[int]]]] = None
+    """List of relevance status (optional), with start/stop position"""
 
 
 @define
@@ -95,56 +109,99 @@ class ConversationHistoryItem(Item):
 
 
 class ConversationNode:
+    @abstractmethod
     def entry(self) -> Record:
         """The current conversation entry"""
         ...
 
+    @abstractmethod
     def history(self) -> ConversationHistory:
         """Preceding conversation entries, from most recent to more ancient"""
         ...
 
+    @abstractmethod
+    def parent(self) -> Optional["ConversationNode"]:
+        ...
+
+    @abstractmethod
+    def children(self) -> List["ConversationNode"]:
+        ...
+
 
-class ConversationTree:
+class ConversationTree(ABC):
+    @abstractmethod
+    def root(self) -> ConversationNode:
+        ...
+
+    @abstractmethod
     def __iter__(self) -> Iterator[ConversationNode]:
         """Iterates over conversation nodes"""
-        pass
+        ...
 
 
 # ---- A conversation tree
 
 
-class SingleConversationTree(ConversationTree):
+class SingleConversationTree(ConversationTree, ABC):
     """Simple conversations, based on a sequence of entries"""
 
     id: str
-    history: Sequence[Record]
+    history: List[Record]
 
     def __init__(self, id: Optional[str], history: List[Record]):
         """Create a simple conversation
 
-        :param history: The entries, in reverse order (i.e. more ancient first)
+        :param history: The entries, in **reverse** order (i.e. more ancient first)
         """
         self.history = history or []
+        self.id = id
 
     def add(self, entry: Record):
         self.history.insert(0, entry)
 
     def __iter__(self) -> Iterator[ConversationNode]:
-        for ix in range(len(self.history)):
+        """Iterates over the conversation (starting with the beginning)"""
+        for ix in reversed(range(len(self.history))):
             yield SingleConversationTreeNode(self, ix)
 
+    def root(self):
+        return SingleConversationTreeNode(self, len(self.history) - 1)
+
 
 @define
 class SingleConversationTreeNode(ConversationNode):
     tree: SingleConversationTree
     index: int
 
+    @property
     def entry(self) -> Record:
         return self.tree.history[self.index]
 
+    @entry.setter
+    def entry(self, record: Record):
+        try:
+            self.tree.history[self.index] = record
+        except Exception as e:
+            print(e)
+            raise
+
     def history(self) -> Sequence[Record]:
         return self.tree.history[self.index + 1 :]
 
+    def parent(self) -> Optional[ConversationNode]:
+        return (
+            SingleConversationTreeNode(self.tree, self.index + 1)
+            if self.index < len(self.tree.history) - 1
+            else []
+        )
+
+    def children(self) -> List[ConversationNode]:
+        return (
+            [SingleConversationTreeNode(self.tree, self.index - 1)]
+            if self.index > 0
+            else []
+        )
+
 
 class ConversationTreeNode(ConversationNode, ConversationTree):
     """A conversation tree node"""
@@ -178,6 +235,15 @@ class ConversationTreeNode(ConversationNode, ConversationTree):
         for child in self.children:
             yield from child
 
+    def parent(self) -> Optional[ConversationNode]:
+        return self.parent
+
+    def children(self) -> List[ConversationNode]:
+        return self.children
+
+    def root(self):
+        return self
+
 
 class ConversationDataset(Base, ABC):
     """A dataset made of conversations"""
@@ -186,4 +252,4 @@ class ConversationDataset(Base, ABC):
     def __iter__(self) -> Iterator[ConversationTree]:
         """Return an iterator over conversations"""
         for i in range(len(self)):
-            return self.get(i)
+            yield self.get(i)

{datamaestro-text-2024.3.10 → datamaestro_text-2025.1.7}/src/datamaestro_text/data/conversation/orconvqa.py

@@ -102,16 +102,26 @@ class OrConvQADataset(ConversationDataset, File):
             # Add to current
             history.append(
                 Record(
-                    IDItem(query_no),
+                    IDItem(entry.query_id),
                     SimpleTextItem(entry.query),
                     SimpleDecontextualizedItem(entry.rewrite),
                     EntryType.USER_QUERY,
                 )
             )
+
+            relevances = {}
+            for rank, relevance in enumerate(entry.retrieval_labels):
+                if relevance > 0:
+                    relevances[rank] = (entry.answer.answer_start, None)
+
+            assert (
+                len(relevances) <= 1
+            ), f"Too many relevance labels ({len(relevances)}) for {entry.query_id}"
+
             history.append(
                 Record(
                     AnswerEntry(entry.answer.text),
-                    RetrievedEntry(entry.evidences, entry.retrieval_labels),
+                    RetrievedEntry(entry.evidences, relevances),
                     EntryType.SYSTEM_ANSWER,
                 )
             )

datamaestro_text-2025.1.7/src/datamaestro_text/data/conversation/qrecc.py

@@ -0,0 +1,99 @@
+from typing import Iterator, List, Optional
+from attr import define
+import json
+from datamaestro.data import File
+from datamaestro.record import Record
+
+from datamaestro_text.data.ir.base import (
+    IDItem,
+    SimpleTextItem,
+)
+
+
+from .base import (
+    AnswerDocumentURL,
+    AnswerEntry,
+    ConversationTree,
+    EntryType,
+    SimpleDecontextualizedItem,
+    SingleConversationTree,
+)
+from . import ConversationDataset
+
+
+@define(kw_only=True)
+class QReCCDatasetEntry:
+    """A query with past history"""
+
+    conversation_no: int
+    """Conversation ID"""
+
+    turn_no: int
+    """The turn in the conversation"""
+
+    conversation_source: str
+    """Conversation source"""
+
+    question: str
+    """The last issued query"""
+
+    rewrite: str
+    """Manually rewritten query"""
+
+    context: List[str]
+    """The list of queries asked by the user"""
+
+    answer: str
+    """The answer"""
+
+    answer_url: str
+    """The URL containing the answer"""
+
+
+class QReCCDataset(ConversationDataset, File):
+    def entries(self) -> Iterator[QReCCDatasetEntry]:
+        """Iterates over re-written query with their context"""
+        with self.path.open("rt") as fp:
+            data = json.load(fp)
+
+        data = [
+            QReCCDatasetEntry(**{key.lower(): value for key, value in entry.items()})
+            for entry in data
+        ]
+        return iter(data)
+
+    def __iter__(self) -> Iterator[ConversationTree]:
+        history: List[Record] = []
+        current_id: Optional[str] = None
+
+        for entry in self.entries():
+            # Creates a new conversation if needed
+            if entry.conversation_no != current_id:
+                if current_id is not None:
+                    history.reverse()
+                    yield SingleConversationTree(current_id, history)
+
+                current_id = entry.conversation_no
+                history = []
+
+            # Add to current
+            history.append(
+                Record(
+                    IDItem(f"{entry.conversation_no}#{entry.turn_no}"),
+                    SimpleTextItem(entry.question),
+                    AnswerDocumentURL(entry.answer_url),
+                    SimpleDecontextualizedItem(entry.rewrite),
+                    EntryType.USER_QUERY,
+                )
+            )
+
+            history.append(
+                Record(
+                    AnswerEntry(entry.answer),
+                    EntryType.SYSTEM_ANSWER,
+                )
+            )
+
+        # Yields the last one
+        history.reverse()
+        yield SingleConversationTree(current_id, history)

{datamaestro-text-2024.3.10 → datamaestro_text-2025.1.7}/src/datamaestro_text/data/ir/__init__.py

@@ -25,6 +25,7 @@ from .base import (  # noqa: F401
     create_record,
     # Other things
     AdhocAssessment,
+    AdhocAssessedTopic,
 )
 
 
@@ -83,7 +84,7 @@ class DocumentStore(Documents):
     def document_int(self, internal_docid: int) -> DocumentRecord:
         """Returns a document given its internal ID"""
         docid = self.docid_internal2external(internal_docid)
-        return self.document(docid)
+        return self.document_ext(docid)
 
     def document_ext(self, docid: str) -> DocumentRecord:
         """Returns a document given its external ID"""
@@ -159,7 +160,7 @@ class TopicsStore(Topics):
 class AdhocAssessments(Base, ABC):
     """Ad-hoc assessments (qrels)"""
 
-    def iter(self) -> Iterator[AdhocAssessment]:
+    def iter(self) -> Iterator[AdhocAssessedTopic]:
         """Returns an iterator over assessments"""
         raise NotImplementedError(f"For class {self.__class__}")
 

{datamaestro-text-2024.3.10 → datamaestro_text-2025.1.7}/src/datamaestro_text/data/ir/formats.py

@@ -1,5 +1,5 @@
 from functools import cached_property
-from typing import ClassVar, Tuple
+from typing import ClassVar, Tuple, List
 from attrs import define
 from datamaestro.record import record_type
 from ir_datasets.datasets.wapo import WapoDocMedia
@@ -117,7 +117,6 @@ class TweetDoc(TextItem):
 
 @define
 class OrConvQADocument(TextItem):
-    id: str
     title: str
     body: str
     aid: str
@@ -127,12 +126,21 @@ class OrConvQADocument(TextItem):
     def text(self):
         return f"{self.title} {self.body}"
 
+@define
+class Touche2020(TextItem):
+    text: str
+    title: str
+    stance: str
+    url: str
 
 @define
-class TrecTopic(TextItem):
+class SciDocs(TextItem):
     text: str
-    query: str
-    narrative: str
+    title: str
+    authors: List[str]
+    year: int
+    cited_by: List[str]
+    references: List[str]
 
 
 @define
@@ -167,6 +175,13 @@ class TrecMb14Query(TextItem):
     def get_text(self):
         return f"{self.query}"
 
+@define 
+class SciDocsTopic(TextItem):
+    text: str
+    authors: List[str]
+    year: int
+    cited_by: List[str]
+    references: List[str]
 
 @define()
 class TrecTopic(SimpleTextItem):

{datamaestro-text-2024.3.10 → datamaestro_text-2025.1.7}/src/datamaestro_text/data/ir/stores.py

@@ -1,16 +1,21 @@
 from collections import namedtuple
-from typing import List
+from typing import List, NamedTuple
 from experimaestro import Constant
 import attrs
 
+from datamaestro.record import Record
+from datamaestro_text.data.ir.base import IDItem
 from datamaestro_text.datasets.irds.data import LZ4DocumentStore
 from datamaestro_text.data.ir.formats import OrConvQADocument
 
 
 class OrConvQADocumentStore(LZ4DocumentStore):
-    NAMED_TUPLE = namedtuple(
-        "OrConvQADocument", [a.name for a in attrs.fields(OrConvQADocument)]
-    )
+    class NAMED_TUPLE(NamedTuple):
+        id: str
+        title: str
+        body: str
+        aid: str
+        bid: int
 
     lookup_field: Constant[str] = "id"
     fields: Constant[List[str]] = list(NAMED_TUPLE._fields)
@@ -18,5 +23,7 @@ class OrConvQADocumentStore(LZ4DocumentStore):
 
     data_cls = NAMED_TUPLE
 
-    def converter(self, data: NAMED_TUPLE) -> OrConvQADocument:
-        return OrConvQADocument(**data._asdict())
+    def converter(self, data: NAMED_TUPLE) -> Record:
+        fields = data._asdict()
+        del fields["id"]
+        return Record(OrConvQADocument(**fields), IDItem(data.id))