datamaestro-text 2023.10.10__py3-none-any.whl → 2023.11.22__py3-none-any.whl

datamaestro_text/data/ir/formats.py

@@ -1,6 +1,8 @@
-from typing import ClassVar
+from typing import ClassVar, Tuple
 from attrs import define
-from .base import IDHolder, Document, GenericTopic
+from ir_datasets.datasets.wapo import WapoDocMedia
+from .base import IDHolder, Document, GenericTopic, IDTopic
+from ir_datasets.datasets.cord19 import Cord19FullTextSection
 
 
 @define
@@ -16,11 +18,172 @@ class CordDocument(IDHolder, Document):
         return f"{self.title} {self.text}"
 
 
+@define
+class DocumentWithTitle(IDHolder, Document):
+    """Web document with title and URL"""
+
+    title: str
+
+    text: str
+
+
+@define
+class CordFullTextDocument(IDHolder, Document):
+    title: str
+    doi: str
+    date: str
+    abstract: str
+    body: Tuple[Cord19FullTextSection, ...]
+
+    has_text: ClassVar[bool] = True
+
+    def get_text(self):
+        return f"{self.abstract}"
+
+
+@define
+class MsMarcoDocument(IDHolder, Document):
+    url: str
+    title: str
+    body: str
+
+    has_text: ClassVar[bool] = True
+
+    def get_text(self):
+        return f"{self.body}"
+
+
+@define
+class NFCorpusDocument(IDHolder, Document):
+    url: str
+    title: str
+    abstract: str
+
+    has_text: ClassVar[bool] = True
+
+    def get_text(self):
+        return f"{self.abstract}"
+
+
+@define
+class TitleDocument(IDHolder, Document):
+    text: str
+    title: str
+    has_text: ClassVar[bool] = True
+
+    def get_text(self):
+        return f"{self.title} {self.text}"
+
+
+@define
+class TitleUrlDocument(IDHolder, Document):
+    text: str
+    title: str
+    url: str
+    has_text: ClassVar[bool] = True
+
+    def get_text(self):
+        return f"{self.title} {self.text}"
+
+
+@define
+class TrecParsedDocument(IDHolder, Document):
+    title: str
+    body: str
+    marked_up_doc: bytes
+
+    has_text: ClassVar[bool] = True
+
+    def get_text(self):
+        return f"{self.title} {self.body}"
+
+
+@define
+class WapoDocument(IDHolder, Document):
+    url: str
+    title: str
+    author: str
+    published_date: int
+    kicker: str
+    body: str
+    body_paras_html: Tuple[str, ...]
+    body_media: Tuple[WapoDocMedia, ...]
+
+    has_text: ClassVar[bool] = True
+
+    def get_text(self):
+        return f"{self.body}"
+
+
+@define
+class TweetDoc(IDHolder, Document):
+    text: str
+    user_id: str
+    created_at: str
+    lang: str
+    reply_doc_id: str
+    retweet_doc_id: str
+    source: bytes
+    source_content_type: str
+
+    def get_text(self):
+        return f"{self.text}"
+
+
 @define
 class TrecTopic(GenericTopic):
     text: str
     query: str
     narrative: str
 
+    def get_text(self):
+        return f"{self.text}"
+
+
+@define
+class UrlTopic(GenericTopic):
+    text: str
+    url: str
+
+    def get_text(self):
+        return f"{self.text}"
+
+
+@define
+class NFCorpusTopic(IDTopic):
+    title: str
+    all: str
+
+    def get_text(self):
+        return f"{self.title}"
+
+
+@define
+class TrecQuery(IDTopic):
+    title: str
+    description: str
+    narrative: str
+
+    def get_text(self):
+        return f"{self.description}"
+
+
+@define
+class TrecMb13Query(IDTopic):
+    query: str
+    time: str
+    tweet_time: str
+
+    def get_text(self):
+        return f"{self.query}"
+
+
+@define
+class TrecMb14Query(IDTopic):
+    query: str
+    time: str
+    tweet_time: str
+    description: str
+
     def get_text(self):
         return f"{self.query}"

datamaestro_text/datasets/irds/data.py

@@ -2,7 +2,13 @@ import logging
 from typing import Any, Iterator, Tuple, Type, List
 import attrs
 import ir_datasets
-from ir_datasets.formats import GenericDoc, GenericQuery, GenericDocPair
+from ir_datasets.formats import (
+    GenericDoc,
+    GenericQuery,
+    GenericDocPair,
+    TrecParsedDoc,
+    TrecQuery,
+)
 import ir_datasets.datasets as _irds
 from experimaestro import Config
 from experimaestro.compat import cached_property
@@ -72,9 +78,10 @@ class tuple_constructor:
         self.fields = fields
 
     def check(self, source_cls: Type):
-        assert (
-            source_cls._fields == self.fields
-        ), f"Internal error: Fields do not match ({source_cls._fields} and {self.fields})"
+        assert source_cls._fields == self.fields, (
+            "Internal error: Fields do not match, "
+            f"source({source_cls.__qualname__})={source_cls._fields} [vs] target={self.fields}"
+        )
 
     def __call__(self, entry):
         return self.target_cls(*tuple(entry))
@@ -91,6 +98,54 @@ class Documents(ir.DocumentStore, IRDSId):
         _irds.beir.BeirCordDoc: tuple_constructor(
             formats.CordDocument, "doc_id", "text", "title", "url", "pubmed_id"
         ),
+        _irds.beir.BeirTitleDoc: tuple_constructor(
+            formats.TitleDocument, "doc_id", "text", "title"
+        ),
+        _irds.beir.BeirTitleUrlDoc: tuple_constructor(
+            formats.TitleUrlDocument, "doc_id", "text", "title", "url"
+        ),
+        _irds.msmarco_document.MsMarcoDocument: tuple_constructor(
+            formats.MsMarcoDocument, "doc_id", "url", "title", "body"
+        ),
+        _irds.cord19.Cord19FullTextDoc: tuple_constructor(
+            formats.CordFullTextDocument,
+            "doc_id",
+            "title",
+            "doi",
+            "date",
+            "abstract",
+            "body",
+        ),
+        _irds.nfcorpus.NfCorpusDoc: tuple_constructor(
+            formats.NFCorpusDocument, "doc_id", "url", "title", "abstract"
+        ),
+        TrecParsedDoc: tuple_constructor(
+            formats.TrecParsedDocument, "doc_id", "title", "body", "marked_up_doc"
+        ),
+        _irds.wapo.WapoDoc: tuple_constructor(
+            formats.WapoDocument,
+            "doc_id",
+            "url",
+            "title",
+            "author",
+            "published_date",
+            "kicker",
+            "body",
+            "body_paras_html",
+            "body_media",
+        ),
+        _irds.tweets2013_ia.TweetDoc: tuple_constructor(
+            formats.TweetDoc,
+            "doc_id",
+            "text",
+            "user_id",
+            "created_at",
+            "lang",
+            "reply_doc_id",
+            "retweet_doc_id",
+            "source",
+            "source_content_type",
+        ),
     }
 
     """Wraps an ir datasets collection -- and provide a default text
@@ -147,6 +202,12 @@ class Documents(ir.DocumentStore, IRDSId):
         return converter
 
 
+if hasattr(_irds, "miracl"):
+    Documents.CONVERTERS[_irds.miracl.MiraclDoc] = tuple_constructor(
+        formats.DocumentWithTitle, "doc_id", "title", "text"
+    )
+
+
 @attrs.define()
 class IRDSQueryWrapper(ir.Topic):
     query: Any
@@ -158,6 +219,26 @@ class Topics(ir.TopicsStore, IRDSId):
         _irds.beir.BeirCovidQuery: tuple_constructor(
             formats.TrecTopic, "query_id", "text", "query", "narrative"
         ),
+        _irds.beir.BeirUrlQuery: tuple_constructor(
+            formats.UrlTopic, "query_id", "text", "url"
+        ),
+        _irds.nfcorpus.NfCorpusQuery: tuple_constructor(
+            formats.NFCorpusTopic, "query_id", "title", "all"
+        ),
+        TrecQuery: tuple_constructor(
+            formats.TrecQuery, "query_id", "title", "description", "narrative"
+        ),
+        _irds.tweets2013_ia.TrecMb13Query: tuple_constructor(
+            formats.TrecMb13Query, "query_id", "query", "time", "tweet_time"
+        ),
+        _irds.tweets2013_ia.TrecMb14Query: tuple_constructor(
+            formats.TrecMb14Query,
+            "query_id",
+            "query",
+            "time",
+            "tweet_time",
+            "description",
+        ),
     }
 
     def iter(self) -> Iterator[ir.Topic]:

datamaestro_text/version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2023.10.10'
-__version_tuple__ = version_tuple = (2023, 10, 10)
+__version__ = version = '2023.11.22'
+__version_tuple__ = version_tuple = (2023, 11, 22)

{datamaestro_text-2023.10.10.dist-info → datamaestro_text-2023.11.22.dist-info}/METADATA

@@ -1,16 +1,13 @@
 Metadata-Version: 2.1
 Name: datamaestro-text
-Version: 2023.10.10
+Version: 2023.11.22
 Summary: Datamaestro module for text-related datasets
-Home-page: https://github.com/experimaestro/datamaestro_text
-Author: Benjamin Piwowarski
 Author-email: Benjamin Piwowarski <benjamin@piwowarski.fr>
 License: GPL-3
 Project-URL: homepage, https://github.com/experimaestro/datamaestro_text
 Project-URL: documentation, https://datamaestro-text.readthedocs.io/en/latest/
 Project-URL: repository, https://github.com/experimaestro/datamaestro_text
 Keywords: dataset manager,information retrieval,experiments
-Platform: any
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
@@ -23,8 +20,6 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: datamaestro >=0.8.16
 Requires-Dist: attrs
-Provides-Extra: test
-Requires-Dist: tox ; extra == 'test'
 
  [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) [![PyPI version](https://badge.fury.io/py/datamaestro-text.svg)](https://badge.fury.io/py/datamaestro-text)
 

{datamaestro_text-2023.10.10.dist-info → datamaestro_text-2023.11.22.dist-info}/RECORD

@@ -1,5 +1,5 @@
 datamaestro_text/__init__.py,sha256=hU8jZpkXl3F74qIfqnJl7v4nJ9YxfoR7IpJpUREFNRI,248
-datamaestro_text/version.py,sha256=Ol68Jx3UH3rRn1Qf0YlokVrHAvWr1Q4Y2-Qb3UjbyLI,421
+datamaestro_text/version.py,sha256=SfBDLSp-ExawWjfQ5F5ZPLPMVBHTFXz89upGMl-xJ38,421
 datamaestro_text/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamaestro_text/config/ai/quac.yaml,sha256=h1D7UJo2z1nZ_9MXpDHuQNJG9Ma2oayUdJV6lyumAIg,1103
 datamaestro_text/config/com/oscar-corpus.py,sha256=dDfdy_uhG6esRQQzUOOORwK8KvEq4c4hZbWt22pv-zY,733
@@ -52,12 +52,12 @@ datamaestro_text/data/ir/base.py,sha256=FOT0fk7_Kw-LRAPJnxNvuBHAhW6hhRCqWepqdBsp
 datamaestro_text/data/ir/cord19.py,sha256=JN31EQeg0UFAJlIkg0Ie0_pq-f-oS1OstZGJLJBeKyY,1130
 datamaestro_text/data/ir/csv.py,sha256=vgBNOeayEALwO01LmrzVOEVbs_MWJn3eIm-o0KiXjiE,1836
 datamaestro_text/data/ir/data.py,sha256=ERmmOxz_9mUkIuccNbzUa5Y6gVLLVDdyc4cCxbCCUbY,20
-datamaestro_text/data/ir/formats.py,sha256=RWmfghR91O3xSOzATXM-VtTY0Nyjj9-I8czuWEksCAE,465
+datamaestro_text/data/ir/formats.py,sha256=6344Tj2yTxQ5KW-YtkBbdbCgWTbSsO6f0AaJlvvibqM,3248
 datamaestro_text/data/ir/huggingface.py,sha256=G71VFDN-SllZy4LFxumEbCumEJvb5-orAbLemHjWhiA,956
 datamaestro_text/data/ir/trec.py,sha256=n98_O_sPPdU2i037fAboD4lB_I7C-RJrOLmmkg3osL8,1741
 datamaestro_text/data/ir/utils.py,sha256=6-GhXVtgkBZGhIs2-ODZua_3DmKjSSVydStpHDqbAwE,833
 datamaestro_text/datasets/irds/__init__.py,sha256=Tq0HN1qojnZYLBumM59BuTkz7r0gcu-5OXmDDLgPpAc,707
-datamaestro_text/datasets/irds/data.py,sha256=IN0JFrvMlTZFoFKNPx_3Y6dmO4D-3mq_R8nuC7MyiOM,6767
+datamaestro_text/datasets/irds/data.py,sha256=hDctKswyzD_VrCRcD6pNIoKiiwvapWQBUwxzdFHesIM,9348
 datamaestro_text/datasets/irds/datasets.py,sha256=4tNTmlcF2OmUttCMyz5YTepi91pvaZB4syy5u-jAKh4,5556
 datamaestro_text/datasets/irds/utils.py,sha256=m30JLIrV_HgilN11TvY9dGTyumES6LLzWZDUAMT915M,1425
 datamaestro_text/download/tmdb.py,sha256=kU_Vz9jhznlyeKMHziVu58IHoWv8zPu6CZTHVNQvmu4,4009
@@ -72,10 +72,9 @@ datamaestro_text/utils/files.py,sha256=n6ZGl5LNrZbHLcV9RFwd7cFT0vPUezit-2dsBzs1v
 datamaestro_text/utils/iter.py,sha256=-m0Y_0YjSlEVbotzZYIA0Ca0Hq0G_bF9GfAZR2yxrAk,520
 datamaestro_text/utils/randomstream.py,sha256=_-boH4IIqN8qcl3IktjpNp9vmF4TWRzHUSNVwg7WAr8,973
 datamaestro_text/utils/shuffle.py,sha256=o8JTz3mr0lYWyv0zEh91jEK12ci1etMiUnzh5GkOHCM,3490
-datamaestro_text-2023.10.10.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamaestro_text-2023.10.10.dist-info/METADATA,sha256=Pl--IRKaXPNbZvg68ba_PNGdteFcl0Cu7tAxfoadPr4,1740
-datamaestro_text-2023.10.10.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-datamaestro_text-2023.10.10.dist-info/entry_points.txt,sha256=lO1P5hE183L5qEEVHlG8d_ik0HNXnX7Eo87cQLdcl-Y,111
-datamaestro_text-2023.10.10.dist-info/top_level.txt,sha256=gYSeqViE8r7eCxSdqFJL74OwljOwKsGPaIhEcCXqc-o,17
-datamaestro_text-2023.10.10.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-datamaestro_text-2023.10.10.dist-info/RECORD,,
+datamaestro_text-2023.11.22.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamaestro_text-2023.11.22.dist-info/METADATA,sha256=227Q9AEKEkApflnZdzL5pC_J6fLPZtSqbBXFJ_A6adQ,1579
+datamaestro_text-2023.11.22.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
+datamaestro_text-2023.11.22.dist-info/entry_points.txt,sha256=lO1P5hE183L5qEEVHlG8d_ik0HNXnX7Eo87cQLdcl-Y,111
+datamaestro_text-2023.11.22.dist-info/top_level.txt,sha256=gYSeqViE8r7eCxSdqFJL74OwljOwKsGPaIhEcCXqc-o,17
+datamaestro_text-2023.11.22.dist-info/RECORD,,

{datamaestro_text-2023.10.10.dist-info → datamaestro_text-2023.11.22.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.2)
+Generator: bdist_wheel (0.41.3)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

datamaestro_text-2023.10.10.dist-info/zip-safe

@@ -1 +0,0 @@
-