datamaestro-text 2023.7.4__py3-none-any.whl → 2023.7.6.1__py3-none-any.whl

datamaestro_text/data/ir/formats.py

@@ -0,0 +1,23 @@
+from typing import ClassVar
+from attrs import define
+from .base import IDHolder, Document, Topic
+
+
+@define
+class CordDocument(IDHolder, Document):
+    text: str
+    title: str
+    url: str
+    pubmed_id: str
+
+    has_text: ClassVar[bool] = True
+
+    def get_text(self):
+        return f"{self.title} {self.text}"
+
+
+@define
+class TrecTopic(IDHolder, Topic):
+    text: str
+    query: str
+    narrative: str

datamaestro_text/datasets/irds/data.py

@@ -1,10 +1,11 @@
-import ir_datasets
-from ir_datasets.formats import GenericDoc, GenericQuery, GenericDocPair
 import logging
+from typing import Any, Iterator, Tuple, Type
 import attrs
+import ir_datasets
+from ir_datasets.formats import GenericDoc, GenericQuery, GenericDocPair
+import ir_datasets.datasets as _irds
 from experimaestro import Config
 from experimaestro.compat import cached_property
-from typing import Any, Iterator, Tuple
 from experimaestro import Option
 import datamaestro_text.data.ir as ir
 from datamaestro_text.data.ir.base import (
@@ -17,6 +18,7 @@ from datamaestro_text.data.ir.base import (
     IDDocument,
     IDTopic,
 )
+import datamaestro_text.data.ir.formats as formats
 
 
 # Interface between ir_datasets and datamaestro:
@@ -64,11 +66,18 @@ class AdhocAssessments(ir.AdhocAssessments, IRDSId):
         return qrels.values()
 
 
-def tuple_constructor(cls):
-    def constructor(entry):
-        return cls(*tuple(entry))
+class tuple_constructor:
+    def __init__(self, target_cls: Type, *fields: str):
+        self.target_cls = target_cls
+        self.fields = fields
+
+    def check(self, source_cls: Type):
+        assert (
+            source_cls._fields == self.fields
+        ), f"Internal error: Fields do not match ({source_cls._fields} and {self.fields})"
 
-    return constructor
+    def __call__(self, entry):
+        return self.target_cls(*tuple(entry))
 
 
 @attrs.define()
@@ -77,7 +86,12 @@ class IRDSDocumentWrapper(ir.Document):
 
 
 class Documents(ir.DocumentStore, IRDSId):
-    CONVERTERS = {GenericDoc: (GenericDocument, tuple_constructor)}
+    CONVERTERS = {
+        GenericDoc: tuple_constructor(GenericDocument, "doc_id", "text"),
+        _irds.beir.BeirCordDoc: tuple_constructor(
+            formats.CordDocument, "doc_id", "text", "title", "url", "pubmed_id"
+        ),
+    }
 
     """Wraps an ir datasets collection -- and provide a default text
     value depending on the collection itself"""
@@ -115,12 +129,13 @@ class Documents(ir.DocumentStore, IRDSId):
 
     @cached_property
     def document_cls(self):
-        return Documents.CONVERTERS[self.dataset.docs_cls()][0]
+        return self.converter.target_cls
 
     @cached_property
     def converter(self):
-        document_cls, constructor = Documents.CONVERTERS[self.dataset.docs_cls()]
-        return constructor(document_cls)
+        converter = Documents.CONVERTERS[self.dataset.docs_cls()]
+        converter.check(self.dataset.docs_cls())
+        return converter
 
 
 @attrs.define()
@@ -129,7 +144,12 @@ class IRDSQueryWrapper(ir.Topic):
 
 
 class Topics(ir.TopicsStore, IRDSId):
-    CONVERTERS = {GenericQuery: (GenericTopic, tuple_constructor)}
+    CONVERTERS = {
+        GenericQuery: tuple_constructor(GenericTopic, "query_id", "text"),
+        _irds.beir.BeirCovidQuery: tuple_constructor(
+            formats.TrecTopic, "query_id", "text", "query", "narrative"
+        ),
+    }
 
     def iter(self) -> Iterator[ir.Topic]:
         """Returns an iterator over topics"""
@@ -167,12 +187,13 @@ class Topics(ir.TopicsStore, IRDSId):
 
     @cached_property
     def topic_cls(self):
-        return Topics.CONVERTERS[self.dataset.queries_cls()][0]
+        return self.converter.target_cls
 
     @cached_property
     def converter(self):
-        topic_cls, constructor = Topics.CONVERTERS[self.dataset.queries_cls()]
-        return constructor(topic_cls)
+        converter = Topics.CONVERTERS[self.dataset.queries_cls()]
+        converter.check(self.dataset.queries_cls())
+        return converter
 
 
 class Adhoc(ir.Adhoc, IRDSId):

datamaestro_text/version.py

@@ -1,4 +1,4 @@
 # file generated by setuptools_scm
 # don't change, don't track in version control
-__version__ = version = '2023.7.4'
-__version_tuple__ = version_tuple = (2023, 7, 4)
+__version__ = version = '2023.7.6.1'
+__version_tuple__ = version_tuple = (2023, 7, 6, 1)

{datamaestro_text-2023.7.4.dist-info → datamaestro_text-2023.7.6.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamaestro-text
-Version: 2023.7.4
+Version: 2023.7.6.1
 Summary: Datamaestro module for text-related datasets
 Home-page: https://github.com/experimaestro/datamaestro_text
 Author: Benjamin Piwowarski

{datamaestro_text-2023.7.4.dist-info → datamaestro_text-2023.7.6.1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 datamaestro_text/__init__.py,sha256=hU8jZpkXl3F74qIfqnJl7v4nJ9YxfoR7IpJpUREFNRI,248
-datamaestro_text/version.py,sha256=ql5h48RqChgNgO-L9vkoUI2FaqbfAWiap3G8hwYBCw0,166
+datamaestro_text/version.py,sha256=Je_wDQltf6IZOe5Cu_KgI0UT7_A2LPGuxgfuW4Mz3dY,171
 datamaestro_text/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamaestro_text/config/ai/quac.yaml,sha256=h1D7UJo2z1nZ_9MXpDHuQNJG9Ma2oayUdJV6lyumAIg,1103
 datamaestro_text/config/com/oscar-corpus.py,sha256=dDfdy_uhG6esRQQzUOOORwK8KvEq4c4hZbWt22pv-zY,733
@@ -52,11 +52,12 @@ datamaestro_text/data/ir/base.py,sha256=FOT0fk7_Kw-LRAPJnxNvuBHAhW6hhRCqWepqdBsp
 datamaestro_text/data/ir/cord19.py,sha256=JN31EQeg0UFAJlIkg0Ie0_pq-f-oS1OstZGJLJBeKyY,1130
 datamaestro_text/data/ir/csv.py,sha256=vgBNOeayEALwO01LmrzVOEVbs_MWJn3eIm-o0KiXjiE,1836
 datamaestro_text/data/ir/data.py,sha256=ERmmOxz_9mUkIuccNbzUa5Y6gVLLVDdyc4cCxbCCUbY,20
+datamaestro_text/data/ir/formats.py,sha256=Nz8BJ1Lb6MygeYNHtUwwqX-flZ1QfrxaDkAJlvcv1fE,405
 datamaestro_text/data/ir/huggingface.py,sha256=G71VFDN-SllZy4LFxumEbCumEJvb5-orAbLemHjWhiA,956
 datamaestro_text/data/ir/trec.py,sha256=n98_O_sPPdU2i037fAboD4lB_I7C-RJrOLmmkg3osL8,1741
 datamaestro_text/data/ir/utils.py,sha256=6-GhXVtgkBZGhIs2-ODZua_3DmKjSSVydStpHDqbAwE,833
 datamaestro_text/datasets/irds/__init__.py,sha256=Tq0HN1qojnZYLBumM59BuTkz7r0gcu-5OXmDDLgPpAc,707
-datamaestro_text/datasets/irds/data.py,sha256=ja3HmwF-BoaWCXkP1jU0tmEvY2hfdLh07DFknBu29z8,5709
+datamaestro_text/datasets/irds/data.py,sha256=L2zjWXzr3uvmJNJaw1ES47sHMo5uMKc9JHhGhkcICJo,6448
 datamaestro_text/datasets/irds/datasets.py,sha256=tI7_lK1e0LW52zaEdoGHFXHSQeoU6eMtA5_6wQSrRkE,5540
 datamaestro_text/datasets/irds/utils.py,sha256=m30JLIrV_HgilN11TvY9dGTyumES6LLzWZDUAMT915M,1425
 datamaestro_text/download/tmdb.py,sha256=kU_Vz9jhznlyeKMHziVu58IHoWv8zPu6CZTHVNQvmu4,4009
@@ -70,10 +71,10 @@ datamaestro_text/utils/__init__.py,sha256=2449YLTAtKJzkmt84Mu8sBRCCveNs5fiaqTCK_
 datamaestro_text/utils/files.py,sha256=n6ZGl5LNrZbHLcV9RFwd7cFT0vPUezit-2dsBzs1vRQ,170
 datamaestro_text/utils/randomstream.py,sha256=_-boH4IIqN8qcl3IktjpNp9vmF4TWRzHUSNVwg7WAr8,973
 datamaestro_text/utils/shuffle.py,sha256=o8JTz3mr0lYWyv0zEh91jEK12ci1etMiUnzh5GkOHCM,3490
-datamaestro_text-2023.7.4.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamaestro_text-2023.7.4.dist-info/METADATA,sha256=CYB52WlcCdVQK-0ATQ517WDS8a3bmkZfBK5q4wPXeiE,1740
-datamaestro_text-2023.7.4.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-datamaestro_text-2023.7.4.dist-info/entry_points.txt,sha256=lO1P5hE183L5qEEVHlG8d_ik0HNXnX7Eo87cQLdcl-Y,111
-datamaestro_text-2023.7.4.dist-info/top_level.txt,sha256=gYSeqViE8r7eCxSdqFJL74OwljOwKsGPaIhEcCXqc-o,17
-datamaestro_text-2023.7.4.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-datamaestro_text-2023.7.4.dist-info/RECORD,,
+datamaestro_text-2023.7.6.1.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamaestro_text-2023.7.6.1.dist-info/METADATA,sha256=24yduxTyKWsLuDl4heKsRrNobJI5xUHkgNSb42G51tU,1742
+datamaestro_text-2023.7.6.1.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+datamaestro_text-2023.7.6.1.dist-info/entry_points.txt,sha256=lO1P5hE183L5qEEVHlG8d_ik0HNXnX7Eo87cQLdcl-Y,111
+datamaestro_text-2023.7.6.1.dist-info/top_level.txt,sha256=gYSeqViE8r7eCxSdqFJL74OwljOwKsGPaIhEcCXqc-o,17
+datamaestro_text-2023.7.6.1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+datamaestro_text-2023.7.6.1.dist-info/RECORD,,