datamaestro-text 2023.10.10__tar.gz → 2023.11.22__tar.gz

{datamaestro-text-2023.10.10 → datamaestro-text-2023.11.22}/.github/workflows/pytest.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8, 3.9, "3.10"]
+        python-version: [3.8, 3.9, "3.10", "3.11"]
 
     steps:
     - uses: actions/checkout@v2

{datamaestro-text-2023.10.10 → datamaestro-text-2023.11.22}/.github/workflows/python-publish.yml

@@ -20,11 +20,11 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install setuptools wheel twine
+          pip install setuptools wheel twine build
       - name: Build and publish
         env:
           TWINE_USERNAME: __token__
           TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
         run: |
-          python setup.py sdist bdist_wheel
+          python -m build --sdist --wheel
           twine upload dist/*

{datamaestro-text-2023.10.10/src/datamaestro_text.egg-info → datamaestro-text-2023.11.22}/PKG-INFO

@@ -1,16 +1,13 @@
 Metadata-Version: 2.1
 Name: datamaestro-text
-Version: 2023.10.10
+Version: 2023.11.22
 Summary: Datamaestro module for text-related datasets
-Home-page: https://github.com/experimaestro/datamaestro_text
-Author: Benjamin Piwowarski
 Author-email: Benjamin Piwowarski <benjamin@piwowarski.fr>
 License: GPL-3
 Project-URL: homepage, https://github.com/experimaestro/datamaestro_text
 Project-URL: documentation, https://datamaestro-text.readthedocs.io/en/latest/
 Project-URL: repository, https://github.com/experimaestro/datamaestro_text
 Keywords: dataset manager,information retrieval,experiments
-Platform: any
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
@@ -20,8 +17,9 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Provides-Extra: test
 License-File: LICENSE
+Requires-Dist: datamaestro>=0.8.16
+Requires-Dist: attrs
 
  [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) [![PyPI version](https://badge.fury.io/py/datamaestro-text.svg)](https://badge.fury.io/py/datamaestro-text)
 

{datamaestro-text-2023.10.10 → datamaestro-text-2023.11.22}/pyproject.toml

@@ -18,6 +18,10 @@ classifiers = [
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
+readme = {file = ["README.md"], content-type = "text/markdown"}
+
 [project.urls]
 homepage = "https://github.com/experimaestro/datamaestro_text"
 documentation = "https://datamaestro-text.readthedocs.io/en/latest/"

datamaestro-text-2023.11.22/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

datamaestro-text-2023.11.22/src/datamaestro_text/data/ir/formats.py

@@ -0,0 +1,189 @@
+from typing import ClassVar, Tuple
+from attrs import define
+from ir_datasets.datasets.wapo import WapoDocMedia
+from .base import IDHolder, Document, GenericTopic, IDTopic
+from ir_datasets.datasets.cord19 import Cord19FullTextSection
+
+
+@define
+class CordDocument(IDHolder, Document):
+    text: str
+    title: str
+    url: str
+    pubmed_id: str
+
+    has_text: ClassVar[bool] = True
+
+    def get_text(self):
+        return f"{self.title} {self.text}"
+
+
+@define
+class DocumentWithTitle(IDHolder, Document):
+    """Web document with title and URL"""
+
+    title: str
+
+    text: str
+
+
+@define
+class CordFullTextDocument(IDHolder, Document):
+    title: str
+    doi: str
+    date: str
+    abstract: str
+    body: Tuple[Cord19FullTextSection, ...]
+
+    has_text: ClassVar[bool] = True
+
+    def get_text(self):
+        return f"{self.abstract}"
+
+
+@define
+class MsMarcoDocument(IDHolder, Document):
+    url: str
+    title: str
+    body: str
+
+    has_text: ClassVar[bool] = True
+
+    def get_text(self):
+        return f"{self.body}"
+
+
+@define
+class NFCorpusDocument(IDHolder, Document):
+    url: str
+    title: str
+    abstract: str
+
+    has_text: ClassVar[bool] = True
+
+    def get_text(self):
+        return f"{self.abstract}"
+
+
+@define
+class TitleDocument(IDHolder, Document):
+    text: str
+    title: str
+    has_text: ClassVar[bool] = True
+
+    def get_text(self):
+        return f"{self.title} {self.text}"
+
+
+@define
+class TitleUrlDocument(IDHolder, Document):
+    text: str
+    title: str
+    url: str
+    has_text: ClassVar[bool] = True
+
+    def get_text(self):
+        return f"{self.title} {self.text}"
+
+
+@define
+class TrecParsedDocument(IDHolder, Document):
+    title: str
+    body: str
+    marked_up_doc: bytes
+
+    has_text: ClassVar[bool] = True
+
+    def get_text(self):
+        return f"{self.title} {self.body}"
+
+
+@define
+class WapoDocument(IDHolder, Document):
+    url: str
+    title: str
+    author: str
+    published_date: int
+    kicker: str
+    body: str
+    body_paras_html: Tuple[str, ...]
+    body_media: Tuple[WapoDocMedia, ...]
+
+    has_text: ClassVar[bool] = True
+
+    def get_text(self):
+        return f"{self.body}"
+
+
+@define
+class TweetDoc(IDHolder, Document):
+    text: str
+    user_id: str
+    created_at: str
+    lang: str
+    reply_doc_id: str
+    retweet_doc_id: str
+    source: bytes
+    source_content_type: str
+
+    def get_text(self):
+        return f"{self.text}"
+
+
+@define
+class TrecTopic(GenericTopic):
+    text: str
+    query: str
+    narrative: str
+
+    def get_text(self):
+        return f"{self.text}"
+
+
+@define
+class UrlTopic(GenericTopic):
+    text: str
+    url: str
+
+    def get_text(self):
+        return f"{self.text}"
+
+
+@define
+class NFCorpusTopic(IDTopic):
+    title: str
+    all: str
+
+    def get_text(self):
+        return f"{self.title}"
+
+
+@define
+class TrecQuery(IDTopic):
+    title: str
+    description: str
+    narrative: str
+
+    def get_text(self):
+        return f"{self.description}"
+
+
+@define
+class TrecMb13Query(IDTopic):
+    query: str
+    time: str
+    tweet_time: str
+
+    def get_text(self):
+        return f"{self.query}"
+
+
+@define
+class TrecMb14Query(IDTopic):
+    query: str
+    time: str
+    tweet_time: str
+    description: str
+
+    def get_text(self):
+        return f"{self.query}"

{datamaestro-text-2023.10.10 → datamaestro-text-2023.11.22}/src/datamaestro_text/datasets/irds/data.py

@@ -2,7 +2,13 @@ import logging
 from typing import Any, Iterator, Tuple, Type, List
 import attrs
 import ir_datasets
-from ir_datasets.formats import GenericDoc, GenericQuery, GenericDocPair
+from ir_datasets.formats import (
+    GenericDoc,
+    GenericQuery,
+    GenericDocPair,
+    TrecParsedDoc,
+    TrecQuery,
+)
 import ir_datasets.datasets as _irds
 from experimaestro import Config
 from experimaestro.compat import cached_property
@@ -72,9 +78,10 @@ class tuple_constructor:
         self.fields = fields
 
     def check(self, source_cls: Type):
-        assert (
-            source_cls._fields == self.fields
-        ), f"Internal error: Fields do not match ({source_cls._fields} and {self.fields})"
+        assert source_cls._fields == self.fields, (
+            "Internal error: Fields do not match, "
+            f"source({source_cls.__qualname__})={source_cls._fields} [vs] target={self.fields}"
+        )
 
     def __call__(self, entry):
         return self.target_cls(*tuple(entry))
@@ -91,6 +98,54 @@ class Documents(ir.DocumentStore, IRDSId):
         _irds.beir.BeirCordDoc: tuple_constructor(
             formats.CordDocument, "doc_id", "text", "title", "url", "pubmed_id"
         ),
+        _irds.beir.BeirTitleDoc: tuple_constructor(
+            formats.TitleDocument, "doc_id", "text", "title"
+        ),
+        _irds.beir.BeirTitleUrlDoc: tuple_constructor(
+            formats.TitleUrlDocument, "doc_id", "text", "title", "url"
+        ),
+        _irds.msmarco_document.MsMarcoDocument: tuple_constructor(
+            formats.MsMarcoDocument, "doc_id", "url", "title", "body"
+        ),
+        _irds.cord19.Cord19FullTextDoc: tuple_constructor(
+            formats.CordFullTextDocument,
+            "doc_id",
+            "title",
+            "doi",
+            "date",
+            "abstract",
+            "body",
+        ),
+        _irds.nfcorpus.NfCorpusDoc: tuple_constructor(
+            formats.NFCorpusDocument, "doc_id", "url", "title", "abstract"
+        ),
+        TrecParsedDoc: tuple_constructor(
+            formats.TrecParsedDocument, "doc_id", "title", "body", "marked_up_doc"
+        ),
+        _irds.wapo.WapoDoc: tuple_constructor(
+            formats.WapoDocument,
+            "doc_id",
+            "url",
+            "title",
+            "author",
+            "published_date",
+            "kicker",
+            "body",
+            "body_paras_html",
+            "body_media",
+        ),
+        _irds.tweets2013_ia.TweetDoc: tuple_constructor(
+            formats.TweetDoc,
+            "doc_id",
+            "text",
+            "user_id",
+            "created_at",
+            "lang",
+            "reply_doc_id",
+            "retweet_doc_id",
+            "source",
+            "source_content_type",
+        ),
     }
 
     """Wraps an ir datasets collection -- and provide a default text
@@ -147,6 +202,12 @@ class Documents(ir.DocumentStore, IRDSId):
         return converter
 
 
+if hasattr(_irds, "miracl"):
+    Documents.CONVERTERS[_irds.miracl.MiraclDoc] = tuple_constructor(
+        formats.DocumentWithTitle, "doc_id", "title", "text"
+    )
+
+
 @attrs.define()
 class IRDSQueryWrapper(ir.Topic):
     query: Any
@@ -158,6 +219,26 @@ class Topics(ir.TopicsStore, IRDSId):
         _irds.beir.BeirCovidQuery: tuple_constructor(
             formats.TrecTopic, "query_id", "text", "query", "narrative"
         ),
+        _irds.beir.BeirUrlQuery: tuple_constructor(
+            formats.UrlTopic, "query_id", "text", "url"
+        ),
+        _irds.nfcorpus.NfCorpusQuery: tuple_constructor(
+            formats.NFCorpusTopic, "query_id", "title", "all"
+        ),
+        TrecQuery: tuple_constructor(
+            formats.TrecQuery, "query_id", "title", "description", "narrative"
+        ),
+        _irds.tweets2013_ia.TrecMb13Query: tuple_constructor(
+            formats.TrecMb13Query, "query_id", "query", "time", "tweet_time"
+        ),
+        _irds.tweets2013_ia.TrecMb14Query: tuple_constructor(
+            formats.TrecMb14Query,
+            "query_id",
+            "query",
+            "time",
+            "tweet_time",
+            "description",
+        ),
     }
 
     def iter(self) -> Iterator[ir.Topic]:

{datamaestro-text-2023.10.10 → datamaestro-text-2023.11.22}/src/datamaestro_text/version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2023.10.10'
-__version_tuple__ = version_tuple = (2023, 10, 10)
+__version__ = version = '2023.11.22'
+__version_tuple__ = version_tuple = (2023, 11, 22)

{datamaestro-text-2023.10.10 → datamaestro-text-2023.11.22/src/datamaestro_text.egg-info}/PKG-INFO

@@ -1,16 +1,13 @@
 Metadata-Version: 2.1
 Name: datamaestro-text
-Version: 2023.10.10
+Version: 2023.11.22
 Summary: Datamaestro module for text-related datasets
-Home-page: https://github.com/experimaestro/datamaestro_text
-Author: Benjamin Piwowarski
 Author-email: Benjamin Piwowarski <benjamin@piwowarski.fr>
 License: GPL-3
 Project-URL: homepage, https://github.com/experimaestro/datamaestro_text
 Project-URL: documentation, https://datamaestro-text.readthedocs.io/en/latest/
 Project-URL: repository, https://github.com/experimaestro/datamaestro_text
 Keywords: dataset manager,information retrieval,experiments
-Platform: any
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
@@ -20,8 +17,9 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Provides-Extra: test
 License-File: LICENSE
+Requires-Dist: datamaestro>=0.8.16
+Requires-Dist: attrs
 
  [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) [![PyPI version](https://badge.fury.io/py/datamaestro-text.svg)](https://badge.fury.io/py/datamaestro-text)
 

{datamaestro-text-2023.10.10 → datamaestro-text-2023.11.22}/src/datamaestro_text.egg-info/SOURCES.txt

@@ -10,8 +10,6 @@ mkdocs.yml
 pyproject.toml
 requirements-dev.txt
 requirements.txt
-setup.cfg
-setup.py
 tox.ini
 .circleci/config.yml
 .github/workflows/pytest.yml
@@ -39,7 +37,6 @@ src/datamaestro_text.egg-info/dependency_links.txt
 src/datamaestro_text.egg-info/entry_points.txt
 src/datamaestro_text.egg-info/requires.txt
 src/datamaestro_text.egg-info/top_level.txt
-src/datamaestro_text.egg-info/zip-safe
 src/datamaestro_text/config/__init__.py
 src/datamaestro_text/config/ai/quac.yaml
 src/datamaestro_text/config/com/oscar-corpus.py

{datamaestro-text-2023.10.10 → datamaestro-text-2023.11.22}/src/datamaestro_text.egg-info/requires.txt

@@ -1,5 +1,2 @@
 datamaestro>=0.8.16
 attrs
-
-[test]
-tox

datamaestro-text-2023.10.10/setup.cfg

@@ -1,49 +0,0 @@
-[metadata]
-name = datamaestro_text
-author = Benjamin Piwowarski
-author-email = benjamin@piwowarski.fr
-home-page = https://github.com/experimaestro/datamaestro_text
-description = "Text related datasets"
-long-description = file: README.md, CHANGELOG.md
-long-description-content-type = text/markdown
-license = GPL-3
-license_file = LICENSE
-platform = any
-keywords = dataset manager
-classifiers = 
-	Development Status :: 4 - Beta
-	Intended Audience :: Science/Research
-	License :: OSI Approved :: GNU General Public License v3 (GPLv3)
-	Operating System :: OS Independent
-	Programming Language :: Python
-	Programming Language :: Python :: 3
-	Topic :: Software Development :: Libraries :: Python Modules
-
-[options]
-zip_safe = true
-include_package_data = true
-python_requires = >= 3.8
-test_suite = datamaestro.test
-setup_requires = 
-	setuptools >= 65.0.0
-	setuptools_scm
-	wheel
-
-[options.extras_require]
-test = 
-	tox
-
-[mypy]
-python_version = 3.7
-warn_unused_ignores = True
-
-[flake8]
-doctests = True
-exclude = .git, .eggs, __pycache__, tests/, docs/, build/, dist/
-max-line-length = 88
-extend-ignore = E203
-
-[egg_info]
-tag_build = 
-tag_date = 0
-

datamaestro-text-2023.10.10/setup.py

@@ -1,9 +0,0 @@
-from pathlib import Path
-from setuptools import setup
-
-basepath = Path(__file__).parent
-
-
-setup(
-    install_requires=(basepath / "requirements.txt").read_text(), use_scm_version=True,
-)

datamaestro-text-2023.10.10/src/datamaestro_text/data/ir/formats.py

@@ -1,26 +0,0 @@
-from typing import ClassVar
-from attrs import define
-from .base import IDHolder, Document, GenericTopic
-
-
-@define
-class CordDocument(IDHolder, Document):
-    text: str
-    title: str
-    url: str
-    pubmed_id: str
-
-    has_text: ClassVar[bool] = True
-
-    def get_text(self):
-        return f"{self.title} {self.text}"
-
-
-@define
-class TrecTopic(GenericTopic):
-    text: str
-    query: str
-    narrative: str
-
-    def get_text(self):
-        return f"{self.query}"

datamaestro-text-2023.10.10/src/datamaestro_text.egg-info/zip-safe

@@ -1 +0,0 @@
-