datamaestro-text 2026.1.1__tar.gz → 2026.2.2__tar.gz

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datamaestro-text
-Version: 2026.1.1
+Version: 2026.2.2
 Summary: Datamaestro module for text-related datasets
 Project-URL: Homepage, https://github.com/experimaestro/datamaestro_text
 Project-URL: Documentation, https://datamaestro-text.readthedocs.io/en/latest/
@@ -25,12 +25,6 @@ Requires-Dist: attrs
 Requires-Dist: datamaestro>=1.6.2
 Requires-Dist: experimaestro
 Requires-Dist: ir-datasets>=0.5.8
-Provides-Extra: dev
-Requires-Dist: docutils; extra == 'dev'
-Requires-Dist: flake8; extra == 'dev'
-Requires-Dist: pytest; extra == 'dev'
-Requires-Dist: sphinx<8; extra == 'dev'
-Requires-Dist: sphobjinv; extra == 'dev'
 Description-Content-Type: text/markdown
 
  [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) [![PyPI version](https://badge.fury.io/py/datamaestro-text.svg)](https://badge.fury.io/py/datamaestro-text)

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/pyproject.toml

@@ -32,15 +32,6 @@ dependencies = [
     "experimaestro",
 ]
 
-[project.optional-dependencies]
-dev = [
-    "pytest",
-    "docutils",
-    "sphobjinv",
-    "flake8",
-    "sphinx<8",
-]
-
 [project.urls]
 Homepage = "https://github.com/experimaestro/datamaestro_text"
 Documentation = "https://datamaestro-text.readthedocs.io/en/latest/"
@@ -79,9 +70,36 @@ norecursedirs = ["node_modules"]
 [dependency-groups]
 dev = [
     "docutils>=0.21.2",
-    "flake8>=7.3.0",
     "git-cliff>=2.11.0",
     "pytest>=8.4.1",
+    "ruff>=0.8",
     "sphinx>=7,<8",
     "sphobjinv>=2.3.1.3",
 ]
+docs = [
+    "datamaestro>=0.8.5",
+    "experimaestro>=2.0.0b29",  # Pre-release needed for EnumType.name() fix
+    "myst-parser>=0.18.0",
+    "sphinx>=6,<8",  # experimaestro.sphinx uses restify removed in Sphinx 8
+    "sphinx-codeautolink>=0.15.0",
+    "sphinx-rtd-theme>=3.1.0",
+    "sphinx-toolbox>=4.1.2",
+]
+
+[tool.ruff]
+line-length = 88
+target-version = "py310"
+
+[tool.ruff.lint]
+select = [
+    "E",      # pycodestyle errors
+    "F",      # pyflakes
+    "W",      # pycodestyle warnings
+    "C90",    # mccabe complexity
+]
+ignore = [
+    "E501",   # line too long (handled by formatter)
+]
+
+[tool.ruff.lint.mccabe]
+max-complexity = 20

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/__init__.py

@@ -1,6 +1,6 @@
 import datamaestro
 
-from .version import version, version_tuple
+from .version import version as version, version_tuple as version_tuple
 
 
 class Repository(datamaestro.Repository):

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/config/com/github/ikat.py

@@ -1,6 +1,5 @@
 # See documentation on https://datamaestro.readthedocs.io
 
-import bz2
 from datamaestro.download import reference
 from datamaestro.definitions import datatasks, datatags, dataset
 from datamaestro_text.data.conversation.base import ConversationUserTopics

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/config/com/oscar-corpus.py

@@ -1,4 +1,4 @@
-from datamaestro.definitions import datatasks, datatags, dataset
+from datamaestro.definitions import dataset
 from datamaestro.download.single import filedownloader
 from datamaestro_text.data.text import TextFile
 from datamaestro.utils import HashCheck

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/config/edu/stanford/aclimdb.py

@@ -1,5 +1,5 @@
 from datamaestro.data.ml import FolderBased, Supervised
-from datamaestro.definitions import datatasks, datatags, dataset
+from datamaestro.definitions import dataset
 from datamaestro.download.archive import tardownloader
 
 

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/config/edu/stanford/glove.py

@@ -5,7 +5,6 @@ GloVe is an unsupervised learning algorithm for obtaining vector representations
 """
 
 from datamaestro.definitions import dataset
-from datamaestro.data import Base, Generic
 from datamaestro.download import reference
 from datamaestro.download.archive import zipdownloader
 from datamaestro.download.single import filedownloader

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/config/gov/nist/ir/covid.py

@@ -1,5 +1,4 @@
-"""CORD-19 dataset
-"""
+"""CORD-19 dataset"""
 
 from datamaestro.annotations.agreement import useragreement
 from datamaestro.definitions import datatasks, dataset

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/config/io/metamind/research/wikitext.py

@@ -1,4 +1,4 @@
-from datamaestro.data import Base, File
+from datamaestro.data import File
 from datamaestro.definitions import (
     datatasks,
     datatags,

datamaestro_text-2026.2.2/src/datamaestro_text/data/conversation/__init__.py

@@ -0,0 +1,8 @@
+from .base import (
+    AnswerEntry as AnswerEntry,
+    ConversationDataset as ConversationDataset,
+    ConversationHistory as ConversationHistory,
+    ConversationHistoryItem as ConversationHistoryItem,
+    DecontextualizedItem as DecontextualizedItem,
+    EntryType as EntryType,
+)

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/data/conversation/base.py

@@ -2,13 +2,13 @@ from abc import ABC, abstractmethod
 from enum import Enum
 from datamaestro_text.data.ir.base import IDItem, SimpleTextItem
 from experimaestro import Param
-from typing import Dict, Generic, Iterator, List, Optional, Sequence, Tuple
+from typing import Dict, Iterator, List, Optional, Sequence, Tuple
 from attr import define
 from datamaestro.record import record_type
 from datamaestro.data import Base
 from datamaestro.record import Record, Item
 from datamaestro_text.data.ir import TopicRecord, Topics
-from datamaestro_text.utils.iter import FactoryIterable, LazyList, RangeView
+from datamaestro_text.utils.iter import FactoryIterable, LazyList
 
 # ---- Basic types
 

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/data/conversation/canard.py

@@ -11,7 +11,6 @@ from datamaestro_text.data.conversation.base import (
     EntryType,
 )
 from datamaestro_text.data.ir import IDItem, SimpleTextItem
-import logging
 
 
 @define(kw_only=True)
@@ -82,9 +81,9 @@ class CanardDataset(ConversationDataset, File):
                 )
             else:
                 # The utterance before the last is the last user query
-                assert (
-                    entry.history[-2] == history[-1][SimpleTextItem].text
-                ), f"{entry.dialogue_id} {entry.history} / {history[-4:-1]}"
+                assert entry.history[-2] == history[-1][SimpleTextItem].text, (
+                    f"{entry.dialogue_id} {entry.history} / {history[-4:-1]}"
+                )
 
                 # The last utterance is the system side
                 history.append(

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/data/conversation/ikat.py

@@ -5,7 +5,6 @@ import logging
 from datamaestro.data import File
 from datamaestro.record import Record
 
-from datamaestro_text.data.ir import Topics
 from datamaestro_text.data.ir.base import (
     IDItem,
     SimpleTextItem,

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/data/conversation/orconvqa.py

@@ -113,9 +113,9 @@ class OrConvQADataset(ConversationDataset, File):
                 if relevance > 0:
                     relevances[rank] = (entry.answer.answer_start, None)
 
-            assert (
-                len(relevances) <= 1
-            ), f"Too many relevance labels ({len(relevances)}) for {entry.query_id}"
+            assert len(relevances) <= 1, (
+                f"Too many relevance labels ({len(relevances)}) for {entry.query_id}"
+            )
 
             history.append(
                 Record(

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/data/embeddings.py

@@ -20,6 +20,7 @@ class WordEmbeddings(Base):
 
 class WordEmbeddingsText(WordEmbeddings, File):
     """Word embeddings as a text word / values"""
+
     encoding: Meta[str] = "utf-8"
 
     def load(self):

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/data/ir/__init__.py

@@ -6,7 +6,7 @@ from functools import cached_property
 import logging
 from pathlib import Path
 from attrs import define
-from typing import Callable, Dict, Iterator, List, Optional, Tuple, Type, TYPE_CHECKING
+from typing import Callable, Dict, Iterator, List, Optional, Tuple, Type
 import random
 from experimaestro import Config
 from datamaestro.definitions import datatasks, Param, Meta

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/data/ir/base.py

@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 from attrs import define
 from typing import List
-from datamaestro.record import Record, Item, record_type
+from datamaestro.record import Record, Item
 
 
 TopicRecord = DocumentRecord = Record

datamaestro_text-2026.2.2/src/datamaestro_text/data/ir/data.py

@@ -0,0 +1 @@
+from .base import *  # noqa: F403

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/data/ir/formats.py

@@ -1,5 +1,5 @@
 from functools import cached_property
-from typing import ClassVar, Tuple, List
+from typing import Tuple, List
 from attrs import define
 from datamaestro.record import record_type
 from ir_datasets.datasets.wapo import WapoDocMedia
@@ -10,6 +10,7 @@ from ir_datasets.datasets.cord19 import Cord19FullTextSection
 @define
 class DocumentWithTitle(TextItem):
     """Web document with title and body"""
+
     title: str
     body: str
 

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/data/ir/stores.py

@@ -82,7 +82,7 @@ class IKatClueWeb22DocumentStore(LZ4DocumentStore):
 
                 file_checksum = hasher.hexdigest()
                 assert file_checksum == checksum, (
-                    f"Expected {checksum}, " f"got {file_checksum} for {filename}"
+                    f"Expected {checksum}, got {file_checksum} for {filename}"
                 )
 
             # Get the MD5 hashes of all the passages

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/data/text.py

@@ -14,6 +14,7 @@ class TrainingText(Supervised):
 
 class TextFolder(Folder):
     "A folder composed of texts"
+
     pass
 
 

datamaestro_text-2026.2.2/src/datamaestro_text/datasets/__init__.py

@@ -0,0 +1 @@
+# IR datasets integration package

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/datasets/irds/data.py

@@ -1,6 +1,5 @@
 import logging
 from abc import ABC, abstractmethod
-from dataclasses import dataclass
 from functools import cached_property, partial
 from pathlib import Path
 from typing import Dict, Iterator, List, NamedTuple, Tuple, Type
@@ -613,11 +612,7 @@ class Cast2022TopicsHandler(CastTopicsHandler):
             records = []
             nodes: Dict[str, ConversationTreeNode] = {}
 
-            for (
-                query
-            ) in (
-                self.dataset.dataset.queries_iter()
-            ):  # type: _irds.trec_cast.Cast2022Query
+            for query in self.dataset.dataset.queries_iter():  # type: _irds.trec_cast.Cast2022Query
                 parent = nodes[query.parent_id] if query.parent_id else None
 
                 if query.participant == "User":

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/download/tmdb.py

@@ -10,7 +10,6 @@ from collections import namedtuple
 
 from datamaestro.download import Download
 from datamaestro.definitions import AbstractDataset
-from datamaestro.utils import TemporaryDirectory
 
 APIKEY_KEY = "org.themoviedb.apikey"
 

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/transforms/ir/__init__.py

@@ -2,7 +2,6 @@ import logging
 import gzip
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Type
 from experimaestro import Config, Task, Param, Annotated, pathgenerator, Option, tqdm
 import numpy as np
 from datamaestro.record import RecordType
@@ -131,22 +130,22 @@ class ShuffledTrainingTripletsLines(Task):
 
     def __validate__(self):
         if self.topic_ids:
-            assert self.data.topic_recordtype.has(
-                ir.IDItem
-            ), f"No topic ID in the source data ({self.data.topic_recordtype})"
+            assert self.data.topic_recordtype.has(ir.IDItem), (
+                f"No topic ID in the source data ({self.data.topic_recordtype})"
+            )
         else:
-            assert self.data.topic_recordtype.has(
-                ir.TextItem
-            ), f"No topic text in the source data ({self.data.topic_recordtype})"
+            assert self.data.topic_recordtype.has(ir.TextItem), (
+                f"No topic text in the source data ({self.data.topic_recordtype})"
+            )
 
         if self.doc_ids:
-            assert self.data.document_recordtype.has(
-                ir.IDItem
-            ), "No doc ID in the source data"
+            assert self.data.document_recordtype.has(ir.IDItem), (
+                "No doc ID in the source data"
+            )
         else:
-            assert self.data.document_recordtype.has(
-                ir.TextItem
-            ), "No doc text in the source data"
+            assert self.data.document_recordtype.has(ir.TextItem), (
+                "No doc text in the source data"
+            )
 
     def task_outputs(self, dep):
         return dep(

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/utils/shuffle.py

@@ -50,7 +50,7 @@ def shuffle(
     *,
     memory=MEMORY,
     random=None,
-    tmp_path: Optional[Path] = None
+    tmp_path: Optional[Path] = None,
 ):
     """Shuffle using temporary file"""
     if random is None:

{datamaestro_text-2026.1.1 → datamaestro_text-2026.2.2}/src/datamaestro_text/version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '2026.1.1'
-__version_tuple__ = version_tuple = (2026, 1, 1)
+__version__ = version = '2026.2.2'
+__version_tuple__ = version_tuple = (2026, 2, 2)
 
 __commit_id__ = commit_id = None

datamaestro_text-2026.1.1/src/datamaestro_text/data/conversation/__init__.py

@@ -1,8 +0,0 @@
-from .base import (
-    AnswerEntry,
-    ConversationDataset,
-    ConversationHistory,
-    ConversationHistoryItem,
-    DecontextualizedItem,
-    EntryType,
-)

datamaestro_text-2026.1.1/src/datamaestro_text/data/ir/data.py

@@ -1 +0,0 @@
-from .base import *