tfds-nightly 4.9.9.dev202508140045__tar.gz → 4.9.9.dev202508160044__tar.gz

{tfds_nightly-4.9.9.dev202508140045/tfds_nightly.egg-info → tfds_nightly-4.9.9.dev202508160044}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tfds-nightly
-Version: 4.9.9.dev202508140045
+Version: 4.9.9.dev202508160044
 Summary: tensorflow/datasets is a library of datasets ready to use with TensorFlow.
 Home-page: https://github.com/tensorflow/datasets
 Download-URL: https://github.com/tensorflow/datasets/tags

tfds_nightly-4.9.9.dev202508160044/tensorflow_datasets/datasets/multi_news/CITATIONS.bib

@@ -0,0 +1,8 @@
+@misc{alex2019multinews,
+    title={Multi-News: a Large-Scale Multi-Document Summarization Dataset and Abstractive Hierarchical Model},
+    author={Alexander R. Fabbri and Irene Li and Tianwei She and Suyi Li and Dragomir R. Radev},
+    year={2019},
+    eprint={1906.01749},
+    archivePrefix={arXiv},
+    primaryClass={cs.CL}
+}

tfds_nightly-4.9.9.dev202508160044/tensorflow_datasets/datasets/multi_news/README.md

@@ -0,0 +1,8 @@
+Multi-News, consists of news articles and human-written summaries
+of these articles from the site newser.com.
+Each summary is professionally written by editors and
+includes links to the original articles cited.
+
+There are two features:
+  - document: text of news articles seperated by special token "|||||".
+  - summary: news summary.

tfds_nightly-4.9.9.dev202508160044/tensorflow_datasets/datasets/multi_news/checksums.tsv

@@ -0,0 +1,6 @@
+https://huggingface.co/datasets/alexfabbri/multi_news/raw/main/data/test.src.cleaned	133	d04c4581d52321a30c246d2caa72853ee7f28c6b7a3985ee436f54c4bc264315	test.src.cleaned
+https://huggingface.co/datasets/alexfabbri/multi_news/raw/main/data/test.tgt	132	afba4aa26d95bb557c0eaa0cb8f7495af2104f1e43f4b5f9ef429b8752477abd	test.tgt
+https://huggingface.co/datasets/alexfabbri/multi_news/raw/main/data/train.src.cleaned	134	75f87b786ff1982bf1bd5803c6a7377d1834b81956ac680a6955789ba047cc0b	train.src.cleaned
+https://huggingface.co/datasets/alexfabbri/multi_news/raw/main/data/train.tgt	133	9f1e9b290a6aae1aa67bd5b361c934ee9db32486e5cd97d83184c097ef8b27e5	train.tgt
+https://huggingface.co/datasets/alexfabbri/multi_news/raw/main/data/val.src.cleaned	133	8df3ef6bd1882094de8120fa635c3abf758e10427f81f306aaa4786df7b57861	val.src.cleaned
+https://huggingface.co/datasets/alexfabbri/multi_news/raw/main/data/val.tgt	132	9c0377a443ea92b17449f7df17f1cdfa7c7ebbfe3a45f2f8cd7b3e0ffb47b1df	val.tgt

tfds_nightly-4.9.9.dev202508160044/tensorflow_datasets/datasets/multi_news/multi_news_dataset_builder.py

@@ -0,0 +1,78 @@
+# coding=utf-8
+# Copyright 2025 The TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""multi_news dataset."""
+
+from etils import epath
+import tensorflow_datasets.public_api as tfds
+
+_URL_PATH = (
+    "https://huggingface.co/datasets/alexfabbri/multi_news/raw/main/data/"
+)
+
+
+_DOCUMENT = "document"
+_SUMMARY = "summary"
+
+
+class Builder(tfds.core.GeneratorBasedBuilder):
+  """DatasetBuilder for multi_news dataset."""
+
+  VERSION = tfds.core.Version("2.0.0")
+
+  def _info(self) -> tfds.core.DatasetInfo:
+    """Returns the dataset metadata."""
+    return self.dataset_info_from_configs(
+        features=tfds.features.FeaturesDict(
+            {_DOCUMENT: tfds.features.Text(), _SUMMARY: tfds.features.Text()}
+        ),
+        supervised_keys=(_DOCUMENT, _SUMMARY),
+        homepage="https://github.com/Alex-Fabbri/Multi-News",
+    )
+
+  def _split_generators(self, dl_manager: tfds.download.DownloadManager):
+    """Returns SplitGenerators."""
+    data_dict = {
+        "train_src": _URL_PATH + "train.src.cleaned",
+        "train_tgt": _URL_PATH + "train.tgt",
+        "val_src": _URL_PATH + "val.src.cleaned",
+        "val_tgt": _URL_PATH + "val.tgt",
+        "test_src": _URL_PATH + "test.src.cleaned",
+        "test_tgt": _URL_PATH + "test.tgt",
+    }
+    files = dl_manager.download_and_extract(data_dict)
+    return {
+        "train": self._generate_examples(
+            files["train_src"], files["train_tgt"]
+        ),
+        "validation": self._generate_examples(
+            files["val_src"], files["val_tgt"]
+        ),
+        "test": self._generate_examples(files["test_src"], files["test_tgt"]),
+    }
+
+  def _generate_examples(self, src_file, tgt_file):
+    """Yields examples."""
+    with epath.Path(src_file).open() as src_f, epath.Path(
+        tgt_file
+    ).open() as tgt_f:
+      for i, (src_line, tgt_line) in enumerate(zip(src_f, tgt_f)):
+        yield i, {
+            # In original file, each line has one example and natural newline
+            # tokens "\n" are being replaced with "NEWLINE_CHAR". Here restore
+            # the natural newline token to avoid special vocab "NEWLINE_CHAR".
+            _DOCUMENT: src_line.strip().replace("NEWLINE_CHAR", "\n"),
+            _SUMMARY: tgt_line.strip().lstrip(),
+        }

tfds_nightly-4.9.9.dev202508160044/tensorflow_datasets/datasets/multi_news/multi_news_dataset_builder_test.py

@@ -0,0 +1,43 @@
+# coding=utf-8
+# Copyright 2025 The TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""multi_news dataset."""
+
+from tensorflow_datasets import testing
+from tensorflow_datasets.datasets.multi_news import multi_news_dataset_builder
+
+
+class MlqaTest(testing.DatasetBuilderTestCase):
+  """Tests for multi_news dataset."""
+
+  DATASET_CLASS = multi_news_dataset_builder.Builder
+  SPLITS = {
+      "train": 1,
+      "validation": 1,
+      "test": 1,
+  }
+
+  DL_EXTRACT_RESULT = {
+      "train_src": "train.src.cleaned",
+      "train_tgt": "train.tgt",
+      "val_src": "val.src.cleaned",
+      "val_tgt": "val.tgt",
+      "test_src": "test.src.cleaned",
+      "test_tgt": "test.tgt",
+  }
+
+
+if __name__ == "__main__":
+  testing.test_main()

tfds_nightly-4.9.9.dev202508160044/tensorflow_datasets/datasets/xtreme_pos/__init__.py

@@ -0,0 +1,15 @@
+# coding=utf-8
+# Copyright 2025 The TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+

{tfds_nightly-4.9.9.dev202508140045 → tfds_nightly-4.9.9.dev202508160044}/tensorflow_datasets/scripts/cli/build.py

@@ -33,42 +33,42 @@ from tensorflow_datasets.scripts.cli import cli_utils
 
 @dataclasses.dataclass(frozen=True, kw_only=True)
 class Args(cli_utils.Args):
-  """Commands for downloading and preparing datasets.
-
-  Attributes:
-    positional_datasets: Name(s) of the dataset(s) to build. Default to current
-      dir. See https://www.tensorflow.org/datasets/cli for accepted values.
-    datasets: Datasets can also be provided as keyword argument.
-    debug: Debug & tests options. Use --pdb to enter post-mortem debugging mode
-      if an exception is raised.
-    paths: Path options.
-    generation: Generation options.
-    publishing: Publishing options.
-    automation: Automation options.
-  """
+  """Commands for downloading and preparing datasets."""
 
-  positional_datasets: list[str] = simple_parsing.field(
+  # Name(s) of the dataset(s) to build. Default to current dir. See
+  # https://www.tensorflow.org/datasets/cli for accepted values.
+  positional_datasets: list[str] = simple_parsing.list_field(
       positional=True,
-      nargs='*',
-      default_factory=list,
       # Need to explicitly set metavar for command-line help.
       metavar='datasets',
   )
-  datasets: list[str] = simple_parsing.field(nargs='*', default_factory=list)
+
+  datasets: list[str] = simple_parsing.list_field(alias='--dataset')
+  """Datasets can also be provided as keyword argument."""
 
   debug: cli_utils.DebugOptions = cli_utils.DebugOptions()
+  """Debug & tests options. Use --pdb to enter post-mortem debugging mode if an
+  exception is raised."""
+
   paths: cli_utils.PathOptions = simple_parsing.field(
       default_factory=cli_utils.PathOptions
   )
+  """Path options."""
+
   generation: cli_utils.GenerationOptions = simple_parsing.field(
       default_factory=cli_utils.GenerationOptions
   )
+  """Generation options."""
+
   publishing: cli_utils.PublishingOptions = simple_parsing.field(
       default_factory=cli_utils.PublishingOptions
   )
+  """Publishing options."""
+
   automation: cli_utils.AutomationOptions = simple_parsing.field(
       default_factory=cli_utils.AutomationOptions
   )
+  """Automation options."""
 
   def execute(self) -> None:
     """Build the given datasets."""

{tfds_nightly-4.9.9.dev202508140045 → tfds_nightly-4.9.9.dev202508160044}/tensorflow_datasets/scripts/cli/cli_utils.py

@@ -184,12 +184,13 @@ class PathOptions:
   """Path options.
 
   Attributes:
-    data_dir: Where to place datasets. Default to `~/tensorflow_datasets/` or
+    data_dir: Where to place datasets. Defaults to `~/tensorflow_datasets/` or
       `TFDS_DATA_DIR` environement variable.
-    download_dir: Where to place downloads. Default to `<data_dir>/downloads/`.
-    extract_dir: Where to extract files. Default to `<download_dir>/extracted/`.
+    download_dir: Where to place downloads. Defaults to `<data_dir>/downloads/`.
+    extract_dir: Where to extract files. Defaults to
+      `<download_dir>/extracted/`.
     manual_dir: Where to manually download data (required for some datasets).
-      Default to `<download_dir>/manual/`.
+      Defaults to `<download_dir>/manual/`.
     add_name_to_manual_dir: If true, append the dataset name to the `manual_dir`
       (e.g. `<download_dir>/manual/<dataset_name>/`). Useful to avoid collisions
       if many datasets are generated.
@@ -243,10 +244,14 @@ class GenerationOptions:
 
   download_only: bool = simple_parsing.flag(default=False)
   config: str | None = simple_parsing.field(default=None, alias='-c')
-  config_idx: int | None = None
+  config_idx: int | None = simple_parsing.field(
+      default=None, alias='--builder_config_id'
+  )
   update_metadata_only: bool = simple_parsing.flag(default=False)
   download_config: str | None = None
-  imports: str | None = simple_parsing.field(default=None, alias='-i')
+  imports: str | None = simple_parsing.field(
+      default=None, alias=['-i', '--module_import']
+  )
   register_checksums: bool = simple_parsing.flag(default=False)
   force_checksums_validation: bool = simple_parsing.flag(default=False)
   beam_pipeline_options: str | None = None
@@ -320,7 +325,7 @@ def download_and_prepare(
   if nondeterministic_order:
     download_config.nondeterministic_order = True
 
-  # Add Apache Beam options to download config
+  # Add Apache Beam options to download config.
   try:
     import apache_beam as beam  # pylint: disable=g-import-not-at-top
 
@@ -338,8 +343,8 @@ def download_and_prepare(
       download_config=download_config,
   )
 
-  # Dataset generated successfully
-  logging.info('Dataset generation complete...')
+  # Dataset generated successfully.
+  logging.info('Dataset generation completed...')
 
   print()
   print(repr(builder.info))

{tfds_nightly-4.9.9.dev202508140045 → tfds_nightly-4.9.9.dev202508160044}/tensorflow_datasets/scripts/download_and_prepare.py

@@ -15,43 +15,21 @@
 
 r"""Wrapper around `tfds build`."""
 
-import typing
-
 from absl import app
 from absl import flags
-from absl import logging
-from tensorflow_datasets.scripts.cli import build
 from tensorflow_datasets.scripts.cli import main as main_cli
 
-module_import = flags.DEFINE_string('module_import', None, '`--imports` flag.')
-dataset = flags.DEFINE_string('dataset', None, 'singleton `--datasets` flag.')
-
-builder_config_id = flags.DEFINE_integer(
-    'builder_config_id', None, '`--config_idx` flag'
-)
-
-
 
 def _parse_flags(argv: list[str]) -> main_cli.Args:
   """Command lines flag parsing."""
   return main_cli._parse_flags([argv[0], 'build'] + argv[1:])  # pylint: disable=protected-access
 
 
-_display_warning = True
-
-
 def main(args: main_cli.Args) -> None:
-  if _display_warning:
-    logging.warning(
-        '***`tfds build` should be used instead of `download_and_prepare`.***'
-    )
-  cmd_args = typing.cast(build.Args, args.command)
-  if module_import.value:
-    cmd_args.generation.imports = module_import.value
-  if dataset.value:
-    cmd_args.datasets = [dataset.value]
-  if builder_config_id.value is not None:
-    cmd_args.generation.config_idx = builder_config_id.value
+  from absl import logging
+  logging.warning(
+      '***`tfds build` should be used instead of `download_and_prepare`.***'
+  )
   main_cli.main(args)
 
 

{tfds_nightly-4.9.9.dev202508140045 → tfds_nightly-4.9.9.dev202508160044}/tensorflow_datasets/summarization/__init__.py

@@ -24,7 +24,6 @@ from tensorflow_datasets.summarization.covid19sum import Covid19sum
 from tensorflow_datasets.summarization.gigaword import Gigaword
 from tensorflow_datasets.summarization.gov_report import GovReport
 from tensorflow_datasets.summarization.media_sum import MediaSum
-from tensorflow_datasets.summarization.multi_news import MultiNews
 from tensorflow_datasets.summarization.newsroom import Newsroom
 from tensorflow_datasets.summarization.opinion_abstracts import OpinionAbstracts
 from tensorflow_datasets.summarization.opinosis import Opinosis

tfds_nightly-4.9.9.dev202508160044/tensorflow_datasets/summarization/multi_news.py

@@ -0,0 +1,25 @@
+# coding=utf-8
+# Copyright 2025 The TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Dataset definition for Multi-News.
+
+DEPRECATED!
+If you want to use the Multi-News dataset builder class, use:
+tfds.builder_cls('multi_news')
+"""
+
+from tensorflow_datasets.core import lazy_builder_import
+
+Tedlium = lazy_builder_import.LazyBuilderImport('multi_news')

{tfds_nightly-4.9.9.dev202508140045 → tfds_nightly-4.9.9.dev202508160044/tfds_nightly.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tfds-nightly
-Version: 4.9.9.dev202508140045
+Version: 4.9.9.dev202508160044
 Summary: tensorflow/datasets is a library of datasets ready to use with TensorFlow.
 Home-page: https://github.com/tensorflow/datasets
 Download-URL: https://github.com/tensorflow/datasets/tags

{tfds_nightly-4.9.9.dev202508140045 → tfds_nightly-4.9.9.dev202508160044}/tfds_nightly.egg-info/SOURCES.txt

@@ -935,6 +935,13 @@ tensorflow_datasets/datasets/mlqa/__init__.py
 tensorflow_datasets/datasets/mlqa/checksums.tsv
 tensorflow_datasets/datasets/mlqa/mlqa_dataset_builder.py
 tensorflow_datasets/datasets/mlqa/mlqa_dataset_builder_test.py
+tensorflow_datasets/datasets/multi_news/CITATIONS.bib
+tensorflow_datasets/datasets/multi_news/README.md
+tensorflow_datasets/datasets/multi_news/TAGS.txt
+tensorflow_datasets/datasets/multi_news/__init__.py
+tensorflow_datasets/datasets/multi_news/checksums.tsv
+tensorflow_datasets/datasets/multi_news/multi_news_dataset_builder.py
+tensorflow_datasets/datasets/multi_news/multi_news_dataset_builder_test.py
 tensorflow_datasets/datasets/natural_instructions/CITATIONS.bib
 tensorflow_datasets/datasets/natural_instructions/README.md
 tensorflow_datasets/datasets/natural_instructions/TAGS.txt
@@ -2105,7 +2112,6 @@ tensorflow_datasets/summarization/covid19sum_test.py
 tensorflow_datasets/summarization/gigaword.py
 tensorflow_datasets/summarization/gigaword_test.py
 tensorflow_datasets/summarization/multi_news.py
-tensorflow_datasets/summarization/multi_news_test.py
 tensorflow_datasets/summarization/newsroom.py
 tensorflow_datasets/summarization/opinion_abstracts.py
 tensorflow_datasets/summarization/opinosis.py

tfds_nightly-4.9.9.dev202508140045/tensorflow_datasets/summarization/multi_news.py

@@ -1,101 +0,0 @@
-# coding=utf-8
-# Copyright 2025 The TensorFlow Datasets Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Multi-News dataset."""
-
-import os
-
-from tensorflow_datasets.core.utils.lazy_imports_utils import tensorflow as tf
-import tensorflow_datasets.public_api as tfds
-
-_CITATION = """
-@misc{alex2019multinews,
-    title={Multi-News: a Large-Scale Multi-Document Summarization Dataset and Abstractive Hierarchical Model},
-    author={Alexander R. Fabbri and Irene Li and Tianwei She and Suyi Li and Dragomir R. Radev},
-    year={2019},
-    eprint={1906.01749},
-    archivePrefix={arXiv},
-    primaryClass={cs.CL}
-}
-"""
-
-_DESCRIPTION = """
-Multi-News, consists of news articles and human-written summaries
-of these articles from the site newser.com.
-Each summary is professionally written by editors and
-includes links to the original articles cited.
-
-There are two features:
-  - document: text of news articles seperated by special token "|||||".
-  - summary: news summary.
-"""
-
-_URL = "https://drive.google.com/uc?export=download&id=1vRY2wM6rlOZrf9exGTm5pXj5ExlVwJ0C"
-
-_DOCUMENT = "document"
-_SUMMARY = "summary"
-
-
-class MultiNews(tfds.core.GeneratorBasedBuilder):
-  """Multi-News dataset."""
-
-  VERSION = tfds.core.Version("1.0.0")
-
-  def _info(self):
-    return tfds.core.DatasetInfo(
-        builder=self,
-        description=_DESCRIPTION,
-        features=tfds.features.FeaturesDict(
-            {_DOCUMENT: tfds.features.Text(), _SUMMARY: tfds.features.Text()}
-        ),
-        supervised_keys=(_DOCUMENT, _SUMMARY),
-        homepage="https://github.com/Alex-Fabbri/Multi-News",
-        citation=_CITATION,
-    )
-
-  def _split_generators(self, dl_manager):
-    """Returns SplitGenerators."""
-    extract_path = os.path.join(
-        dl_manager.download_and_extract(_URL), "multi-news-original"
-    )
-    return [
-        tfds.core.SplitGenerator(
-            name=tfds.Split.TRAIN,
-            gen_kwargs={"path": os.path.join(extract_path, "train")},
-        ),
-        tfds.core.SplitGenerator(
-            name=tfds.Split.VALIDATION,
-            gen_kwargs={"path": os.path.join(extract_path, "val")},
-        ),
-        tfds.core.SplitGenerator(
-            name=tfds.Split.TEST,
-            gen_kwargs={"path": os.path.join(extract_path, "test")},
-        ),
-    ]
-
-  def _generate_examples(self, path=None):
-    """Yields examples."""
-    with tf.io.gfile.GFile(
-        os.path.join(path + ".src")
-    ) as src_f, tf.io.gfile.GFile(os.path.join(path + ".tgt")) as tgt_f:
-      for i, (src_line, tgt_line) in enumerate(zip(src_f, tgt_f)):
-        yield i, {
-            # In original file, each line has one example and natural newline
-            # tokens "\n" are being replaced with "NEWLINE_CHAR". Here restore
-            # the natural newline token to avoid special vocab "NEWLINE_CHAR".
-            _DOCUMENT: src_line.strip().replace("NEWLINE_CHAR", "\n"),
-            # Remove the starting token "- " for every target sequence.
-            _SUMMARY: tgt_line.strip().lstrip("- "),
-        }

tfds_nightly-4.9.9.dev202508140045/tensorflow_datasets/summarization/multi_news_test.py

@@ -1,33 +0,0 @@
-# coding=utf-8
-# Copyright 2025 The TensorFlow Datasets Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Test for Multi-News dataset."""
-
-from tensorflow_datasets import testing
-from tensorflow_datasets.summarization import multi_news
-
-
-class MultiNewsTest(testing.DatasetBuilderTestCase):
-  DATASET_CLASS = multi_news.MultiNews
-  SPLITS = {
-      "train": 1,  # Number of fake train example
-      "validation": 1,  # Number of fake validation example
-      "test": 1,  # Number of fake test example
-  }
-  DL_EXTRACT_RESULT = ""
-
-
-if __name__ == "__main__":
-  testing.test_main()