ebook2text 2.1.2__tar.gz → 2.2.0__tar.gz

{ebook2text-2.1.2 → ebook2text-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: ebook2text
-Version: 2.1.2
+Version: 2.2.0
 Summary: Convert common book file types to text for machine learning
 Author: Ashlynn Antrobus
 Author-email: Ashlynn Antrobus <ashlynn@prosepal.io>
@@ -21,10 +21,10 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Text Processing
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: beautifulsoup4>=4.12.3
-Requires-Dist: ebooklib-autoupdate>=0.18.1
+Requires-Dist: ebooklib-autoupdate>=0.18.3
 Requires-Dist: openai>=1.54.3
 Requires-Dist: pdfminer-six>=20240706
 Requires-Dist: pillow>=10.4.0

ebook2text-2.2.0/ebook2text/VERSION.py

@@ -0,0 +1 @@
+__version__ = "2.2.0"

{ebook2text-2.1.2 → ebook2text-2.2.0}/ebook2text/chapter_check.py

@@ -48,9 +48,15 @@ NOT_CHAPTER = {
 def roman_to_int(roman_num: str) -> int:
     """
     Convert a Roman numeral to an integer.
+
     Arguments:
         roman_num (str): A string representing the Roman numeral.
-    Returns int: The integer value of the Roman numeral.
+
+    Returns:
+        int: The integer value of the Roman numeral.
+
+    Raises:
+        ValueError if string is not a Roman numeral.
     """
     if not isinstance(roman_num, str):
         raise TypeError("Input must be a string")
@@ -61,7 +67,7 @@ def roman_to_int(roman_num: str) -> int:
         if roman.count(numeral) > 1:
             raise ValueError("Not roman numeral")
 
-    ROMAN_NUMERALS = {
+    roman_numerals = {
         "I": 1,
         "V": 5,
         "X": 10,
@@ -77,7 +83,7 @@ def roman_to_int(roman_num: str) -> int:
     previous_char: str = ""
 
     for char in reversed(roman):
-        if char not in ROMAN_NUMERALS:
+        if char not in roman_numerals:
             raise ValueError("Not roman numeral")
 
         consecutive_count = (
@@ -86,7 +92,7 @@ def roman_to_int(roman_num: str) -> int:
         if consecutive_count > 3:
             raise ValueError("Not roman numeral")
 
-        value = ROMAN_NUMERALS[char]
+        value = roman_numerals[char]
         if value >= prev_value:
             total += value
         else:

{ebook2text-2.1.2 → ebook2text-2.2.0}/ebook2text/convert_file.py

@@ -1,5 +1,4 @@
 from pathlib import Path
-from typing import Union
 
 from ebook2text.docx_conversion import DocxConverter, initialize_docx_converter
 from ebook2text.epub_conversion import EpubConverter, initialize_epub_converter
@@ -9,7 +8,7 @@ from ebook2text.text_parser import TextParser
 
 def _initialize_converter(
     file_path: Path, metadata: dict, extension: str
-) -> Union[DocxConverter, EpubConverter, PDFConverter, TextParser]:
+) -> DocxConverter | EpubConverter | PDFConverter | TextParser:
     """
     Initialize the appropriate converter based on the file extension.
 
@@ -51,8 +50,8 @@ def convert_file(
     metadata: dict,
     *,
     save_file: bool = True,
-    save_path: Union[Path, None] = None,
-) -> Union[str, None]:
+    save_path: Path | None = None,
+) -> str | None:
     """
     Converts a book to a text file with 3 asterisks for chapter breaks
     Args:
@@ -77,4 +76,4 @@ def convert_file(
     for content in converter.parse_file():
         if content:
             converter.write_text(content, save_path)
-    return
+    return None

{ebook2text-2.1.2 → ebook2text-2.2.0}/ebook2text/docx_conversion/__init__.py

@@ -1,5 +1,5 @@
+from collections.abc import Generator
 from pathlib import Path
-from typing import Generator
 
 from ebook2text.docx_conversion.docx_converter import DocxConverter
 from ebook2text.docx_conversion.docx_image_extractor import DocxImageExtractor

{ebook2text-2.1.2 → ebook2text-2.2.0}/ebook2text/docx_conversion/docx_converter.py

@@ -1,5 +1,5 @@
+from collections.abc import Generator
 from pathlib import Path
-from typing import Generator, Tuple
 
 import docx
 from docx.oxml.exceptions import XmlchemyError
@@ -211,7 +211,7 @@ class DocxConverter:
 
     def _process_text(
         self, paragraph_text: str, current_para_index: int
-    ) -> Tuple[str, int]:
+    ) -> tuple[str, int]:
         """
         Process a paragraph's text to determine if it starts a new chapter and
         format it accordingly.

{ebook2text-2.1.2 → ebook2text-2.2.0}/ebook2text/docx_conversion/docx_image_extractor.py

@@ -1,5 +1,3 @@
-from typing import List
-
 from ebook2text import logger
 from ebook2text._types import Paragraph
 from ebook2text.docx_conversion._namespaces import docx_ns_map
@@ -11,7 +9,7 @@ class DocxImageExtractor:
     A class dedicated to extracting images from docx Paragraph objects.
     """
 
-    def extract_images(self, paragraph: Paragraph) -> List[str]:
+    def extract_images(self, paragraph: Paragraph) -> list[str]:
         """
         Extracts and converts images found in the paragraph into
         base64-encoded strings.
@@ -48,8 +46,8 @@ class DocxImageExtractor:
         blips = paragraph._p.findall(".//a:blip", namespaces=namespace)
         for blip in blips:
             try:
-                rId = blip.attrib[f"{{{namespace['r']}}}embed"]
-                image_part = paragraph.part.related_parts[rId]
+                r_id = blip.attrib[f"{{{namespace['r']}}}embed"]
+                image_part = paragraph.part.related_parts[r_id]
                 blob = image_part.blob
                 image_blobs.append(blob)
             except (KeyError, ValueError, AttributeError) as e:

{ebook2text-2.1.2 → ebook2text-2.2.0}/ebook2text/epub_conversion/__init__.py

@@ -1,5 +1,5 @@
+from collections.abc import Generator
 from pathlib import Path
-from typing import Generator
 
 from ebook2text.epub_conversion.epub_converter import EpubConverter
 from ebook2text.epub_conversion.epub_text_extractor import EpubTextExtractor

{ebook2text-2.1.2 → ebook2text-2.2.0}/ebook2text/epub_conversion/epub_converter.py

@@ -1,5 +1,5 @@
+from collections.abc import Generator
 from pathlib import Path
-from typing import Generator
 
 import ebooklib
 from bs4 import BeautifulSoup
@@ -82,9 +82,9 @@ class EpubConverter:
         Returns:
             str: String containing the text of the chapter.
         """
-        TEXT_ELEMENTS = ["p", "img", "h1", "h2", "h3", "h4", "h5", "h6"]
+        text_elements = ["p", "img", "h1", "h2", "h3", "h4", "h5", "h6"]
         soup = BeautifulSoup(item.content, "html.parser")
-        elements: ResultSet[Tag] = soup.find_all(TEXT_ELEMENTS)
+        elements: ResultSet[Tag] = soup.find_all(text_elements)
 
         for i, element in enumerate(elements[: self.max_lines_to_check]):
             text = self.text_extractor.extract_text(element, self.epub_book)

{ebook2text-2.1.2 → ebook2text-2.2.0}/ebook2text/epub_conversion/epub_text_extractor.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from ebook2text._types import EpubBook, Tag
 from ebook2text.ocr import encode_image_bytes, run_ocr
 
@@ -9,9 +7,7 @@ class EpubTextExtractor:
     Extracts text from EPUB elements, handling image OCR.
     """
 
-    def extract_text(
-        self, element: Tag, book: Optional[EpubBook] = None
-    ) -> str:
+    def extract_text(self, element: Tag, book: EpubBook | None = None) -> str:
         """
         Extracts text from an element, using OCR for images.
 

{ebook2text-2.1.2 → ebook2text-2.2.0}/ebook2text/ocr.py

@@ -9,7 +9,9 @@ from . import logger
 from ._exceptions import NoResponseError
 
 load_dotenv()
-api_key: str = os.getenv("OPENAI_API_KEY", "")
+api_key: str = os.environ["OPENAI_API_KEY"]
+MODEL: str = os.environ["OPENAI_MODEL"]
+
 CLIENT: OpenAI = OpenAI(api_key=api_key)
 
 GPT_REFUSALS = [
@@ -96,7 +98,7 @@ def run_ocr(
 
     try:
         response: ChatCompletion = client.chat.completions.create(
-            model="gpt-4o-mini", messages=payload, max_tokens=10
+            model=MODEL, messages=payload, max_tokens=10
         )
         if response.choices and response.choices[0].message.content:
             answer: str = response.choices[0].message.content

{ebook2text-2.1.2 → ebook2text-2.2.0}/ebook2text/pdf_conversion/__init__.py

@@ -1,5 +1,5 @@
+from collections.abc import Generator
 from pathlib import Path
-from typing import Generator
 
 from ebook2text.pdf_conversion.pdf_converter import PDFConverter
 from ebook2text.pdf_conversion.pdf_image_extractor import PDFImageExtractor

{ebook2text-2.1.2 → ebook2text-2.2.0}/ebook2text/pdf_conversion/pdf_converter.py

@@ -1,6 +1,6 @@
 import re
+from collections.abc import Generator
 from pathlib import Path
-from typing import Generator, List
 
 from pdfminer.high_level import extract_pages
 from pdfminer.pdfparser import PDFSyntaxError
@@ -40,7 +40,7 @@ class PDFConverter:
         self._max_lines_to_check = 6
         self._chapter_separator = "***\n"
 
-        self._page: List[str] = []
+        self._page: list[str] = []
 
     def _read_file(self, file_path: Path) -> Generator[LTPage, None, None]:
         """
@@ -64,8 +64,7 @@ class PDFConverter:
 
     def _is_title_page(self, page_lines: list[str]) -> bool:
         """
-        Determines if the page is a title page by comparing the number of lines in
-        the page to a threshold.
+        Determines if title page by comparing page length to threshold.
         """
         return len(page_lines) < self._max_lines_to_check
 

{ebook2text-2.1.2 → ebook2text-2.2.0}/ebook2text/pdf_conversion/pdf_image_extractor.py

@@ -2,7 +2,6 @@ import base64
 import itertools
 from io import BytesIO
 from pathlib import Path
-from typing import List, Tuple
 
 from pdfminer.pdfdocument import PDFDocument
 from pdfminer.pdfinterp import resolve1
@@ -83,7 +82,7 @@ class PDFImageExtractor:
     def __init__(self, file_path: Path) -> None:
         self.file_path = file_path
 
-    def extract_images(self, obj_nums: List[int]) -> List[str]:
+    def extract_images(self, obj_nums: list[int]) -> list[str]:
         """
         Processes and extracts images from PDF objects based on the provided
         object numbers.
@@ -187,14 +186,16 @@ class PDFImageExtractor:
         """
         if attempt > 1:
             logger.warning(
-                f"Unable to extract image from object: {obj_num} or {obj_num - 1}"
+                f"Unable to extract image from object: {obj_num} "
+                f"or {obj_num - 1}"
             )
             return ""
         try:
             obj = resolve1(self.document.getobj(obj_num))
             if not isinstance(obj, PDFStream):
                 raise TypeError(
-                    f"Invalid object. Received {type(obj)} instead of PDFStream"
+                    "Invalid object. "
+                    f"Received {type(obj)} instead of PDFStream"
                 )
             if _convert_psliteral_to_str(obj.get("Filter")) == "DCTDecode":
                 return self._transcode_to_png(obj.get_data())
@@ -216,7 +217,7 @@ class PDFImageExtractor:
             logger.exception(f"Exception: {e}")
             return ""
 
-    def _find_next_image(self, obj_num: int, attempt: int) -> Tuple[int, int]:
+    def _find_next_image(self, obj_num: int, attempt: int) -> tuple[int, int]:
         """
         Finds the next image object in the PDF document.
 
@@ -238,7 +239,7 @@ class PDFImageExtractor:
 
     def _parse_image_data(
         self, stream: PDFStream
-    ) -> Tuple[int, int, str, bytes]:
+    ) -> tuple[int, int, str, bytes]:
         """
         Parses image data from a PDF object.
 
@@ -261,7 +262,7 @@ class PDFImageExtractor:
         stream = _expand_bits(stream.get_data(), bit_depth)
         return width, height, mode, stream
 
-    def _extract_color_data(self, stream: PDFStream) -> Tuple[str, int]:
+    def _extract_color_data(self, stream: PDFStream) -> tuple[str, int]:
         """
         Extracts color data from a PDF object.
 

{ebook2text-2.1.2 → ebook2text-2.2.0}/ebook2text/pdf_conversion/pdf_text_extractor.py

@@ -1,5 +1,3 @@
-from typing import List, Tuple, Union
-
 from pdfminer.pdfdocument import PDFSyntaxError
 
 from ebook2text import logger
@@ -24,7 +22,7 @@ class PDFTextExtractor:
         self._image_obj_nums: list[int] = []
         self._pdf_text_list: list[str] = []
 
-    def _match_objects(self, obj_type: str, obj_data: Union[int, str]) -> None:
+    def _match_objects(self, obj_type: str, obj_data: int | str) -> None:
         """
         Matches objects to their respective types.
 
@@ -46,16 +44,17 @@ class PDFTextExtractor:
         """
         Extracts text and image data from a PDF page.
 
-        This function receives an LTPage object extracted by PDFMiner and returns
-        a tuple containing a list of image object numbers and a list of strings
-        containing the text from the page.
+        This function receives an LTPage object extracted by PDFMiner and
+        returns a tuple containing a list of image object numbers and a list
+        of strings containing the text from the page.
 
         Args:pdf
             page (LTPage): The page to extract text and image data from.
 
         Returns:
-            tuple[list[int], list[str]]: A tuple containing a list of image object
-                numbers and a list of strings containing the text from the page.
+            tuple[list[int], list[str]]: A tuple containing a list of image
+                object numbers and a list of strings containing the text from
+                the page.
         """
 
         for element in page:
@@ -63,7 +62,7 @@ class PDFTextExtractor:
             self._match_objects(obj_type, obj_data)
         return self._image_obj_nums, self._pdf_text_list
 
-    def extract_text(self, page: LTPage) -> List[str]:
+    def extract_text(self, page: LTPage) -> list[str]:
         """
         Extracts text from a PDF page, including OCR text from images.
 
@@ -82,8 +81,8 @@ class PDFTextExtractor:
             page (LTPage): The PDF page represented as an LTPage object.
 
         Returns:
-            List[str]: The extracted text content from the PDF page, including OCR
-                text from images.
+            List[str]: The extracted text content from the PDF page, including
+                OCR text from images.
         """
         image_obj_nums, pdf_text_list = self._extract_element_data(page)
         self._image_obj_nums = []
@@ -99,7 +98,7 @@ class PDFTextExtractor:
             raise PDFConversionError from e
         return [ocr_text] + pdf_text_list
 
-    def _process_element(self, element: LTItem) -> Tuple[str, Union[int, str]]:
+    def _process_element(self, element: LTItem) -> tuple[str, int | str]:
         """
         Processes a PDF layout element to identify its type and extract
         relevant data.
@@ -128,17 +127,16 @@ class PDFTextExtractor:
                 return self._process_element(child)
         return "other", ""
 
-    def _extract_image_text(self, image_obj_nums: List[int]) -> str:
+    def _extract_image_text(self, image_obj_nums: list[int]) -> str:
         """
-        Collect list of Base64 encoded images and process with LLM to extract text.
-
+        Collect list of Base64 encoded images and extract text with LLM.
         Args:
             image_obj_nums (List[int]): List of image object numbers.
 
         Returns:
             str: Extracted text from images.
         """
-        base64_images: List[str] = self.image_extractor.extract_images(
+        base64_images: list[str] = self.image_extractor.extract_images(
             image_obj_nums
         )
         return run_ocr(base64_images)

{ebook2text-2.1.2 → ebook2text-2.2.0}/ebook2text/text_parser.py

@@ -1,5 +1,5 @@
+from collections.abc import Generator
 from pathlib import Path
-from typing import Generator
 
 from ebook2text._exceptions import TextConversionError
 from ebook2text._logger import logger

{ebook2text-2.1.2 → ebook2text-2.2.0}/ebook2text.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: ebook2text
-Version: 2.1.2
+Version: 2.2.0
 Summary: Convert common book file types to text for machine learning
 Author: Ashlynn Antrobus
 Author-email: Ashlynn Antrobus <ashlynn@prosepal.io>
@@ -21,10 +21,10 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Text Processing
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: beautifulsoup4>=4.12.3
-Requires-Dist: ebooklib-autoupdate>=0.18.1
+Requires-Dist: ebooklib-autoupdate>=0.18.3
 Requires-Dist: openai>=1.54.3
 Requires-Dist: pdfminer-six>=20240706
 Requires-Dist: pillow>=10.4.0

{ebook2text-2.1.2 → ebook2text-2.2.0}/ebook2text.egg-info/requires.txt

@@ -1,5 +1,5 @@
 beautifulsoup4>=4.12.3
-ebooklib-autoupdate>=0.18.1
+ebooklib-autoupdate>=0.18.3
 openai>=1.54.3
 pdfminer-six>=20240706
 pillow>=10.4.0

{ebook2text-2.1.2 → ebook2text-2.2.0}/pyproject.toml

@@ -21,10 +21,10 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Artificial Intelligence",
     "Topic :: Text Processing"
 ]
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 dependencies = [
     "beautifulsoup4>=4.12.3",
-    "ebooklib-autoupdate>=0.18.1",
+    "ebooklib-autoupdate>=0.18.3",
     "openai>=1.54.3",
     "pdfminer-six>=20240706",
     "pillow>=10.4.0",
@@ -42,15 +42,27 @@ build-backend = "setuptools.build_meta"
 
 [tool.ruff]
 line-length = 79
-force-exclude = true
+target-version = "py310"
 exclude = [".git", "__pycache__", ".venv", "dist", "build", ".vscode", ".egg-info"]
 
-[tool.isort]
-line_length = 79
+[tool.ruff.lint]
+select = ["E", "F", "I", "N", "UP"]
+
+[tool.ruff.lint.per-file-ignores]
+"**/{tests,docs,tools}/*" = ["F401", "I001", "E501", "N806"]
+"**/VERSION.py" = ["N999"]
+
+[tool.ruff.format]
+docstring-code-format = true
+docstring-code-line-length = "dynamic"
+line-ending = "lf"
 
 [tool.mypy]
-disable_error_code = "import-untyped"
-exclude = "ebooklib"
+mypy_path = "$MYPY_CONFIG_FILE_DIR/ebook2text"
+exclude_gitignore = true
+ignore_missing_imports = true
+follow_imports = "silent"
+follow_untyped_imports = false
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]

{ebook2text-2.1.2 → ebook2text-2.2.0}/tests/test_pdf_conversion.py

@@ -91,21 +91,21 @@ class TestPDFConverter:
     def test_parse_file_text_extraction(self, pdf_converter):
         """Test that parse_file extracts only the main content and excludes unwanted sections."""
         parsed_text = "".join(pdf_converter.parse_file())
-        assert (
-            "Sample Title" not in parsed_text
-        ), "Title text should be removed from output"
-        assert (
-            "Introduction" not in parsed_text
-        ), "Introduction section should be removed from output"
-        assert (
-            "Chapter 1" not in parsed_text
-        ), "Chapter 1 header should be removed from output"
-        assert (
-            "Chapter 2" not in parsed_text
-        ), "Chapter 2 header should be removed from output"
-        assert (
-            "First chapter paragraph text." in parsed_text
-        ), "Text after Chapter 1 should be included"
+        assert "Sample Title" not in parsed_text, (
+            "Title text should be removed from output"
+        )
+        assert "Introduction" not in parsed_text, (
+            "Introduction section should be removed from output"
+        )
+        assert "Chapter 1" not in parsed_text, (
+            "Chapter 1 header should be removed from output"
+        )
+        assert "Chapter 2" not in parsed_text, (
+            "Chapter 2 header should be removed from output"
+        )
+        assert "First chapter paragraph text." in parsed_text, (
+            "Text after Chapter 1 should be included"
+        )
         assert (
             "Lorem ipsum odor amet, consectetuer adipiscing elit."
             in parsed_text
@@ -113,9 +113,9 @@ class TestPDFConverter:
 
     def test_chapter_separator_insertion(self, pdf_converter):
         parsed_text = "".join(pdf_converter.parse_file())
-        assert (
-            pdf_converter._chapter_separator in parsed_text
-        ), "Chapter separator should be inserted between chapters"
+        assert pdf_converter._chapter_separator in parsed_text, (
+            "Chapter separator should be inserted between chapters"
+        )
 
     def test_page_combining(self, pdf_converter):
         page_lines = [
@@ -135,29 +135,29 @@ class TestPDFConverter:
         cleaned_text = pdf_converter.remove_extra_whitespace(
             text_with_whitespace
         )
-        assert (
-            cleaned_text == "Line 1\nLine 2 Line 3"
-        ), "Extra whitespace should be removed"
+        assert cleaned_text == "Line 1\nLine 2 Line 3", (
+            "Extra whitespace should be removed"
+        )
 
     def test_image_processing(self, pdf_converter_with_images):
         """Test parsing with an image-based PDF containing 'Chapter 1' as an image."""
         parsed_text = "".join(pdf_converter_with_images.parse_file())
 
-        assert (
-            "Sample Title" not in parsed_text
-        ), "Title text should be removed from output"
-        assert (
-            "Introduction" not in parsed_text
-        ), "Introduction section should be removed from output"
-        assert (
-            "Chapter 1" not in parsed_text
-        ), "Chapter 1 header should be removed from output"
-        assert (
-            "Chapter 2" not in parsed_text
-        ), "Chapter 2 header should be removed from output"
-        assert (
-            "First chapter paragraph text." in parsed_text
-        ), "Text after Chapter 1 should be included"
+        assert "Sample Title" not in parsed_text, (
+            "Title text should be removed from output"
+        )
+        assert "Introduction" not in parsed_text, (
+            "Introduction section should be removed from output"
+        )
+        assert "Chapter 1" not in parsed_text, (
+            "Chapter 1 header should be removed from output"
+        )
+        assert "Chapter 2" not in parsed_text, (
+            "Chapter 2 header should be removed from output"
+        )
+        assert "First chapter paragraph text." in parsed_text, (
+            "Text after Chapter 1 should be included"
+        )
         assert (
             "Lorem ipsum odor amet, consectetuer adipiscing elit."
             in parsed_text

ebook2text-2.1.2/ebook2text/VERSION.py

@@ -1 +0,0 @@
-__version__ = "2.1.2"