ebook2text 1.1.0__tar.gz

ebook2text-1.1.0/PKG-INFO

@@ -0,0 +1,129 @@
+Metadata-Version: 2.1
+Name: ebook2text
+Version: 1.1.0
+Summary: Convert common book file types to text for machine learning
+Author: Ashlynn Antrobus
+Author-email: Ashlynn Antrobus <ashlynn@prosepal.io>
+License: MIT
+Project-URL: Respository, https://github.com/ashrobertsdragon/Ebook-conversion-to-Text-for-Machine-Learning
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE.txt
+Requires-Dist: annotated-types>=0.4.0
+Requires-Dist: anyio<5,>3.5
+Requires-Dist: beautifulsoup4>=4.12.3
+Requires-Dist: certifi>=2024.2.2
+Requires-Dist: cffi>=1.12
+Requires-Dist: colorama>=0.4.6
+Requires-Dist: cryptography>=42.0.7
+Requires-Dist: distro<2,>=1.7
+Requires-Dist: EbookLib==0.18
+Requires-Dist: h11<=0.15,>=0.13
+Requires-Dist: httpcore>1
+Requires-Dist: httpx<1,>=0.23.0
+Requires-Dist: idna<4,>2.8.0
+Requires-Dist: lxml>3.1.0
+Requires-Dist: openai>=1.30.1
+Requires-Dist: pillow>=10.2.0
+Requires-Dist: pycparser==2.22
+Requires-Dist: pydantic<3,>1.9
+Requires-Dist: pydantic-core==2.18.2
+Requires-Dist: python-docx>=1.1.0
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: six==1.16.0
+Requires-Dist: sniffio>=1.1.0
+Requires-Dist: soupsieve>1.2
+Requires-Dist: tqdm>=4.0.0
+Requires-Dist: typing-extensions>=4.9.0
+
+
+# Convert Ebook File
+
+## Overview
+
+This Python script provides functionality for converting various ebook file formats (EPUB, DOCX, PDF, TXT) into a standardized text format. The script processes each file, identifying chapters, and replaces chapter headers with asterisks. It also performs OCR (Optical Character Recognition) for image-based text using GPT-4o and standardizes the text by desmartenizing punctuation.
+
+## Features
+
+- **File Format Support**: Handles EPUB, DOCX, PDF, and TXT formats.
+- **Chapter Identification**: Detects and marks chapter breaks.
+- **OCR Capability**: Converts text from images using OCR.
+- **Text Standardization**: Replaces smart punctuation with ASCII equivalents.
+
+## Requirements
+
+To run this script, you need Python 3.8 or above and the following packages:
+
+- `python-docx`
+- `ebooklib`
+- `openai`
+- `python-dotenv`
+- `bs4`
+- `ebooklib`
+- `pdfminer.six`
+- `pillow`
+
+## Usage
+
+1. Ensure all dependencies are installed.
+2. Set your environment variable for the OpenAI API key.
+3. Place your ebook files in a known directory.
+4. Run the script with the path to the ebook file and a metadata dictionary with keys of 'title' and 'author' as arguments.
+
+## Functions
+
+- `read_text_file(file: str) -> str`: Reads a text file and returns its content.
+- `write_to_file(content: str, file: str)`: Writes content to a file.
+- `convert_file(file_path: str, metadata: dict) -> str`: Main function to convert an ebook file to text.
+
+## Contributing
+
+Contributions to this project are welcome. Please ensure that your code follows the existing style for consistency.
+
+## License
+
+This project is licensed by ProsePal LLC under the MIT license
+
+## Version History
+
+- **v0.1.0** (Release date: November 30, 2023)
+  - Initial release
+
+- **v0.1.1** (Release date: December 2, 2023)
+  - fixed false positives for is_number
+
+- **v0.2.0** (Release date: December 3, 2023)
+  - Conversion of docx files
+
+- **v0.3.0** (Release date: December 8, 2023)
+  - Conversion of PDF files
+
+- **v0.3.1** (Release date: Januar 23, 2024)
+  - fixed concantation of text in pdf conversion
+  - updated pillow version to secure version
+
+- **v1.0.0** (Release date: January 23, 2024)
+  - created library instead of single module
+
+- **v1.0.1** (Release date: March 13, 2024)
+  - setup.py and requirements.txt typo fixed
+
+- **v1.0.2** (Release date: May 17, 2024)
+  - added tests, fixex minor typos
+
+- **v1.1.0** (Release date: May 30, 2024)
+  - Change to abstract factory pattern

ebook2text-1.1.0/README.md

@@ -0,0 +1,77 @@
+
+# Convert Ebook File
+
+## Overview
+
+This Python script provides functionality for converting various ebook file formats (EPUB, DOCX, PDF, TXT) into a standardized text format. The script processes each file, identifying chapters, and replaces chapter headers with asterisks. It also performs OCR (Optical Character Recognition) for image-based text using GPT-4o and standardizes the text by desmartenizing punctuation.
+
+## Features
+
+- **File Format Support**: Handles EPUB, DOCX, PDF, and TXT formats.
+- **Chapter Identification**: Detects and marks chapter breaks.
+- **OCR Capability**: Converts text from images using OCR.
+- **Text Standardization**: Replaces smart punctuation with ASCII equivalents.
+
+## Requirements
+
+To run this script, you need Python 3.8 or above and the following packages:
+
+- `python-docx`
+- `ebooklib`
+- `openai`
+- `python-dotenv`
+- `bs4`
+- `ebooklib`
+- `pdfminer.six`
+- `pillow`
+
+## Usage
+
+1. Ensure all dependencies are installed.
+2. Set your environment variable for the OpenAI API key.
+3. Place your ebook files in a known directory.
+4. Run the script with the path to the ebook file and a metadata dictionary with keys of 'title' and 'author' as arguments.
+
+## Functions
+
+- `read_text_file(file: str) -> str`: Reads a text file and returns its content.
+- `write_to_file(content: str, file: str)`: Writes content to a file.
+- `convert_file(file_path: str, metadata: dict) -> str`: Main function to convert an ebook file to text.
+
+## Contributing
+
+Contributions to this project are welcome. Please ensure that your code follows the existing style for consistency.
+
+## License
+
+This project is licensed by ProsePal LLC under the MIT license
+
+## Version History
+
+- **v0.1.0** (Release date: November 30, 2023)
+  - Initial release
+
+- **v0.1.1** (Release date: December 2, 2023)
+  - fixed false positives for is_number
+
+- **v0.2.0** (Release date: December 3, 2023)
+  - Conversion of docx files
+
+- **v0.3.0** (Release date: December 8, 2023)
+  - Conversion of PDF files
+
+- **v0.3.1** (Release date: Januar 23, 2024)
+  - fixed concantation of text in pdf conversion
+  - updated pillow version to secure version
+
+- **v1.0.0** (Release date: January 23, 2024)
+  - created library instead of single module
+
+- **v1.0.1** (Release date: March 13, 2024)
+  - setup.py and requirements.txt typo fixed
+
+- **v1.0.2** (Release date: May 17, 2024)
+  - added tests, fixex minor typos
+
+- **v1.1.0** (Release date: May 30, 2024)
+  - Change to abstract factory pattern

ebook2text-1.1.0/ebook2text/VERSION.py

@@ -0,0 +1 @@
+__version__ = "1.1.0"

ebook2text-1.1.0/ebook2text/__init__.py

@@ -0,0 +1,6 @@
+from .convert_file import convert_file
+from .file_handling import read_text_file, write_to_file
+from .VERSION import __version__
+
+__version__ = __version__
+__all__ = ["convert_file", "read_text_file", "write_to_file"]

ebook2text-1.1.0/ebook2text/_exceptions.py

@@ -0,0 +1,24 @@
+class ImageSizeError(Exception):
+    """
+    Custom exception class for image size-related errors.
+    """
+
+    pass
+
+
+class ImageTooSmallError(ImageSizeError):
+    """
+    Custom exception class for handling errors related to an image being too
+    small.
+    """
+
+    pass
+
+
+class ImageTooLargeError(ImageSizeError):
+    """
+    Custom exception class for handling errors related to an image being too
+    large.
+    """
+
+    pass

ebook2text-1.1.0/ebook2text/_namespaces.py

@@ -0,0 +1,6 @@
+docx_ns_map = {
+    "a": "http://schemas.openxmlformats.org/drawingml/2006/main",
+    "pic": "http://schemas.openxmlformats.org/drawingml/2006/picture",
+    "w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main",
+    "r": "http://schemas.openxmlformats.org/officeDocument/2006/relationships",
+}

ebook2text-1.1.0/ebook2text/_types.py

@@ -0,0 +1,20 @@
+from bs4.element import ResultSet, Tag
+from docx.document import Document
+from docx.text.paragraph import Paragraph
+from ebooklib.epub import EpubBook, EpubItem
+from pdfminer.layout import LTChar, LTContainer, LTPage, LTText
+from pdfminer.pdftypes import PDFStream
+
+__all__ = [
+    "Document",
+    "EpubBook",
+    "EpubItem",
+    "LTChar",
+    "LTContainer",
+    "LTText",
+    "LTPage",
+    "Paragraph",
+    "PDFStream",
+    "ResultSet",
+    "Tag",
+]

ebook2text-1.1.0/ebook2text/abstract_book.py

@@ -0,0 +1,68 @@
+from abc import ABC, abstractmethod
+from typing import Generic, List, TypeVar
+
+from .text_conversion import desmarten_text
+
+T = TypeVar("T")
+
+
+class BookConversion(ABC, Generic[T]):
+    def __init__(self, file_path: str, metadata: dict):
+        self.file_path: str = file_path
+        self.metadata: dict = metadata
+        self.book = self._read_file(file_path)
+
+    @abstractmethod
+    def _read_file(self, file_path: str) -> T:
+        raise NotImplementedError("Must be implemented in child class")
+
+    @abstractmethod
+    def split_chapters(self) -> str:
+        raise NotImplementedError("Must be implemented in child class")
+
+    @abstractmethod
+    def extract_text(self, T) -> str:
+        raise NotImplementedError("Must be implemented in child class")
+
+    @abstractmethod
+    def extract_images(self, T) -> List[str]:
+        raise NotImplementedError("Must be implemented in child class")
+
+
+class ImageExtraction(ABC, Generic[T]):
+    @abstractmethod
+    def extract_images(self, T) -> List[str]:
+        raise NotImplementedError("Must be implemented in child class")
+
+
+class TextExtraction(ABC, Generic[T]):
+    @abstractmethod
+    def extract_text(self, T) -> str:
+        raise NotImplementedError("Must be implemented in child class")
+
+    @abstractmethod
+    def _extract_image_text(self, T) -> str:
+        raise NotImplementedError("Must be implemented in child class")
+
+
+class ChapterSplit(ABC, Generic[T]):
+    @abstractmethod
+    def __init__(
+        self, text_obj: T, metadata: dict, converter: BookConversion
+    ) -> None:
+        self.text_obj = text_obj
+        self.metadata = metadata
+        self.converter = converter
+
+        self.MAX_LINES_TO_CHECK: int = 3
+        self.CHAPTER_SEPARATOR: str = "***"
+
+    @abstractmethod
+    def split_chapters(self) -> str:
+        raise NotImplementedError("Must be implemented in child class")
+
+    def clean_text(self, text: str) -> str:
+        """
+        Removes smart punctuation from text
+        """
+        return desmarten_text(text)

ebook2text-1.1.0/ebook2text/chapter_check.py

@@ -0,0 +1,229 @@
+NOT_CHAPTER = {
+    "about",
+    "acknowledgements",
+    "afterward",
+    "annotation",
+    "appendix",
+    "assessment",
+    "backmatter",
+    "bibliography",
+    "colophon",
+    "conclusion",
+    "contents",
+    "contributors",
+    "copyright",
+    "cover",
+    "credits",
+    "dedication",
+    "division",
+    "endnotes",
+    "epigraph",
+    "errata",
+    "footnotes",
+    "forward",
+    "frontmatter",
+    "glossary",
+    "imprintur",
+    "imprint",
+    "index",
+    "introduction",
+    "landmarks",
+    "list",
+    "notice",
+    "page",
+    "preamble",
+    "preface",
+    "prologue",
+    "question",
+    "rear",
+    "revision",
+    "sign up",
+    "table",
+    "toc",
+    "volume",
+    "warning",
+}
+
+
+def roman_to_int(roman_num: str) -> int:
+    """
+    Convert a Roman numeral to an integer.
+    Arguments:
+        roman_num (str): A string representing the Roman numeral.
+    Returns int: The integer value of the Roman numeral.
+    """
+    if not isinstance(roman_num, str):
+        raise TypeError("Input must be a string")
+
+    roman: str = roman_num.upper()
+
+    for numeral in ("V", "L", "D"):
+        if roman.count(numeral) > 1:
+            raise ValueError("Not roman numeral")
+
+    ROMAN_NUMERALS = {
+        "I": 1,
+        "V": 5,
+        "X": 10,
+        "L": 50,
+        "C": 100,
+        "D": 500,
+        "M": 1000,
+    }
+
+    total: int = 0
+    prev_value: int = 0
+    consecutive_count: int = 1
+    previous_char: str = ""
+
+    for char in reversed(roman):
+        if char not in ROMAN_NUMERALS:
+            raise ValueError("Not roman numeral")
+
+        consecutive_count = (
+            consecutive_count + 1 if char == previous_char else 1
+        )
+        if consecutive_count > 3:
+            raise ValueError("Not roman numeral")
+
+        value = ROMAN_NUMERALS[char]
+        if value >= prev_value:
+            total += value
+        else:
+            if previous_char == "":
+                raise ValueError("Not roman numeral")
+            elif previous_char in ("V", "X") and char != "I":
+                raise ValueError("Not roman numeral")
+            elif previous_char in ("L", "C") and char != "X":
+                raise ValueError("Not roman numeral")
+            elif previous_char in ("D", "M") and char != "C":
+                raise ValueError("Not roman numeral")
+            total -= value
+        prev_value = value
+        previous_char = char
+    if not total:
+        raise ValueError("Not roman numeral")
+    return total
+
+
+def word_to_num(number_string: str) -> int:
+    """
+    Convert a spelled-out number without spaces or hyphens to a string of the
+    integer. The function supports numbers from 0 to 99 and is case
+    insensitive.
+    Args:
+        number_string (str): A string representing the spelled-out number.
+    Returns:
+        int: The integer value of the spelled-out number.
+    """
+    if not isinstance(number_string, str):
+        raise TypeError("Must be a string")
+    if len(number_string) == 0:
+        raise ValueError("Must have a value")
+
+    num_words = {
+        "zero": 0,
+        "one": 1,
+        "two": 2,
+        "three": 3,
+        "four": 4,
+        "five": 5,
+        "six": 6,
+        "seven": 7,
+        "eight": 8,
+        "nine": 9,
+        "ten": 10,
+        "teen": 10,
+        "eleven": 11,
+        "twelve": 12,
+        "thirteen": 13,
+        "twenty": 20,
+        "thirty": 30,
+        "forty": 40,
+        "fifty": 50,
+        "sixty": 60,
+        "seventy": 70,
+        "eighty": 80,
+        "ninety": 90,
+    }
+
+    num_str_lower: str = number_string.lower()
+    cleaned_num_str: str = num_str_lower.replace("-", "").replace(" ", "")
+    total: int = 0
+    temp_word: str = ""
+
+    for char in reversed(cleaned_num_str):
+        temp_word = char + temp_word
+        if temp_word in num_words:
+            total += num_words[temp_word]
+            temp_word = ""
+
+    if temp_word:
+        raise ValueError(f"Unknown number word: {temp_word}")
+    return total
+
+
+def is_spelled_out_number(s: str) -> bool:
+    """
+    Try to convert the word to an integer. If it's not a valid spelled-out
+    number, it will raise a ValueError
+    """
+
+    try:
+        word_to_num(s)
+        return True
+    except ValueError:
+        return False
+
+
+def is_roman_numeral(word: str) -> bool:
+    """
+    Try to convert the word to an integer. If it"s not a valid roman numeral,
+    it will raise a ValueError
+    """
+
+    try:
+        roman_to_int(word)
+        return True
+    except ValueError:
+        return False
+    except TypeError:
+        return False
+
+
+def is_number(s: str) -> bool:
+    """
+    Check if a string contains a number value either as digits,
+    roman numerals, or spelled-out numbers.
+    """
+
+    return s.isdigit() or is_roman_numeral(s) or is_spelled_out_number(s)
+
+
+def is_chapter(s: str) -> bool:
+    """
+    Check if a string contains the word "chapter", a Roman numeral, a
+    spelled-out number, or a digit.
+    Arguments:
+        s (str): The string to check.
+    Returns bool: True if the string meets the criteria, False otherwise.
+    """
+    lower_s = s.lower().strip()
+    return lower_s.startswith("chapter") or (
+        len(lower_s.split()) == 1 and is_number(lower_s)
+    )
+
+
+def is_not_chapter(paragraph: str, metadata: dict) -> bool:
+    """
+    Checks if the given line is not a chapter.
+    """
+    title = metadata.get("title", "no title found")
+    author = metadata.get("author", "no author found")
+    paragraph = paragraph.lower()
+    return any(
+        paragraph.startswith(title.lower())
+        or paragraph.startswith(author.lower())
+        or paragraph.startswith(not_chapter_word)
+        for not_chapter_word in NOT_CHAPTER
+    )

ebook2text-1.1.0/ebook2text/convert_file.py

@@ -0,0 +1,46 @@
+import logging
+import os
+
+from .docx_conversion import read_docx
+from .epub_conversion import read_epub
+from .file_handling import read_text_file, write_to_file
+from .pdf_conversion import read_pdf
+from .text_conversion import parse_text_file
+
+
+def convert_file(file_path: str, metadata: dict) -> None:
+    """
+    Converts a book to a text file with 3 asterisks for chapter breaks
+    Args:
+        book_name: Name of the book.
+        folder_name: Name of the folder containing the book.
+    """
+
+    book_content = ""
+    folder, book_file = os.path.split(file_path)
+
+    book_file = book_file.replace(" ", "_")
+    book_file = book_file.replace("-", "_")
+    filename_list = book_file.split(".")
+
+    if len(filename_list) > 1:
+        base_name = "_".join(filename_list[:-1])
+    else:
+        base_name = filename_list[0]
+    extension = filename_list[-1].lower()
+
+    if extension == "epub":
+        book_content = read_epub(file_path, metadata)
+    elif extension == "docx":
+        book_content = read_docx(file_path, metadata)
+    elif extension == "pdf":
+        book_content = read_pdf(file_path, metadata)
+    elif extension == "txt" or extension == "text":
+        book_content = read_text_file(file_path)
+        book_content = parse_text_file(book_content)
+    else:
+        logging.error(f"Invalid file type {extension} for file {file_path}")
+
+    book_name = f"{base_name}.txt"
+    book_path = os.path.join(folder, book_name)
+    write_to_file(book_content, book_path)