web-novel-scraper 1.0.2__tar.gz

web_novel_scraper-1.0.2/.github/scripts/update_version.py

@@ -0,0 +1,65 @@
+import sys
+import re
+
+def update_version(file_path, increment):
+    """
+    Updates the version in the specified file.
+
+    Args:
+        file_path (str): Path to the file containing the version.
+        increment (str): The type of increment: "patch", "minor", or "major".
+    """
+    try:
+        # Read the file
+        with open(file_path, "r", encoding="utf-8") as file:
+            content = file.read()
+
+        # Find the current version using regex
+        match = re.search(r'__version__ = "(\d+)\.(\d+)\.(\d+)"', content)
+        if not match:
+            raise ValueError("Version not found in the file.")
+
+        # Extract MAJOR, MINOR, and PATCH values
+        major, minor, patch = map(int, match.groups())
+
+        # Increment the appropriate part
+        if increment == "major":
+            major += 1
+            minor = 0
+            patch = 0
+        elif increment == "minor":
+            minor += 1
+            patch = 0
+        elif increment == "patch":
+            patch += 1
+        else:
+            raise ValueError("Increment type must be 'major', 'minor', or 'patch'.")
+
+        # Generate the new version
+        new_version = f'{major}.{minor}.{patch}'
+
+        # Update the file content with the new version
+        updated_content = re.sub(r'__version__ = "(\d+)\.(\d+)\.(\d+)"',
+                                 f'__version__ = "{new_version}"',
+                                 content)
+
+        # Write the updated content back to the file
+        with open(file_path, "w", encoding="utf-8") as file:
+            file.write(updated_content)
+
+        print(new_version)  # Print the new version for the workflow
+
+    except Exception as e:
+        print(f"Error updating the version: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        print("Usage: python update_version.py <file_path> <increment>")
+        print("Example: python update_version.py web_novel_scraper/version.py patch")
+        sys.exit(1)
+
+    file_path = sys.argv[1]
+    increment = sys.argv[2].lower()
+    update_version(file_path, increment)

web_novel_scraper-1.0.2/.github/workflows/build.yaml

@@ -0,0 +1,31 @@
+name: Build Artifacts
+on:
+    [workflow_call, workflow_dispatch]
+permissions:
+    contents: read
+
+jobs:
+  release-build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: main
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      - name: Build release distributions
+        run: |
+          python -m pip install build hatchling
+          python -m build
+
+      - name: Verify build artifacts
+        run: test -n "$(ls -A dist)" || exit 1
+
+      - name: Upload distributions
+        uses: actions/upload-artifact@v4
+        with:
+          name: release-dists
+          path: dist/

web_novel_scraper-1.0.2/.github/workflows/publish.yaml

@@ -0,0 +1,88 @@
+name: Test, Build and Publish
+
+on:
+  workflow_dispatch:
+    inputs:
+      increment:
+        description: "Version increment type (PATCH, MINOR, MAJOR)"
+        required: false
+        default: "PATCH"
+        type: choice
+        options:
+        - "PATCH"
+        - "MINOR"
+        - "MAJOR"
+      publish_to_pypi:
+        description: "Publish to PyPI (yes or no)"
+        required: false
+        default: true
+        type: boolean
+
+permissions:
+  contents: write
+  id-token: write
+
+jobs:
+  run-tests:
+    uses: ./.github/workflows/test.yaml
+
+  bump-version:
+    needs: run-tests
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      - name: Bump version
+        run: |
+          NEW_VERSION=$(python .github/scripts/update_version.py web_novel_scraper/version.py ${{ inputs.increment }})
+          echo "NEW_VERSION=$NEW_VERSION" >> $GITHUB_ENV
+          
+      - name: Commit updated version
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git add web_novel_scraper/version.py
+          git commit -m "Bump version to ${{ env.NEW_VERSION }}"
+          git push
+
+      - name: Tag the new version
+        run: |
+          git tag -a "${{ env.NEW_VERSION }}" -m "Version ${{ env.NEW_VERSION }}"
+          git push origin "${{ env.NEW_VERSION }}"
+
+  build:
+    needs: bump-version
+    uses: ./.github/workflows/build.yaml
+    permissions:
+      contents: read
+      packages: write
+      actions: write
+
+  pypi-publish:
+    needs: build
+    if: ${{ inputs.publish_to_pypi }}
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    environment:
+      name: pypi
+      url: https://pypi.org/p/web-novel-scraper
+    steps:
+      - name: Retrieve release distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: release-dists
+          path: dist/
+
+      - name: Publish release distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist/

web_novel_scraper-1.0.2/.github/workflows/test.yaml

@@ -0,0 +1,21 @@
+name: Run tests
+on:
+    [workflow_call, workflow_dispatch]
+permissions:
+    contents: read
+
+jobs:
+  release-build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      - name: Build release distributions
+        run: |
+          python -m pip install .
+          web-novel-scraper --help

web_novel_scraper-1.0.2/.gitignore

@@ -0,0 +1,15 @@
+**.jpg
+**.png
+**.html
+**/tmp
+**/output
+**.epub
+**/__pycache__
+test.py
+**/.env
+**.txt
+!requirements.txt
+**/novelas
+web_novel_scraper/web_novel_scraper**
+dist
+docs/build

web_novel_scraper-1.0.2/.readthedocs.yaml

@@ -0,0 +1,25 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version, and other tools you might need
+build:
+  os: ubuntu-24.04
+  tools:
+    python: "3.13"
+
+# Build documentation in the "docs/source" directory with Sphinx
+sphinx:
+   configuration: docs/source/conf.py
+
+# Optionally, but recommended,
+# declare the Python requirements required to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+   install:
+   - requirements: docs/requirements.txt
+   # We need to install the requirements to autogenerate the click docs
+   - requirements: requirements.txt
+        

web_novel_scraper-1.0.2/PKG-INFO

@@ -0,0 +1,231 @@
+Metadata-Version: 2.4
+Name: web-novel-scraper
+Version: 1.0.2
+Summary: Python tool that allows you to scrape web novels from various sources and save them to more readable formats like EPUB.
+Project-URL: Homepage, https://github.com/ImagineBrkr/web-novel-scraper
+Project-URL: Documentation, https://web-novel-scraper.readthedocs.io
+Project-URL: Repository, https://github.com/ImagineBrkr/web-novel-scraper.git
+Author-email: ImagineBrkr <salvattore_25@hotmail.com>
+Keywords: Novel Downloader,Scraper,Web Novel,Web Novel Downloader,Web Novel Scraper
+Requires-Python: >=3.10
+Requires-Dist: bs4>=0.0.2
+Requires-Dist: click<9,>=8.0
+Requires-Dist: dataclasses-json<1,>=0.6.7
+Requires-Dist: ebooklib<1,>=0.18
+Requires-Dist: platformdirs
+Requires-Dist: python-dotenv
+Requires-Dist: requests
+Description-Content-Type: text/markdown
+
+# Web Novel scraper CLI
+
+## Table of Contents
+- [Introduction](#introduction)
+- [Installation](#installation)
+- [Basic Concepts](#basic-concepts)
+- [Commands](#commands)
+- [Basic Examples](#basic-examples)
+
+
+## Introduction
+This tool allows you to scrape web novels from various sources. I made it because my hands hurt from scrolling too much.
+
+## Installation
+To install the Web Novel Scraping CLI, you can use pip:
+
+```bash
+    pip install web-novel-scraper
+```
+Or you can manually install it:
+
+1. Clone the repository:
+    ```bash
+    git clone https://github.com/ImagineBrkr/web-novel-scraper.git
+    ```
+2. Navigate to the project directory:
+    ```bash
+    cd web-novel-scraper
+    ```
+3. Install the project:
+    ```bash
+    python -m pip install .
+    ```
+4. Run the CLI tool:
+    ```bash
+    web-novel-scraper
+    ```
+
+## Basic Concepts
+### Novel
+Refers to a novel which has at least, a Table of Contents (can be one or more) and chapters. 
+It also has some metadata that can be saved like author, language, tags, creation or end date, etc.
+
+### Table of Contents (TOC)
+Source of Truth for all the chapters the novel will have. It can be from a main URL (it will be requested and saved; if there is more than one page, they will also get requested and saved), or the HTML files can be added directly from a file. All the chapters are autogenerated from this TOC.
+
+### Chapters
+A Chapter comes from a URL, is requested and saved as a file on your local machine. Once a file is saved, you will not need to request it anymore.
+From this chapter you can get the Title and the Chapter Content.
+
+### Decoder
+A set of rules used to extract information from a chapter, such as links, content, title, etc.
+We use the host to identify which set of rules we will use. This can be added manually or generated from a TOC URL.
+Example:
+```json
+{
+    "host": "novelbin.me",
+    "has_pagination": false,
+    "title": {
+        "element": "h2 a.chr-title",
+        "id": null,
+        "class": null,
+        "selector": null,
+        "attributes": null,
+        "array": false,
+        "extract": {
+            "type": "attr",
+            "key": "title"
+        }
+    },
+    "content": {
+        "element": "div#chr-content",
+        "id": null,
+        "class": null,
+        "selector": null,
+        "attributes": null,
+        "array": true
+    },
+    "index": {
+        "element": null,
+        "id": null,
+        "class": null,
+        "selector": "ul.list-chapter li a",
+        "attributes": null,
+        "array": true
+    },
+    "next_page": {
+        "element": null,
+        "id": null,
+        "class": null,
+        "selector": null,
+        "attributes": null,
+        "array": true
+    }
+}
+```
+Uses BeautifulSoup selectors for more flexibility. You can specify the element, id, class, selector, and whether multiple tags will be used.
+
+- `has_pagination`: Used if there is a `toc_main_url` to find the URL of the next page, using `next_page`.
+- `index`: Gets the `href` of all tags found when searching the TOC.
+- `title` and `content`: The title and content of the chapter, respectively.
+
+In the example above:
+- The title is in an `a` tag within an `h2` tag with class `chr-title`, extracting the `title` attribute:
+    ```html
+    <h2><a class="chr-title" href="https://url-of-chapter" title="Chapter 1"><span class="chr-text">Chapter 1</span></a></h2>
+    ```
+- The content is in a `div` with id `chr-content`:
+    ```html
+    <div id="chr-content" class="chr-c" style="font-family: Arial, sans-serif, serif; font-size: 18px; line-height: 160%; margin-top: 15px;">Content...</div>
+    ```
+- The URL of each chapter is in the `href` of an `a` tag within an `li` tag, which is within a `ul` tag with class `list-chapter`:
+    ```html
+    <ul class="list-chapter">
+        <li><span class="glyphicon glyphicon-certificate"></span>&nbsp;<a href="https://url-of-chapter-1" title="Chapter 1"><span class="nchr-text chapter-title">Chapter 1</span></a></li>
+    </ul>
+    ```
+## Commands
+The following commands are available in the Web Novel Scraping CLI:
+
+```bash
+Usage: main.py [OPTIONS] COMMAND [ARGS]...
+
+  CLI Tool for web novel scraping.
+
+Options:
+  --help  Show this message and exit.
+
+Commands:
+  add-tags                Add tags to a novel.
+  add-toc-html            Add TOC HTML to a novel.
+  clean-files             Clean files of a novel.
+  create-novel            Create a new novel.
+  delete-toc              Delete the TOC of a novel.
+  remove-tags             Remove tags from a novel.
+  request-all-chapters    Request all chapters of a novel.
+  save-novel-to-epub      Save the novel to EPUB format.
+  scrap-chapter           Scrap a chapter of a novel.
+  set-cover-image         Set the cover image for a novel.
+  set-host                Set the host for a novel.
+  set-metadata            Set metadata for a novel.
+  set-scraper-behavior   Set scraper behavior for a novel.
+  set-toc-main-url        Set the main URL for the TOC of a novel.
+  show-chapters           Show chapters of a novel.
+  show-metadata           Show metadata of a novel.
+  show-novel-info         Show information about a novel.
+  show-scraper-behavior  Show scraper behavior of a novel.
+  show-tags               Show tags of a novel.
+  show-toc                Show the TOC of a novel.
+  sync-toc                Sync the TOC of a novel.
+  version                 Show program version.
+```
+
+## Basic Examples
+Here are some basic examples:
+
+### Example 1: Creating a Novel using a main URL
+```bash
+python src/main.py create-novel --title 'Novel 1' --author 'ImagineBrkr' --toc-main-url 'https://page.me/Novel-1/toc' --cover 'cover.jpg'
+```
+Some pages have too much JavaScript, so you can just copy the HTML manually to a file and create the novel from it:
+```bash
+python src/main.py create-novel --title 'Novel 1' --author 'ImagineBrkr' --toc-html 'toc.html' --host 'page.me' --cover 'cover.jpg'
+```
+If there is more than one page for the TOC, you can add them:
+```bash
+python src/main.py add-toc-html --title 'Novel 1' --toc-html 'toc2.html'
+```
+You can create the chapters from this TOC, or synchronize if they were already created but there are new chapters.
+```bash
+python src/main.py sync-toc --title 'Novel 1'
+```
+The default directory will be %APPDATA%/ImagineBrkr/web-novel-scraper for Windows, all the files will be saved there, but you can change it.
+
+### Example 2: Requesting files
+We can now download all the chapters
+```bash
+python src/main.py request-all-chapters --title 'Novel 1'
+```
+
+### Example 3: Saving to EPUB
+With 
+```bash
+python src/main.py save-novel-to-epub --title 'Novel 1'
+```
+
+For more detailed usage and options, use --help for each command.
+
+## Configuration
+### Environment Variables
+
+The Web Novel Scraping CLI uses the following environment variables for configuration:
+
+- `SCRAPER_LOGGING_LEVEL`: Sets the logging level for the application. By default no logs are written, it accepts the following log levels: (DEBUG, INFO, WARNING, ERROR, CRITICAL).
+    ```bash
+    export SCRAPER_LOGGING_LEVEL=INFO
+    ```
+
+- `SCRAPER_LOGGING_FILE`: Specifies the file where logs will be written. Default is written to the terminal.
+    ```bash
+    export SCRAPER_LOGGING_FILE=/path/to/logfile.log
+    ```
+
+- `SCRAPER_BASE_DATA_DIR`: Defines the base directory for storing novel data. Default is the user data directory.
+    ```bash
+    export SCRAPER_BASE_DATA_DIR=/path/to/data/dir
+    ```
+
+- `SCRAPER_FLARESOLVER_URL`: URL for the FlareSolverr service. Default is `http://localhost:8191/v1`.
+    ```bash
+    export SCRAPER_FLARESOLVER_URL=http://localhost:8191/v1
+    ```

web_novel_scraper-1.0.2/README.md

@@ -0,0 +1,212 @@
+# Web Novel scraper CLI
+
+## Table of Contents
+- [Introduction](#introduction)
+- [Installation](#installation)
+- [Basic Concepts](#basic-concepts)
+- [Commands](#commands)
+- [Basic Examples](#basic-examples)
+
+
+## Introduction
+This tool allows you to scrape web novels from various sources. I made it because my hands hurt from scrolling too much.
+
+## Installation
+To install the Web Novel Scraping CLI, you can use pip:
+
+```bash
+    pip install web-novel-scraper
+```
+Or you can manually install it:
+
+1. Clone the repository:
+    ```bash
+    git clone https://github.com/ImagineBrkr/web-novel-scraper.git
+    ```
+2. Navigate to the project directory:
+    ```bash
+    cd web-novel-scraper
+    ```
+3. Install the project:
+    ```bash
+    python -m pip install .
+    ```
+4. Run the CLI tool:
+    ```bash
+    web-novel-scraper
+    ```
+
+## Basic Concepts
+### Novel
+Refers to a novel which has at least, a Table of Contents (can be one or more) and chapters. 
+It also has some metadata that can be saved like author, language, tags, creation or end date, etc.
+
+### Table of Contents (TOC)
+Source of Truth for all the chapters the novel will have. It can be from a main URL (it will be requested and saved; if there is more than one page, they will also get requested and saved), or the HTML files can be added directly from a file. All the chapters are autogenerated from this TOC.
+
+### Chapters
+A Chapter comes from a URL, is requested and saved as a file on your local machine. Once a file is saved, you will not need to request it anymore.
+From this chapter you can get the Title and the Chapter Content.
+
+### Decoder
+A set of rules used to extract information from a chapter, such as links, content, title, etc.
+We use the host to identify which set of rules we will use. This can be added manually or generated from a TOC URL.
+Example:
+```json
+{
+    "host": "novelbin.me",
+    "has_pagination": false,
+    "title": {
+        "element": "h2 a.chr-title",
+        "id": null,
+        "class": null,
+        "selector": null,
+        "attributes": null,
+        "array": false,
+        "extract": {
+            "type": "attr",
+            "key": "title"
+        }
+    },
+    "content": {
+        "element": "div#chr-content",
+        "id": null,
+        "class": null,
+        "selector": null,
+        "attributes": null,
+        "array": true
+    },
+    "index": {
+        "element": null,
+        "id": null,
+        "class": null,
+        "selector": "ul.list-chapter li a",
+        "attributes": null,
+        "array": true
+    },
+    "next_page": {
+        "element": null,
+        "id": null,
+        "class": null,
+        "selector": null,
+        "attributes": null,
+        "array": true
+    }
+}
+```
+Uses BeautifulSoup selectors for more flexibility. You can specify the element, id, class, selector, and whether multiple tags will be used.
+
+- `has_pagination`: Used if there is a `toc_main_url` to find the URL of the next page, using `next_page`.
+- `index`: Gets the `href` of all tags found when searching the TOC.
+- `title` and `content`: The title and content of the chapter, respectively.
+
+In the example above:
+- The title is in an `a` tag within an `h2` tag with class `chr-title`, extracting the `title` attribute:
+    ```html
+    <h2><a class="chr-title" href="https://url-of-chapter" title="Chapter 1"><span class="chr-text">Chapter 1</span></a></h2>
+    ```
+- The content is in a `div` with id `chr-content`:
+    ```html
+    <div id="chr-content" class="chr-c" style="font-family: Arial, sans-serif, serif; font-size: 18px; line-height: 160%; margin-top: 15px;">Content...</div>
+    ```
+- The URL of each chapter is in the `href` of an `a` tag within an `li` tag, which is within a `ul` tag with class `list-chapter`:
+    ```html
+    <ul class="list-chapter">
+        <li><span class="glyphicon glyphicon-certificate"></span>&nbsp;<a href="https://url-of-chapter-1" title="Chapter 1"><span class="nchr-text chapter-title">Chapter 1</span></a></li>
+    </ul>
+    ```
+## Commands
+The following commands are available in the Web Novel Scraping CLI:
+
+```bash
+Usage: main.py [OPTIONS] COMMAND [ARGS]...
+
+  CLI Tool for web novel scraping.
+
+Options:
+  --help  Show this message and exit.
+
+Commands:
+  add-tags                Add tags to a novel.
+  add-toc-html            Add TOC HTML to a novel.
+  clean-files             Clean files of a novel.
+  create-novel            Create a new novel.
+  delete-toc              Delete the TOC of a novel.
+  remove-tags             Remove tags from a novel.
+  request-all-chapters    Request all chapters of a novel.
+  save-novel-to-epub      Save the novel to EPUB format.
+  scrap-chapter           Scrap a chapter of a novel.
+  set-cover-image         Set the cover image for a novel.
+  set-host                Set the host for a novel.
+  set-metadata            Set metadata for a novel.
+  set-scraper-behavior   Set scraper behavior for a novel.
+  set-toc-main-url        Set the main URL for the TOC of a novel.
+  show-chapters           Show chapters of a novel.
+  show-metadata           Show metadata of a novel.
+  show-novel-info         Show information about a novel.
+  show-scraper-behavior  Show scraper behavior of a novel.
+  show-tags               Show tags of a novel.
+  show-toc                Show the TOC of a novel.
+  sync-toc                Sync the TOC of a novel.
+  version                 Show program version.
+```
+
+## Basic Examples
+Here are some basic examples:
+
+### Example 1: Creating a Novel using a main URL
+```bash
+python src/main.py create-novel --title 'Novel 1' --author 'ImagineBrkr' --toc-main-url 'https://page.me/Novel-1/toc' --cover 'cover.jpg'
+```
+Some pages have too much JavaScript, so you can just copy the HTML manually to a file and create the novel from it:
+```bash
+python src/main.py create-novel --title 'Novel 1' --author 'ImagineBrkr' --toc-html 'toc.html' --host 'page.me' --cover 'cover.jpg'
+```
+If there is more than one page for the TOC, you can add them:
+```bash
+python src/main.py add-toc-html --title 'Novel 1' --toc-html 'toc2.html'
+```
+You can create the chapters from this TOC, or synchronize if they were already created but there are new chapters.
+```bash
+python src/main.py sync-toc --title 'Novel 1'
+```
+The default directory will be %APPDATA%/ImagineBrkr/web-novel-scraper for Windows, all the files will be saved there, but you can change it.
+
+### Example 2: Requesting files
+We can now download all the chapters
+```bash
+python src/main.py request-all-chapters --title 'Novel 1'
+```
+
+### Example 3: Saving to EPUB
+With 
+```bash
+python src/main.py save-novel-to-epub --title 'Novel 1'
+```
+
+For more detailed usage and options, use --help for each command.
+
+## Configuration
+### Environment Variables
+
+The Web Novel Scraping CLI uses the following environment variables for configuration:
+
+- `SCRAPER_LOGGING_LEVEL`: Sets the logging level for the application. By default no logs are written, it accepts the following log levels: (DEBUG, INFO, WARNING, ERROR, CRITICAL).
+    ```bash
+    export SCRAPER_LOGGING_LEVEL=INFO
+    ```
+
+- `SCRAPER_LOGGING_FILE`: Specifies the file where logs will be written. Default is written to the terminal.
+    ```bash
+    export SCRAPER_LOGGING_FILE=/path/to/logfile.log
+    ```
+
+- `SCRAPER_BASE_DATA_DIR`: Defines the base directory for storing novel data. Default is the user data directory.
+    ```bash
+    export SCRAPER_BASE_DATA_DIR=/path/to/data/dir
+    ```
+
+- `SCRAPER_FLARESOLVER_URL`: URL for the FlareSolverr service. Default is `http://localhost:8191/v1`.
+    ```bash
+    export SCRAPER_FLARESOLVER_URL=http://localhost:8191/v1
+    ```