par-scrape 0.1.0__tar.gz

par_scrape-0.1.0/.gitignore

@@ -0,0 +1,74 @@
+### PythonVanilla template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+.aider*
+**/venv
+**/.venv
+**/.env
+**/.idea
+/config.json
+/output/

par_scrape-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Paul Robello
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

par_scrape-0.1.0/PKG-INFO

@@ -0,0 +1,185 @@
+Metadata-Version: 2.3
+Name: par_scrape
+Version: 0.1.0
+Summary: A versatile web scraping tool with options for Selenium or Playwright, featuring OpenAI-powered data extraction and formatting.
+Project-URL: Homepage, https://github.com/paulrobello/par_scrape
+Project-URL: Documentation, https://github.com/paulrobello/par_scrape/blob/main/README.md
+Project-URL: Repository, https://github.com/paulrobello/par_scrape
+Project-URL: Issues, https://github.com/paulrobello/par_scrape/issues
+Project-URL: Discussions, https://github.com/paulrobello/par_scrape/discussions
+Project-URL: Wiki, https://github.com/paulrobello/par_scrape/wiki
+Author-email: Paul Robello <probello@gmail.com>
+Maintainer-email: Paul Robello <probello@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2024 Paul Robello
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: data extraction,openai,playwright,selenium,web scraping
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP :: Browsers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: Markup :: HTML
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: aiofiles>=24.1.0
+Requires-Dist: beautifulsoup4>=4.12.3
+Requires-Dist: html2text>=2024.2.26
+Requires-Dist: openai>=1.43.0
+Requires-Dist: openpyxl>=3.1.5
+Requires-Dist: pandas>=2.2.2
+Requires-Dist: playwright>=1.46.0
+Requires-Dist: pydantic>=2.9.0
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: rich>=13.8.0
+Requires-Dist: selenium>=4.24.0
+Requires-Dist: tabulate>=0.9.0
+Requires-Dist: tiktoken>=0.7.0
+Requires-Dist: typer>=0.12.5
+Requires-Dist: typing-extensions>=4.12.2
+Requires-Dist: webdriver-manager>=4.0.2
+Description-Content-Type: text/markdown
+
+# PAR Scrape
+
+[![PyPI](https://img.shields.io/pypi/v/par_scrape)](https://pypi.org/project/par_scrape/)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/par_scrape.svg)](https://pypi.org/project/par_scrape/)  
+![Runs on Linux | MacOS | Windows](https://img.shields.io/badge/runs%20on-Linux%20%7C%20MacOS%20%7C%20Windows-blue)
+![Arch x86-63 | ARM | AppleSilicon](https://img.shields.io/badge/arch-x86--64%20%7C%20ARM%20%7C%20AppleSilicon-blue)  
+![PyPI - License](https://img.shields.io/pypi/l/par_scrape)
+
+## About
+PAR Scrape is a versatile web scraping tool with options for Selenium or Playwright, featuring OpenAI-powered data extraction and formatting.
+
+[!["Buy Me A Coffee"](https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png)](https://buymeacoffee.com/probello3)
+
+## Screenshots
+![PAR Scrape Screenshot](https://raw.githubusercontent.com/paulrobello/par_scrape/main/Screenshot.png)
+
+## Features
+
+- Web scraping using Selenium or Playwright
+- OpenAI-powered data extraction and formatting
+- Supports multiple output formats (JSON, Excel, CSV, Markdown)
+- Customizable field extraction
+- Token usage and cost estimation
+
+## Installation
+
+To install PAR Scrape, make sure you have Python 3.11 or higher and [uv](https://pypi.org/project/uv/) installed.
+
+### Installation From Source
+
+Then, follow these steps:
+
+1. Clone the repository:
+   ```bash
+   git clone https://github.com/paulrobello/par_scrape.git
+   cd par_scrape
+   ```
+
+2. Install the package dependencies using uv:
+   ```bash
+   uv sync
+   ```
+### Installation From PyPI
+
+To install PAR Scrape from PyPI, run any of the following commands:
+
+```bash
+uv tool install par_scrape
+```
+
+```bash
+pipx install par_scrape
+```
+
+## Usage
+
+To use PAR Scrape, you can run it from the command line with various options. Here's a basic example:
+
+### Running from source
+```bash
+uv run par_scrape --url "https://openai.com/api/pricing/" --fields "Model" --fields "Pricing Input" --fields "Pricing Output" --scraper selenium --model gpt-4o-mini --display-output md
+```
+
+### Running if installed from PyPI
+```bash
+par_scrape --url "https://openai.com/api/pricing/" --fields "Title" "Number of Points" "Creator" "Time Posted" "Number of Comments" --scraper selenium --model gpt-4o-mini --display-output md
+```
+
+### Options
+
+- `--url`: The URL to scrape (default: "https://openai.com/api/pricing/")
+- `--fields`: Fields to extract from the webpage (default: ["Model", "Pricing Input", "Pricing Output"])
+- `--scraper`: Scraper to use: 'selenium' or 'playwright' (default: "selenium")
+- `--remove-output`: Remove output folder before running
+- `--headless`: Run in headless mode (for Selenium) (default: True)
+- `--model`: OpenAI model to use for processing (default: "gpt-4o-mini")
+- `--display-output`: Display output in terminal (md, csv, or json)
+- `--output-folder`: Specify the location of the output folder (default: "./output")
+- `--silent`: Run in silent mode, suppressing output
+- `--run-name`: Specify a name for this run
+- `--cleanup`: Remove output folder before exiting
+
+### Examples
+
+1. Basic usage with default options:
+```bash
+par_scrape --url "https://openai.com/api/pricing/" --fields "Model" "Pricing Input" "Pricing Output"
+```
+
+2. Using Playwright and displaying JSON output:
+```bash
+par_scrape --url "https://example.com" --fields "Title" "Description" "Price" --scraper playwright --display-output json
+```
+
+3. Specifying a custom model and output folder:
+```bash
+par_scrape --url "https://example.com" --fields "Title" "Description" "Price" --model gpt-4 --output-folder ./custom_output
+```
+
+4. Running in silent mode with a custom run name:
+```bash
+par_scrape --url "https://example.com" --fields "Title" "Description" "Price" --silent --run-name my_custom_run
+```
+
+5. Using the cleanup option to remove the output folder after scraping:
+```bash
+par_scrape --url "https://example.com" --fields "Title" "Description" "Price" --cleanup
+```
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+## Author
+
+Paul Robello - probello@gmail.com

par_scrape-0.1.0/README.md

@@ -0,0 +1,120 @@
+# PAR Scrape
+
+[![PyPI](https://img.shields.io/pypi/v/par_scrape)](https://pypi.org/project/par_scrape/)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/par_scrape.svg)](https://pypi.org/project/par_scrape/)  
+![Runs on Linux | MacOS | Windows](https://img.shields.io/badge/runs%20on-Linux%20%7C%20MacOS%20%7C%20Windows-blue)
+![Arch x86-63 | ARM | AppleSilicon](https://img.shields.io/badge/arch-x86--64%20%7C%20ARM%20%7C%20AppleSilicon-blue)  
+![PyPI - License](https://img.shields.io/pypi/l/par_scrape)
+
+## About
+PAR Scrape is a versatile web scraping tool with options for Selenium or Playwright, featuring OpenAI-powered data extraction and formatting.
+
+[!["Buy Me A Coffee"](https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png)](https://buymeacoffee.com/probello3)
+
+## Screenshots
+![PAR Scrape Screenshot](https://raw.githubusercontent.com/paulrobello/par_scrape/main/Screenshot.png)
+
+## Features
+
+- Web scraping using Selenium or Playwright
+- OpenAI-powered data extraction and formatting
+- Supports multiple output formats (JSON, Excel, CSV, Markdown)
+- Customizable field extraction
+- Token usage and cost estimation
+
+## Installation
+
+To install PAR Scrape, make sure you have Python 3.11 or higher and [uv](https://pypi.org/project/uv/) installed.
+
+### Installation From Source
+
+Then, follow these steps:
+
+1. Clone the repository:
+   ```bash
+   git clone https://github.com/paulrobello/par_scrape.git
+   cd par_scrape
+   ```
+
+2. Install the package dependencies using uv:
+   ```bash
+   uv sync
+   ```
+### Installation From PyPI
+
+To install PAR Scrape from PyPI, run any of the following commands:
+
+```bash
+uv tool install par_scrape
+```
+
+```bash
+pipx install par_scrape
+```
+
+## Usage
+
+To use PAR Scrape, you can run it from the command line with various options. Here's a basic example:
+
+### Running from source
+```bash
+uv run par_scrape --url "https://openai.com/api/pricing/" --fields "Model" --fields "Pricing Input" --fields "Pricing Output" --scraper selenium --model gpt-4o-mini --display-output md
+```
+
+### Running if installed from PyPI
+```bash
+par_scrape --url "https://openai.com/api/pricing/" --fields "Title" "Number of Points" "Creator" "Time Posted" "Number of Comments" --scraper selenium --model gpt-4o-mini --display-output md
+```
+
+### Options
+
+- `--url`: The URL to scrape (default: "https://openai.com/api/pricing/")
+- `--fields`: Fields to extract from the webpage (default: ["Model", "Pricing Input", "Pricing Output"])
+- `--scraper`: Scraper to use: 'selenium' or 'playwright' (default: "selenium")
+- `--remove-output`: Remove output folder before running
+- `--headless`: Run in headless mode (for Selenium) (default: True)
+- `--model`: OpenAI model to use for processing (default: "gpt-4o-mini")
+- `--display-output`: Display output in terminal (md, csv, or json)
+- `--output-folder`: Specify the location of the output folder (default: "./output")
+- `--silent`: Run in silent mode, suppressing output
+- `--run-name`: Specify a name for this run
+- `--cleanup`: Remove output folder before exiting
+
+### Examples
+
+1. Basic usage with default options:
+```bash
+par_scrape --url "https://openai.com/api/pricing/" --fields "Model" "Pricing Input" "Pricing Output"
+```
+
+2. Using Playwright and displaying JSON output:
+```bash
+par_scrape --url "https://example.com" --fields "Title" "Description" "Price" --scraper playwright --display-output json
+```
+
+3. Specifying a custom model and output folder:
+```bash
+par_scrape --url "https://example.com" --fields "Title" "Description" "Price" --model gpt-4 --output-folder ./custom_output
+```
+
+4. Running in silent mode with a custom run name:
+```bash
+par_scrape --url "https://example.com" --fields "Title" "Description" "Price" --silent --run-name my_custom_run
+```
+
+5. Using the cleanup option to remove the output folder after scraping:
+```bash
+par_scrape --url "https://example.com" --fields "Title" "Description" "Price" --cleanup
+```
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+## Author
+
+Paul Robello - probello@gmail.com

par_scrape-0.1.0/pyproject.toml

@@ -0,0 +1,101 @@
+[project]
+name = "par_scrape"
+dynamic = ["version"]
+description = "A versatile web scraping tool with options for Selenium or Playwright, featuring OpenAI-powered data extraction and formatting."
+url = "https://github.com/paulrobello/par_scrape"
+readme = "README.md"
+requires-python = ">=3.11"
+license = { file = "LICENSE" }
+authors = [{ name = "Paul Robello", email = "probello@gmail.com" }]
+maintainers = [{ name = "Paul Robello", email = "probello@gmail.com" }]
+classifiers = [
+    "License :: OSI Approved :: MIT License",
+    "Environment :: Console",
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Internet :: WWW/HTTP :: Browsers",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Text Processing :: Markup :: HTML",
+    "Typing :: Typed"
+]
+keywords = ["web scraping", "selenium", "playwright", "openai", "data extraction"]
+dependencies = [
+    "aiofiles>=24.1.0",
+    "beautifulsoup4>=4.12.3",
+    "html2text>=2024.2.26",
+    "openai>=1.43.0",
+    "openpyxl>=3.1.5",
+    "pandas>=2.2.2",
+    "playwright>=1.46.0",
+    "pydantic>=2.9.0",
+    "python-dotenv>=1.0.1",
+    "rich>=13.8.0",
+    "selenium>=4.24.0",
+    "tabulate>=0.9.0",
+    "tiktoken>=0.7.0",
+    "typer>=0.12.5",
+    "typing-extensions>=4.12.2",
+    "webdriver-manager>=4.0.2",
+]
+
+packages = [
+    "src/par_scrape"
+]
+
+[project.urls]
+Homepage = "https://github.com/paulrobello/par_scrape"
+Documentation = "https://github.com/paulrobello/par_scrape/blob/main/README.md"
+Repository = "https://github.com/paulrobello/par_scrape"
+Issues = "https://github.com/paulrobello/par_scrape/issues"
+Discussions = "https://github.com/paulrobello/par_scrape/discussions"
+Wiki = "https://github.com/paulrobello/par_scrape/wiki"
+
+[project.scripts]
+par_scrape = "par_scrape.__main__:app"
+
+[build-system]
+requires = ["hatchling", "wheel"]
+build-backend = "hatchling.build"
+
+[tool.uv]
+dev-dependencies = [
+    "build>=1.2.1",
+    "twine>=5.1.1",
+    "black>=24.8.0",
+    "pylint>=3.2.7",
+    "pyright>=1.1.379",
+    "pre-commit>=3.8.0",
+]
+
+[tool.hatch.version]
+path = "src/par_scrape/__init__.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/par_scrape"]
+include = [
+    "*.py",
+    "py.typed",
+    "*.png",
+    "*.md",
+    "*.tcss",
+    "*.png",
+    "*.md",
+    "*.tcss"
+]
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "src/par_scrape",
+    "LICENSE",
+    "README.md",
+    "pyproject.toml"
+]
+exclude = [
+    "*.pyc",
+    "__pycache__",
+    "*.so",
+    "*.dylib"
+]

par_scrape-0.1.0/src/par_scrape/__init__.py

@@ -0,0 +1,29 @@
+"""PAR Scrape - A versatile web scraping tool."""
+
+from __future__ import annotations
+
+import os
+
+__author__ = "Paul Robello"
+__copyright__ = "Copyright 2024, Paul Robello"
+__credits__ = ["Paul Robello"]
+__maintainer__ = "Paul Robello"
+__email__ = "probello@gmail.com"
+__version__ = "0.1.0"
+__licence__ = "MIT"
+__application_title__ = "PAR Scrape"
+__application_binary__ = "par_scrape"
+
+os.environ["USER_AGENT"] = f"{__application_title__} {__version__}"
+
+__all__: list[str] = [
+    "__author__",
+    "__copyright__",
+    "__credits__",
+    "__maintainer__",
+    "__email__",
+    "__version__",
+    "__licence__",
+    "__application_title__",
+    "__application_binary__",
+]

par_scrape-0.1.0/src/par_scrape/__main__.py

@@ -0,0 +1,306 @@
+"""Main entry point for par_scrape."""
+
+import csv
+import json
+import os
+import shutil
+import asyncio
+from datetime import datetime
+from io import StringIO
+from pathlib import Path
+from typing import List, Optional, Annotated
+from enum import Enum
+from uuid import uuid4
+from contextlib import nullcontext
+from asyncio import run as aiorun
+import aiofiles
+import aiofiles.os as aos
+
+import typer
+from dotenv import load_dotenv
+from rich.markdown import Markdown
+from rich.panel import Panel
+from rich.syntax import Syntax
+from rich.table import Table
+from rich.text import Text
+
+from par_scrape.scrape_data import (
+    save_raw_data,
+    create_dynamic_listing_model,
+    create_listings_container_model,
+    format_data,
+    save_formatted_data,
+)
+from par_scrape.fetch_html import (
+    fetch_html_selenium,
+    fetch_html_playwright,
+    html_to_markdown_with_readability,
+)
+from par_scrape.pricing import calculate_price
+from par_scrape.utils import console
+from par_scrape import __version__, __application_title__
+
+# Load the .env file from the project folder
+load_dotenv(dotenv_path=".env")
+# Load the .env file from the users home folder
+load_dotenv(dotenv_path=Path("~/.par-scrape.env").expanduser())
+
+# Initialize Typer app
+app = typer.Typer(help="Web scraping tool with options for Selenium or Playwright")
+
+
+class DisplayOutputFormat(str, Enum):
+    """Enum for display output format choices."""
+
+    MD = "md"
+    CSV = "csv"
+    JSON = "json"
+
+
+class ScraperChoice(str, Enum):
+    """Enum for scraper choices."""
+
+    SELENIUM = "selenium"
+    PLAYWRIGHT = "playwright"
+
+
+def version_callback(value: bool) -> None:
+    """Print version and exit."""
+    if value:
+        print(f"{__application_title__}: {__version__}")
+        raise typer.Exit()
+
+
+# pylint: disable=too-many-statements,dangerous-default-value
+@app.command()
+def main(
+    url: Annotated[
+        str, typer.Option("--url", "-u", help="URL to scrape", prompt=True)
+    ] = "https://openai.com/api/pricing/",
+    fields: Annotated[
+        List[str],
+        typer.Option(
+            "--fields", "-f", help="Fields to extract from the webpage", prompt=True
+        ),
+    ] = [
+        "Model",
+        "Pricing Input",
+        "Pricing Output",
+    ],
+    scraper: Annotated[
+        ScraperChoice,
+        typer.Option(
+            "--scraper",
+            "-s",
+            help="Scraper to use: 'selenium' or 'playwright'",
+            case_sensitive=False,
+        ),
+    ] = ScraperChoice.SELENIUM,
+    remove_output: Annotated[
+        bool,
+        typer.Option(
+            "--remove-output", "-r", help="Remove output folder before running"
+        ),
+    ] = False,
+    headless: Annotated[
+        bool,
+        typer.Option("--headless", "-h", help="Run in headless mode (for Selenium)"),
+    ] = True,
+    model: Annotated[
+        str, typer.Option("--model", "-m", help="OpenAI model to use for processing")
+    ] = "gpt-4o-mini",
+    display_output: Annotated[
+        Optional[DisplayOutputFormat],
+        typer.Option(
+            "--display-output",
+            "-d",
+            help="Display output in terminal (md, csv, or json)",
+        ),
+    ] = None,
+    output_folder: Annotated[
+        Path,
+        typer.Option(
+            "--output-folder", "-o", help="Specify the location of the output folder"
+        ),
+    ] = "./output",
+    silent: Annotated[
+        bool,
+        typer.Option("--silent", "-q", help="Run in silent mode, suppressing output"),
+    ] = False,
+    run_name: Annotated[
+        str,
+        typer.Option("--run-name", "-n", help="Specify a name for this run"),
+    ] = "",
+    version: Annotated[  # pylint: disable=unused-argument
+        Optional[bool],
+        typer.Option("--version", "-v", callback=version_callback, is_eager=True),
+    ] = None,
+    cleanup: Annotated[
+        bool,
+        typer.Option("--cleanup", "-c", help="Remove output folder before exiting"),
+    ] = False,
+):
+    """Scrape and analyze data from a website."""
+
+    async def _main():  # pylint: disable=too-many-locals, too-many-branches, too-many-statements
+        """Scrape and analyze data from a website."""
+        nonlocal run_name
+        if not os.environ.get("OPENAI_API_KEY"):
+            console.print(
+                "[bold red]OPENAI_API_KEY environment variable not set. Exiting...[/bold red]"
+            )
+            raise typer.Exit(1)
+        with console.capture() if silent else nullcontext():
+            if remove_output:
+                if await aos.path.exists(output_folder):
+                    shutil.rmtree(output_folder)
+                    console.print(
+                        f"[bold green]Removed existing output folder: {output_folder}[/bold green]"
+                    )
+            try:
+                # Generate run_name if not provided
+                if not run_name:
+                    run_name = datetime.now().strftime("%Y%m%d_%H%M%S")
+                else:
+                    # Ensure run_name is filesystem-friendly
+                    run_name = "".join(
+                        c for c in run_name if c.isalnum() or c in ("-", "_")
+                    )
+                    if not run_name:
+                        run_name = str(uuid4())
+
+                # Display summary of options
+                console.print(
+                    Panel.fit(
+                        Text.assemble(
+                            ("URL: ", "cyan"),
+                            (f"{url}", "green"),
+                            "\n",
+                            ("Model: ", "cyan"),
+                            (f"{model}", "green"),
+                            "\n",
+                            ("Scraper: ", "cyan"),
+                            (f"{scraper}", "green"),
+                            "\n",
+                            ("Headless: ", "cyan"),
+                            (f"{headless}", "green"),
+                            "\n",
+                            ("Fields to extract: ", "cyan"),
+                            (", ".join(fields), "green"),
+                            "\n",
+                            ("Display output: ", "cyan"),
+                            (f"{display_output or 'None'}", "green"),
+                            "\n",
+                            ("Silent mode: ", "cyan"),
+                            (f"{silent}", "green"),
+                        ),
+                        title="[bold]Scraping Configuration",
+                        border_style="bold",
+                    )
+                )
+
+                with console.status(
+                    "[bold green]Working on data extraction and processing..."
+                ) as status:
+                    # Scrape data
+                    status.update("[bold cyan]Fetching HTML...")
+                    if scraper == ScraperChoice.PLAYWRIGHT:
+                        raw_html = await fetch_html_playwright(url)
+                    else:
+                        raw_html = await fetch_html_selenium(url, headless)
+
+                    status.update("[bold cyan]Converting HTML to Markdown...")
+                    markdown = await html_to_markdown_with_readability(raw_html)
+
+                    # Save raw data
+                    status.update("[bold cyan]Saving raw data...")
+                    await save_raw_data(markdown, run_name, output_folder)
+
+                    # Create the dynamic listing model
+                    status.update("[bold cyan]Creating dynamic models...")
+                    dynamic_listing_model = create_dynamic_listing_model(fields)
+                    dynamic_listings_container = create_listings_container_model(
+                        dynamic_listing_model
+                    )
+
+                    # Format data
+                    status.update("[bold cyan]Formatting data...")
+                    formatted_data = await format_data(
+                        markdown, dynamic_listings_container, model
+                    )
+
+                    # Save formatted data
+                    status.update("[bold cyan]Saving formatted data...")
+                    _, file_paths = await save_formatted_data(
+                        formatted_data, run_name, output_folder
+                    )
+
+                    # Convert formatted_data back to text for token counting
+                    formatted_data_text = json.dumps(formatted_data.model_dump())
+
+                # Display output if requested
+                if display_output:
+                    if display_output.value in file_paths:
+                        async with aiofiles.open(
+                            file_paths[display_output.value], "rt", encoding="utf-8"
+                        ) as f:
+                            content = await f.read()
+                        if display_output == DisplayOutputFormat.MD:
+                            console.print(Markdown(content))
+                        elif display_output == DisplayOutputFormat.CSV:
+                            # Convert CSV to rich Table
+                            table = Table(title="CSV Data")
+                            csv_reader = csv.reader(StringIO(content))
+                            headers = next(csv_reader)
+                            for header in headers:
+                                table.add_column(header, style="cyan")
+                            for row in csv_reader:
+                                table.add_row(*row)
+                            console.print(table)
+                        elif display_output == DisplayOutputFormat.JSON:
+                            console.print(Syntax(content, "json"))
+                    else:
+                        console.print(
+                            f"[bold red]Invalid output type: {display_output.value}[/bold red]"
+                        )
+
+                # Automatically calculate the token usage and cost for all input and output
+                status.update("[bold cyan]Calculating token usage and cost...")
+                input_tokens, output_tokens, total_cost = await calculate_price(
+                    markdown, formatted_data_text, model=model
+                )
+
+                console.print(
+                    Panel.fit(
+                        Text.assemble(
+                            ("Input token count: ", "cyan"),
+                            (f"{input_tokens}", "green"),
+                            "\n",
+                            ("Output token count: ", "cyan"),
+                            (f"{output_tokens}", "green"),
+                            "\n",
+                            ("Estimated total cost: ", "cyan"),
+                            (f"${total_cost:.4f}", "green bold"),
+                        ),
+                        title="[bold]Summary",
+                        border_style="bold",
+                    )
+                )
+
+            except Exception as e:  # pylint: disable=broad-except
+                console.print(f"[bold red]An error occurred:[/bold red] {str(e)}")
+
+            finally:
+                if cleanup:
+                    with console.status("[bold yellow]Cleaning up..."):
+                        if await aos.path.exists(output_folder):
+                            await asyncio.to_thread(shutil.rmtree, output_folder)
+                            console.print(
+                                f"[bold green]Removed output folder and its contents: {output_folder}[/bold green]"
+                            )
+
+    aiorun(_main())
+
+
+if __name__ == "__main__":
+    app()

par_scrape-0.1.0/src/par_scrape/fetch_html.py

@@ -0,0 +1,197 @@
+"""Fetch HTML from URL and save it to a file."""
+
+import json
+import logging
+import os
+import asyncio
+import random
+from pathlib import Path
+
+import aiofiles
+
+import html2text
+from bs4 import BeautifulSoup
+from playwright.async_api import async_playwright
+from selenium import webdriver
+from selenium.webdriver.chrome.webdriver import WebDriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.chrome.service import Service
+from webdriver_manager.chrome import ChromeDriverManager
+
+from .utils import console
+
+
+async def setup_selenium(headless: bool = True) -> WebDriver:
+    """Set up ChromeDriver for Selenium."""
+
+    logger = logging.getLogger("selenium")
+    logger.setLevel(logging.DEBUG)
+
+    options = Options()
+
+    # adding arguments
+    options.add_argument("--disable-gpu")
+    options.add_argument("--disable-dev-shm-usage")
+    options.add_argument("--window-size=1920,1080")
+    options.add_experimental_option(
+        "excludeSwitches", ["enable-logging"]
+    )  # Disable logging
+    options.add_argument("--log-level=3")  # Suppress console logging
+    options.add_argument("--silent")
+    options.add_argument("--disable-extensions")
+
+    # Enable headless mode if specified
+    if headless:
+        options.add_argument("--headless")
+
+    # Randomize user-agent to mimic different users
+    options.add_argument(
+        "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"  # pylint: disable=line-too-long
+    )
+
+    # Load ChromeDriver path from config.json
+    config_path = Path("config.json")
+    if config_path.exists():
+        async with aiofiles.open(config_path, "rt", encoding="utf-8") as config_file:
+            config = json.loads(await config_file.read())
+            chromedriver_path = config.get("chromedriver_path", "")
+    else:
+        chromedriver_path = ""
+
+    # Check if ChromeDriver exists, if not, download it
+    if not chromedriver_path or not os.path.exists(chromedriver_path):
+        console.print(
+            "[yellow]ChromeDriver not found or path invalid. Attempting to download...[/yellow]"
+        )
+        try:
+            chromedriver_path = await asyncio.to_thread(ChromeDriverManager().install)
+            console.print(
+                f"[green]ChromeDriver downloaded successfully to: {chromedriver_path}[/green]"
+            )
+
+            # Save the new path to config.json
+            async with aiofiles.open(
+                config_path, "wt", encoding="utf-8"
+            ) as config_file:
+                await config_file.write(
+                    json.dumps({"chromedriver_path": chromedriver_path}, indent=4)
+                )
+            console.print("[green]Updated ChromeDriver path in config.json[/green]")
+        except Exception as e:
+            console.print(
+                f"[bold red]Error downloading ChromeDriver:[/bold red] {str(e)}"
+            )
+            raise
+
+    service = Service(chromedriver_path, log_output=os.devnull)
+
+    try:
+        # Initialize the WebDriver
+        driver = await asyncio.to_thread(
+            webdriver.Chrome, service=service, options=options
+        )
+        return driver
+    except Exception as e:
+        console.print(
+            f"[bold red]Error initializing Chrome WebDriver:[/bold red] {str(e)}"
+        )
+        console.print(
+            "[yellow]Please ensure Chrome is installed and up to date.[/yellow]"
+        )
+        console.print(
+            "[yellow]Also, make sure the ChromeDriver version matches your Chrome browser version.[/yellow]"
+        )
+        raise
+
+
+async def fetch_html_selenium(url: str, headless: bool = True) -> str:
+    """Fetch HTML content from a URL using Selenium."""
+    driver = await setup_selenium(headless)
+    try:
+        await asyncio.to_thread(driver.get, url)
+
+        # Add random delays to mimic human behavior
+        await asyncio.sleep(
+            random.uniform(3, 6)
+        )  # Adjust this to simulate time for user to read or interact
+
+        # Add more realistic actions like scrolling
+        await asyncio.to_thread(
+            driver.execute_script, "window.scrollTo(0, document.body.scrollHeight);"
+        )
+        await asyncio.sleep(
+            random.uniform(3, 5)
+        )  # Simulate time taken to scroll and read
+
+        html = await asyncio.to_thread(lambda: driver.page_source)
+        return html
+    finally:
+        await asyncio.to_thread(driver.quit)
+
+
+async def fetch_html_playwright(url: str) -> str:
+    """
+    Fetch HTML content from a URL using Playwright.
+
+    Args:
+        url (str): The URL to fetch HTML from.
+
+    Returns:
+        str: The HTML content of the page.
+    """
+    async with async_playwright() as p:
+        browser = await p.chromium.launch(headless=True)
+        page = await browser.new_page()
+        await page.goto(url)
+
+        # Add random delays to mimic human behavior
+        await asyncio.sleep(
+            3
+        )  # Adjust this to simulate time for user to read or interact
+
+        # Add more realistic actions like scrolling
+        await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+        await asyncio.sleep(3)  # Simulate time taken to scroll and read
+
+        html = await page.content()
+        await browser.close()
+        return html
+
+
+async def clean_html(html_content: str) -> str:
+    """
+    Clean HTML content by removing headers and footers.
+
+    Args:
+        html_content (str): The raw HTML content.
+
+    Returns:
+        str: The cleaned HTML content.
+    """
+    soup = BeautifulSoup(html_content, "html.parser")
+
+    # Remove headers and footers based on common HTML tags or classes
+    for element in soup.find_all(["header", "footer"]):
+        element.decompose()  # Remove these tags and their content
+
+    return str(soup)
+
+
+async def html_to_markdown_with_readability(html_content: str) -> str:
+    """
+    Convert HTML content to Markdown format.
+
+    Args:
+        html_content (str): The HTML content to convert.
+
+    Returns:
+        str: The converted markdown content.
+    """
+    cleaned_html = await clean_html(html_content)
+
+    # Convert to markdown
+    markdown_converter = html2text.HTML2Text()
+    markdown_converter.ignore_links = False
+    markdown_content = await asyncio.to_thread(markdown_converter.handle, cleaned_html)
+
+    return markdown_content

par_scrape-0.1.0/src/par_scrape/pricing.py

@@ -0,0 +1,86 @@
+"""Pricing functions for Par Scrape."""
+
+from typing import Tuple
+import asyncio
+
+import tiktoken
+
+pricing = {
+    "gpt-4o": {
+        "input": 0.000005,  # $5.00 per 1M input tokens
+        "output": 0.000015,  # $15.00 per 1M output tokens
+    },
+    "gpt-4o-2024-08-06": {
+        "input": 0.0000025,  # $2.50 per 1M input tokens
+        "output": 0.00001,  # $10.00 per 1M output tokens
+    },
+    "gpt-4o-2024-05-13": {
+        "input": 0.000005,  # $5.00 per 1M input tokens
+        "output": 0.000015,  # $15.00 per 1M output tokens
+    },
+    "gpt-4o-mini": {
+        "input": 0.00000015,  # $0.15 per 1M input tokens
+        "output": 0.0000006,  # $0.60 per 1M output tokens
+    },
+    "gpt-4o-mini-2024-07-18": {
+        "input": 0.00000015,  # $0.15 per 1M input tokens
+        "output": 0.0000006,  # $0.60 per 1M output tokens
+    },
+    "gpt-4": {
+        "input": 0.00003,  # $30.00 per 1M input tokens
+        "output": 0.00006,  # $60.00 per 1M output tokens
+    },
+    "gpt-4-32k": {
+        "input": 0.00006,  # $60.00 per 1M input tokens
+        "output": 0.00012,  # $120.00 per 1M output tokens
+    },
+    "gpt-4-turbo": {
+        "input": 0.00001,  # $10.00 per 1M input tokens
+        "output": 0.00003,  # $30.00 per 1M output tokens
+    },
+    "gpt-4-turbo-2024-04-09": {
+        "input": 0.00001,  # $10.00 per 1M input tokens
+        "output": 0.00003,  # $30.00 per 1M output tokens
+    },
+    "gpt-3.5-turbo-0125": {
+        "input": 0.0000005,  # $0.50 per 1M input tokens
+        "output": 0.0000015,  # $1.50 per 1M output tokens
+    },
+}
+
+
+async def calculate_price(
+    input_text: str, output_text: str, model: str
+) -> Tuple[int, int, float]:
+    """
+    Calculate the price for processing input and output text using a specified model.
+
+    Args:
+        input_text (str): The input text.
+        output_text (str): The output text.
+        model (str, optional): The model to use for pricing. Defaults to model_used.
+
+    Returns:
+        Tuple[int, int, float]: A tuple containing the input token count, output token count, and total cost.
+        Returns (0, 0, 0.0) if there's an error during calculation.
+    """
+    try:
+        # Initialize the encoder for the specific model
+        encoder = tiktoken.encoding_for_model(model)
+
+        # Encode the input text to get the number of input tokens
+        input_token_count = await asyncio.to_thread(len, encoder.encode(input_text))
+
+        # Encode the output text to get the number of output tokens
+        output_token_count = await asyncio.to_thread(len, encoder.encode(output_text))
+
+        # Calculate the costs
+        # Convert price per million tokens to price per token, then multiply by token count
+        input_cost = input_token_count * pricing[model]["input"]
+        output_cost = output_token_count * pricing[model]["output"]
+        total_cost = input_cost + output_cost
+
+        return input_token_count, output_token_count, total_cost
+    except Exception as e:  # pylint: disable=broad-except
+        print(f"Error calculating price: {str(e)}")
+        return 0, 0, 0.0

par_scrape-0.1.0/src/par_scrape/scrape_data.py

@@ -0,0 +1,230 @@
+"""Scrape data from Web."""
+
+import json
+import os
+from typing import List, Type, Tuple, Dict
+
+import aiofiles
+import pandas as pd
+import tiktoken
+from aiofiles import os as aos
+from openai import AsyncOpenAI, OpenAIError
+from pydantic import BaseModel, create_model, ConfigDict
+from rich.panel import Panel
+
+from par_scrape.utils import console
+
+
+async def save_raw_data(
+    raw_data: str, run_name: str, output_folder: str = "output"
+) -> str:
+    """
+    Save raw data to a file.
+
+    Args:
+        raw_data (str): The raw data to save.
+        run_name (str): The run name to use in the filename.
+        output_folder (str, optional): The folder to save the file in. Defaults to 'output'.
+
+    Returns:
+        str: The path to the saved file.
+    """
+    # Ensure the output folder exists
+    await aos.makedirs(output_folder, exist_ok=True)
+
+    # Save the raw markdown data with run_name in filename
+    raw_output_path = os.path.join(output_folder, f"rawData_{run_name}.md")
+    async with aiofiles.open(raw_output_path, "wt", encoding="utf-8") as f:
+        await f.write(raw_data)
+    console.print(
+        Panel(f"Raw data saved to [bold green]{raw_output_path}[/bold green]")
+    )
+    return raw_output_path
+
+
+def create_dynamic_listing_model(field_names: List[str]) -> Type[BaseModel]:
+    """
+    Dynamically creates a Pydantic model based on provided fields.
+
+    Args:
+        field_names (List[str]): A list of names of the fields to extract from the markdown.
+
+    Returns:
+        Type[BaseModel]: A dynamically created Pydantic model.
+    """
+    # Create field definitions using aliases for Field parameters
+    field_definitions = {field: (str, ...) for field in field_names}
+    # Dynamically create the model with all fields
+    dynamic_listing_model = create_model(
+        "DynamicListingModel",
+        **field_definitions,
+    )
+    dynamic_listing_model.model_config = ConfigDict(arbitrary_types_allowed=True)
+    return dynamic_listing_model
+
+
+def create_listings_container_model(listing_model: Type[BaseModel]) -> Type[BaseModel]:
+    """
+    Create a container model that holds a list of the given listing model.
+
+    Args:
+        listing_model (Type[BaseModel]): The Pydantic model for individual listings.
+
+    Returns:
+        Type[BaseModel]: A container model for a list of listings.
+    """
+    return create_model("DynamicListingsContainer", listings=(List[listing_model], ...))
+
+
+def trim_to_token_limit(text: str, model: str, max_tokens: int = 200000) -> str:
+    """
+    Trim text to a specified token limit for a given model.
+
+    Args:
+        text (str): The input text to trim.
+        model (str): The model name to use for tokenization.
+        max_tokens (int, optional): The maximum number of tokens. Defaults to 200000.
+
+    Returns:
+        str: The trimmed text.
+    """
+    encoder = tiktoken.encoding_for_model(model)
+    tokens = encoder.encode(text)
+    if len(tokens) > max_tokens:
+        trimmed_text = encoder.decode(tokens[:max_tokens])
+        return trimmed_text
+    return text
+
+
+async def format_data(
+    data: str, dynamic_listings_container: Type[BaseModel], model: str
+) -> BaseModel:
+    """
+    Format data using OpenAI's API asynchronously.
+
+    Args:
+        data (str): The input data to format.
+        dynamic_listings_container (Type[BaseModel]): The Pydantic model to use for parsing.
+        model (str): The OpenAI model to use for processing.
+
+    Returns:
+        BaseModel: The formatted data as a Pydantic model instance.
+    """
+    client = AsyncOpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+
+    system_message = """
+You are an intelligent text extraction and conversion assistant. Your task is to extract structured information
+from the given text and convert it into a pure JSON format. The JSON should contain only the structured data extracted from the text,
+with no additional commentary, explanations, or extraneous information.
+You could encounter cases where you can't find the data of the fields you have to extract or the data will be in a foreign language.
+Please process the following text and provide the output in pure JSON format with no words before or after the JSON:"""
+
+    user_message = f"Extract the following information from the provided text:\nPage content:\n\n{data}"
+
+    try:
+        completion = await client.beta.chat.completions.parse(
+            model=model,
+            messages=[
+                {"role": "system", "content": system_message},
+                {"role": "user", "content": user_message},
+            ],
+            response_format=dynamic_listings_container,
+        )
+        return completion.choices[0].message.parsed
+    except (OpenAIError, json.JSONDecodeError) as e:
+        console.print(
+            f"[bold red]Error in API call or parsing response:[/bold red] {str(e)}"
+        )
+        return dynamic_listings_container(listings=[])
+
+
+async def save_formatted_data(
+    formatted_data: BaseModel, run_name: str, output_folder: str = "output"
+) -> Tuple[pd.DataFrame | None, Dict[str, str]]:
+    """
+    Save formatted data to JSON, Excel, CSV, and Markdown files.
+
+    Args:
+        formatted_data (BaseModel): The formatted data to save.
+        run_name (str): The run name to use in the filenames.
+        output_folder (str, optional): The folder to save the files in. Defaults to 'output'.
+
+    Returns:
+        Tuple[pd.DataFrame | None, Dict[str, str]]: The DataFrame created from the formatted data and a dictionary of
+        file paths, or None and an empty dict if an error occurred.
+    """
+    file_paths: Dict[str, str] = {}
+    # Ensure the output folder exists
+    await aos.makedirs(output_folder, exist_ok=True)
+
+    # Prepare formatted data as a dictionary
+    formatted_data_dict = formatted_data.model_dump()
+
+    # Save the formatted data as JSON with run_name in filename
+    json_output_path = os.path.join(output_folder, f"sorted_data_{run_name}.json")
+    with open(json_output_path, "wt", encoding="utf-8") as f:
+        json.dump(formatted_data_dict, f, indent=4)
+    console.print(
+        Panel(
+            f"Formatted data saved to JSON at [bold green]{json_output_path}[/bold green]"
+        )
+    )
+    file_paths["json"] = json_output_path
+
+    # Prepare data for DataFrame
+    if isinstance(formatted_data_dict, dict):
+        # If the data is a dictionary containing lists, assume these lists are records
+        data_for_df = (
+            next(iter(formatted_data_dict.values()))
+            if len(formatted_data_dict) == 1
+            else formatted_data_dict
+        )
+    elif isinstance(formatted_data_dict, list):
+        data_for_df = formatted_data_dict
+    else:
+        raise ValueError(
+            "Formatted data is neither a dictionary nor a list, cannot convert to DataFrame"
+        )
+
+    # Create DataFrame
+    try:
+        df = pd.DataFrame(data_for_df)
+        console.print(Panel("[bold green]DataFrame created successfully.[/bold green]"))
+
+        # Save the DataFrame to an Excel file
+        excel_output_path = os.path.join(output_folder, f"sorted_data_{run_name}.xlsx")
+        df.to_excel(excel_output_path, index=False)
+        console.print(
+            Panel(
+                f"Formatted data saved to Excel at [bold green]{excel_output_path}[/bold green]"
+            )
+        )
+        file_paths["excel"] = excel_output_path
+
+        # Save the DataFrame to a CSV file
+        csv_output_path = os.path.join(output_folder, f"sorted_data_{run_name}.csv")
+        df.to_csv(csv_output_path, index=False)
+        console.print(
+            Panel(
+                f"Formatted data saved to CSV at [bold green]{csv_output_path}[/bold green]"
+            )
+        )
+        file_paths["csv"] = csv_output_path
+
+        # Save the DataFrame as a Markdown table
+        markdown_output_path = os.path.join(output_folder, f"sorted_data_{run_name}.md")
+        with open(markdown_output_path, "wt", encoding="utf-8") as f:
+            f.write(df.to_markdown(index=False) or "")
+        console.print(
+            Panel(
+                f"Formatted data saved as Markdown table at [bold green]{markdown_output_path}[/bold green]"
+            )
+        )
+        file_paths["md"] = markdown_output_path
+
+        return df, file_paths
+    except Exception as e:  # pylint: disable=broad-except
+        console.print(
+            f"[bold red]Error creating DataFrame or saving files:[/bold red] {str(e)}"
+        )
+        return None, {}

par_scrape-0.1.0/src/par_scrape/utils.py

@@ -0,0 +1,6 @@
+"""Utility functions for par_scrape."""
+
+from rich.console import Console
+
+# Initialize rich console
+console = Console()