pywebscrapr 0.1.0__tar.gz

pywebscrapr-0.1.0/LICENSE

@@ -0,0 +1,7 @@
+MIT License (Modified)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to make derivative works based on the Software, provided that any substantial changes to the Software are clearly distinguished from the original work and are distributed under a different name.
+
+The original copyright notice and disclaimer must be retained in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS," WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT, OR OTHERWISE, ARISING FROM, OUT OF, OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

pywebscrapr-0.1.0/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 2.1
+Name: pywebscrapr
+Version: 0.1.0
+Summary: An open-source Python library for web scraping tasks.
+Home-page: https://github.com/infinitode/pywebscrapr
+Author: Infinitode Pty Ltd
+Author-email: infinitode.ltd@gmail.com
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests
+Requires-Dist: beautifulsoup4
+
+An open-source Python library for web scraping tasks. Includes support for both image scraping and text scraping.

pywebscrapr-0.1.0/README.md

@@ -0,0 +1,76 @@
+# PyWebScrapr
+![Python Version](https://img.shields.io/badge/python-3.12-blue.svg)
+![Code Size](https://img.shields.io/github/languages/code-size/Infinitode/PyWebScrapr)
+![Downloads](https://pepy.tech/badge/pywebscapr)
+![License Compliance](https://img.shields.io/badge/license-compliance-brightgreen.svg)
+![PyPI Version](https://img.shields.io/pypi/v/pywebscrapr)
+
+An open-source Python library for web scraping tasks. Includes support for both text and image scraping.
+
+## Installation
+
+You can install PyWebScrapr using pip:
+
+```bash
+pip install pywebscrapr
+```
+
+## Supported Python Versions
+
+PyWebScrapr supports the following Python versions:
+
+- Python 3.6
+- Python 3.7
+- Python 3.8
+- Python 3.9
+- Python 3.10
+- Python 3.11/Later (Preferred)
+
+Please ensure that you have one of these Python versions installed before using PyWebScrapr. PyWebScrapr may not work as expected on lower versions of Python than the supported.
+
+## Features
+
+- **Text Scraping**: Extract textual content from specified URLs.
+- **Image Scraping**: Download images from specified URLs.
+
+<sub>*for a full list check out the PyWebScrapr Documentation.</sub>
+
+## Usage
+
+### Text Scraping
+
+```python
+from pywebscrapr import scrape_text
+
+# Specify links in a file or list
+links_file = 'links.txt'
+links_array = ['https://example.com/page1', 'https://example.com/page2']
+
+# Scrape text and save to the 'output.txt' file
+scrape_text(links_file=links_file, links_array=links_array, output_file='output.txt')
+```
+
+### Image Scraping
+
+```python
+from pywebscrapr import scrape_images
+
+# Specify links in a file or list
+links_file = 'image_links.txt'
+links_array = ['https://example.com/image1.jpg', 'https://example.com/image2.png']
+
+# Scrape images and save to the 'images' folder
+scrape_images(links_file=links_file, links_array=links_array, save_folder='images')
+```
+
+## Contributing
+
+Contributions are welcome! If you encounter any issues, have suggestions, or want to contribute to PyWebScrapr, please open an issue or submit a pull request on [GitHub](https://github.com/infinitode/pywebscrapr).
+
+## License
+
+PyWebScrapr is released under the terms of the **MIT License (Modified)**. Please see the [LICENSE](https://github.com/infinitode/pywebscrapr/blob/main/LICENSE) file for the full text.
+
+**Modified License Clause**
+
+The modified license clause grants users the permission to make derivative works based on the PyWebScrapr software. However, it requires any substantial changes to the software to be clearly distinguished from the original work and distributed under a different name.

pywebscrapr-0.1.0/pywebscrapr/__init__.py

@@ -0,0 +1,122 @@
+import os
+import csv
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urlparse, urljoin
+
+def _get_absolute_url(base_url, relative_url):
+    """Helper function to get absolute URLs."""
+    return urljoin(base_url, relative_url)
+
+def _download_image(url, save_path):
+    """Helper function to download an image."""
+    response = requests.get(url)
+    with open(save_path, 'wb') as file:
+        file.write(response.content)
+
+def scrape_images(links_file=None, links_array=None, save_folder='images'):
+    """
+    Scrape image content from the given links and save to specified output folder.
+
+    Parameters:
+    - links_file (str): Path to a file containing links, with each link on a new line.
+    - links_array (list): List of links to scrape text from.
+    - output_folder (str): Folder to save the scraped images.
+
+    Example:
+    ```python
+    from pywebscrapr import scrape_images
+
+    # Using links from a file and saving images to output_images folder.
+    scrape_text(links_file='links.txt', output_folder='output_images')
+    ```
+    """
+    if not os.path.exists(save_folder):
+        os.makedirs(save_folder)
+
+    links = []
+    if links_file:
+        with open(links_file, 'r') as file:
+            links = file.read().splitlines()
+    elif links_array:
+        links = links_array
+    else:
+        raise ValueError("Either 'links_file' or 'links_array' must be provided.")
+
+    for link in links:
+        response = requests.get(link)
+        soup = BeautifulSoup(response.text, 'html.parser')
+
+        for img_tag in soup.find_all('img'):
+            img_url = _get_absolute_url(link, img_tag.get('src'))
+
+            # Skip data URLs
+            if not img_url.startswith('data:'):
+                img_name = os.path.basename(urlparse(img_url).path)
+                save_path = os.path.join(save_folder, img_name)
+                _download_image(img_url, save_path)
+
+def scrape_text(links_file=None, links_array=None, output_file='output.txt', csv_output_file=None, remove_extra_whitespace=True):
+    """
+    Scrape textual content from the given links and save to specified output file(s).
+
+    Parameters:
+    - links_file (str): Path to a file containing links, with each link on a new line.
+    - links_array (list): List of links to scrape text from.
+    - output_file (str): File to save the scraped text.
+    - csv_output_file (str): File to save the URL and text information in CSV format.
+    - remove_extra_whitespace (bool): If True, remove extra whitespace and empty lines from the output.
+
+    Example:
+    ```python
+    from pywebscrapr import scrape_text
+
+    # Using links from a file and saving text to output.txt
+    scrape_text(links_file='links.txt', output_file='output.txt')
+
+    # Using links directly and saving text to output.txt and csv_output.csv with extra whitespace removal
+    links = ['https://example.com/page1', 'https://example.com/page2']
+    scrape_text(links_array=links, output_file='output.txt', csv_output_file='csv_output.csv', remove_extra_whitespace=True)
+    ```
+    """
+    links = []
+    if links_file:
+        with open(links_file, 'r') as file:
+            links = file.read().splitlines()
+    elif links_array:
+        links = links_array
+    else:
+        raise ValueError("Either 'links_file' or 'links_array' must be provided.")
+
+    all_text = ""
+    csv_data = []
+
+    for link in links:
+        response = requests.get(link)
+        soup = BeautifulSoup(response.text, 'html.parser')
+
+        for element in soup.find_all(lambda tag: tag.name not in ['script', 'style']):
+            if remove_extra_whitespace:
+                text = element.get_text(strip=True)  # Remove extra whitespace
+                if text:  # Skip empty lines
+                    all_text += text + "\n"
+            else:
+                text = element.get_text()
+                all_text += text + "\n"
+            if csv_output_file:
+                csv_data.append({'URL': link, 'Text': text})
+
+    # Save text to output file
+    with open(output_file, 'w', encoding='utf-8') as text_file:
+        if remove_extra_whitespace:
+            text_file.write(all_text.rstrip())  # Remove trailing whitespace
+        else:
+            text_file.write(all_text)
+
+    # Save CSV data to CSV file
+    if csv_output_file:
+        with open(csv_output_file, 'w', newline='', encoding='utf-8') as csv_file:
+            fieldnames = ['URL', 'Text']
+            writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
+            writer.writeheader()
+            writer.writerows(csv_data)

pywebscrapr-0.1.0/pywebscrapr.egg-info/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 2.1
+Name: pywebscrapr
+Version: 0.1.0
+Summary: An open-source Python library for web scraping tasks.
+Home-page: https://github.com/infinitode/pywebscrapr
+Author: Infinitode Pty Ltd
+Author-email: infinitode.ltd@gmail.com
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests
+Requires-Dist: beautifulsoup4
+
+An open-source Python library for web scraping tasks. Includes support for both image scraping and text scraping.

pywebscrapr-0.1.0/pywebscrapr.egg-info/SOURCES.txt

@@ -0,0 +1,9 @@
+LICENSE
+README.md
+setup.py
+pywebscrapr/__init__.py
+pywebscrapr.egg-info/PKG-INFO
+pywebscrapr.egg-info/SOURCES.txt
+pywebscrapr.egg-info/dependency_links.txt
+pywebscrapr.egg-info/requires.txt
+pywebscrapr.egg-info/top_level.txt

pywebscrapr-0.1.0/pywebscrapr.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

pywebscrapr-0.1.0/pywebscrapr.egg-info/requires.txt

@@ -0,0 +1,2 @@
+requests
+beautifulsoup4

pywebscrapr-0.1.0/pywebscrapr.egg-info/top_level.txt

@@ -0,0 +1 @@
+pywebscrapr

pywebscrapr-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

pywebscrapr-0.1.0/setup.py

@@ -0,0 +1,30 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='pywebscrapr',
+    version='0.1.0',
+    author='Infinitode Pty Ltd',
+    author_email='infinitode.ltd@gmail.com',
+    description='An open-source Python library for web scraping tasks.',
+    long_description='An open-source Python library for web scraping tasks. Includes support for both image scraping and text scraping.',
+    long_description_content_type='text/markdown',
+    url='https://github.com/infinitode/pywebscrapr',
+    packages=find_packages(),
+    install_requires=[
+        'requests',
+        'beautifulsoup4',
+    ],
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: MIT License',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
+    ],
+    python_requires='>=3.6',
+)