pywebscrapr 0.1.0__py3-none-any.whl

pywebscrapr/__init__.py

@@ -0,0 +1,122 @@
+import os
+import csv
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urlparse, urljoin
+
+def _get_absolute_url(base_url, relative_url):
+    """Helper function to get absolute URLs."""
+    return urljoin(base_url, relative_url)
+
+def _download_image(url, save_path):
+    """Helper function to download an image."""
+    response = requests.get(url)
+    with open(save_path, 'wb') as file:
+        file.write(response.content)
+
+def scrape_images(links_file=None, links_array=None, save_folder='images'):
+    """
+    Scrape image content from the given links and save to specified output folder.
+
+    Parameters:
+    - links_file (str): Path to a file containing links, with each link on a new line.
+    - links_array (list): List of links to scrape text from.
+    - output_folder (str): Folder to save the scraped images.
+
+    Example:
+    ```python
+    from pywebscrapr import scrape_images
+
+    # Using links from a file and saving images to output_images folder.
+    scrape_text(links_file='links.txt', output_folder='output_images')
+    ```
+    """
+    if not os.path.exists(save_folder):
+        os.makedirs(save_folder)
+
+    links = []
+    if links_file:
+        with open(links_file, 'r') as file:
+            links = file.read().splitlines()
+    elif links_array:
+        links = links_array
+    else:
+        raise ValueError("Either 'links_file' or 'links_array' must be provided.")
+
+    for link in links:
+        response = requests.get(link)
+        soup = BeautifulSoup(response.text, 'html.parser')
+
+        for img_tag in soup.find_all('img'):
+            img_url = _get_absolute_url(link, img_tag.get('src'))
+
+            # Skip data URLs
+            if not img_url.startswith('data:'):
+                img_name = os.path.basename(urlparse(img_url).path)
+                save_path = os.path.join(save_folder, img_name)
+                _download_image(img_url, save_path)
+
+def scrape_text(links_file=None, links_array=None, output_file='output.txt', csv_output_file=None, remove_extra_whitespace=True):
+    """
+    Scrape textual content from the given links and save to specified output file(s).
+
+    Parameters:
+    - links_file (str): Path to a file containing links, with each link on a new line.
+    - links_array (list): List of links to scrape text from.
+    - output_file (str): File to save the scraped text.
+    - csv_output_file (str): File to save the URL and text information in CSV format.
+    - remove_extra_whitespace (bool): If True, remove extra whitespace and empty lines from the output.
+
+    Example:
+    ```python
+    from pywebscrapr import scrape_text
+
+    # Using links from a file and saving text to output.txt
+    scrape_text(links_file='links.txt', output_file='output.txt')
+
+    # Using links directly and saving text to output.txt and csv_output.csv with extra whitespace removal
+    links = ['https://example.com/page1', 'https://example.com/page2']
+    scrape_text(links_array=links, output_file='output.txt', csv_output_file='csv_output.csv', remove_extra_whitespace=True)
+    ```
+    """
+    links = []
+    if links_file:
+        with open(links_file, 'r') as file:
+            links = file.read().splitlines()
+    elif links_array:
+        links = links_array
+    else:
+        raise ValueError("Either 'links_file' or 'links_array' must be provided.")
+
+    all_text = ""
+    csv_data = []
+
+    for link in links:
+        response = requests.get(link)
+        soup = BeautifulSoup(response.text, 'html.parser')
+
+        for element in soup.find_all(lambda tag: tag.name not in ['script', 'style']):
+            if remove_extra_whitespace:
+                text = element.get_text(strip=True)  # Remove extra whitespace
+                if text:  # Skip empty lines
+                    all_text += text + "\n"
+            else:
+                text = element.get_text()
+                all_text += text + "\n"
+            if csv_output_file:
+                csv_data.append({'URL': link, 'Text': text})
+
+    # Save text to output file
+    with open(output_file, 'w', encoding='utf-8') as text_file:
+        if remove_extra_whitespace:
+            text_file.write(all_text.rstrip())  # Remove trailing whitespace
+        else:
+            text_file.write(all_text)
+
+    # Save CSV data to CSV file
+    if csv_output_file:
+        with open(csv_output_file, 'w', newline='', encoding='utf-8') as csv_file:
+            fieldnames = ['URL', 'Text']
+            writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
+            writer.writeheader()
+            writer.writerows(csv_data)

pywebscrapr-0.1.0.dist-info/LICENSE

@@ -0,0 +1,7 @@
+MIT License (Modified)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to make derivative works based on the Software, provided that any substantial changes to the Software are clearly distinguished from the original work and are distributed under a different name.
+
+The original copyright notice and disclaimer must be retained in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS," WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT, OR OTHERWISE, ARISING FROM, OUT OF, OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

pywebscrapr-0.1.0.dist-info/METADATA

@@ -0,0 +1,24 @@
+Metadata-Version: 2.1
+Name: pywebscrapr
+Version: 0.1.0
+Summary: An open-source Python library for web scraping tasks.
+Home-page: https://github.com/infinitode/pywebscrapr
+Author: Infinitode Pty Ltd
+Author-email: infinitode.ltd@gmail.com
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests
+Requires-Dist: beautifulsoup4
+
+An open-source Python library for web scraping tasks. Includes support for both image scraping and text scraping.

pywebscrapr-0.1.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+pywebscrapr/__init__.py,sha256=cVYuZOKDc1-6HPbkWcWc_wR0lvA3MZv2To_prUZuGAU,4552
+pywebscrapr-0.1.0.dist-info/LICENSE,sha256=ZIftxnYzzlrappVuiyXoFI6QAiB1s2fzg9w07mxF-QI,949
+pywebscrapr-0.1.0.dist-info/METADATA,sha256=2BA3XAFDW-1isEB9sNEIcgR903Mli0gHTekHxDIPjRE,983
+pywebscrapr-0.1.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+pywebscrapr-0.1.0.dist-info/top_level.txt,sha256=-3_Ebeblzcx7vsig3hq8MJ43Zy_iBXFcge2NaOvaPLE,12
+pywebscrapr-0.1.0.dist-info/RECORD,,

pywebscrapr-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.42.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

pywebscrapr-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+pywebscrapr