softhauzpy 0.0.7__tar.gz → 0.0.9__tar.gz

{softhauzpy-0.0.7 → softhauzpy-0.0.9}/PKG-INFO

@@ -1,8 +1,11 @@
 Metadata-Version: 2.1
 Name: softhauzpy
-Version: 0.0.7
+Version: 0.0.9
+Summary: is a comprehensive Python toolkit built for developers creating intelligent, data-driven web applications. It provides a powerful suite of web utilities including web scraping tools, crawling systems, content extraction pipelines, and search engine components that help developers build fully customizable in-house website search solutions.
+Home-page: https://softhauz.ca
 Author: Karen Urate
 Author-email: karen.urate@softhauz.ca
+License: MIT
 Description-Content-Type: text/markdown
 
 # SofthauzPy

softhauzpy-0.0.9/setup.py

@@ -0,0 +1,22 @@
+from setuptools import setup, find_packages
+
+with open("README.md", "r", encoding="utf-8") as f:
+    long_description = f.read()
+
+setup(
+    name='softhauzpy',
+    version='0.0.9',
+    author='Karen Urate',
+    author_email='karen.urate@softhauz.ca',
+    packages=find_packages(),
+    install_requires=[
+        'requests>=2.32.3',
+        'beautifulsoup4>=4.12.3',
+        'nltk>=3.9.4'
+    ],
+    url="https://softhauz.ca",
+    license="MIT",
+    description="is a comprehensive Python toolkit built for developers creating intelligent, data-driven web applications. It provides a powerful suite of web utilities including web scraping tools, crawling systems, content extraction pipelines, and search engine components that help developers build fully customizable in-house website search solutions.",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+)

{softhauzpy-0.0.7 → softhauzpy-0.0.9}/softhauzpy/__init__.py

@@ -3,7 +3,7 @@ from .main import incremental_update, highlight_query_terms, build_sitemap_urls
 from .main import fingerprint_page, generate_snippet
 
 # extractions
-from .main import extract_structured_data, extract_headings
+from .main import detect_input_type, extract_structured_data, extract_headings
 from .main import extract_metadata, extract_links, extract_pure_text
 
 # indexing

{softhauzpy-0.0.7 → softhauzpy-0.0.9}/softhauzpy/main.py

@@ -15,7 +15,7 @@ External Package List:
 - nltk >= (v. 3.9.4)
 
 """
-
+import os
 import re
 import json
 import math
@@ -56,18 +56,51 @@ except Exception:
     }
     _NLTK_AVAILABLE = False
 
+"""
+    Detect whether the input is a URL, an HTML file path, or a raw string.
+
+    Parameters
+    ----------
+    value           : String - The input to evaluate.
+
+    Returns
+    -------
+    url             : String - 'url', 'html_file', or 'raw_string'
+"""
+def detect_input_type(value: str) -> str:
+    # Check if it's a URL (http/https/ftp scheme)
+    try:
+        parsed = urlparse(value)
+        if parsed.scheme in ("http", "https", "ftp", "ftps"):
+            return "url"
+    except Exception:
+        pass
+
+    # Check if it's a path to an existing HTML file
+    if os.path.isfile(value) and value.lower().endswith((".html", ".htm")):
+        return "html_file"
+
+    # Check if it looks like an HTML string even if not a file
+    stripped = value.strip()
+    if stripped.startswith("<") and stripped.endswith(">"):
+        return "raw_string"
+
+    # Default: treat as plain raw string
+    return "raw_string"
+
 """
 
     Fetch a webpage and return only the pure text content found within its HTML tags.
 
     Parameters
     ----------
-    url           : String - The URL to fetch.
+    page_url      : String - Accepts either the URL to fetch, a local path to an HTML file, or the text/HTML String content of the page.
     title         : String - Optional document title (included in the returned text header when provided).
     author        : String - Optional document author (included in the returned text header when provided).
     description   : String - Optional description (included in the returned text header when provided).
     creation_date : String - Optional creation date string (included in the returned text header when provided).
     modified_date : String - Optional last-modified date string (included in the returned text header when provided).
+    assigned_location : String - Optional the URL to assign if the value passed in page_url is the text/HTML String content of the page. 
 
 
     Returns
@@ -88,8 +121,6 @@ except Exception:
         If the server returns a non-2xx status code.
     
 """
-
-
 def extract_pure_text(
         page_url: str,
         *,
@@ -97,11 +128,24 @@ def extract_pure_text(
         author: str | None = None,
         description: str | None = None,
         creation_date: str | None = None,
-        modified_date: str | None = None) -> dict:
-    response = fetch_page(page_url, timeout=15)
-    response.raise_for_status()
+        modified_date: str | None = None,
+        assigned_location: str | None = None) -> dict:
+
+    input_type = detect_input_type(page_url)
 
-    soup = BeautifulSoup(response.text, "html.parser")
+    if input_type == "url":
+        response = fetch_page(page_url, timeout=15)
+        response.raise_for_status()
+        html_content = response.text
+
+    elif input_type == "html_file":
+        with open(page_url, "r", encoding="utf-8") as f:
+            html_content = f.read()
+
+    else:
+        html_content = page_url
+
+    soup = BeautifulSoup(html_content, "html.parser")
 
     for tag in soup.find_all(_SKIP_TAGS):
         tag.decompose()
@@ -121,11 +165,12 @@ def extract_pure_text(
     if modified_date:
         header_parts.append(f"Last Modified: {modified_date}")
     if page_url:
-        header_parts.append(f"URL:           {page_url}")
+        if (input_type == "url") or (input_type == "html"):
+            header_parts.append(f"URL:           {page_url}")
 
     header = " ".join(header_parts)
     result = {
-        "url": page_url,
+        "url": page_url if ((input_type == "url") or (input_type == "html")) else assigned_location,
         "title": title,
         "author": author,
         "description": description,
@@ -184,9 +229,9 @@ def get_search_results_list(page_list=[], keywords='') -> list:
         description = page[3] or ''
         creation_date = page[4] or ''
         modified_date = page[5] or ''
-        
-        
-        if keywords in (extract_pure_text(url, title=title, author=author, description=description, creation_date=creation_date, modified_date=modified_date)["content"]).lower():
+        assigned_location = page[6] or ''
+
+        if keywords in (extract_pure_text(url, title=title, author=author, description=description, creation_date=creation_date, modified_date=modified_date, assigned_location=assigned_location)["content"]).lower():
             results.append((url, title, author, description, creation_date, modified_date))
 
     return results

{softhauzpy-0.0.7 → softhauzpy-0.0.9}/softhauzpy.egg-info/PKG-INFO

@@ -1,8 +1,11 @@
 Metadata-Version: 2.1
 Name: softhauzpy
-Version: 0.0.7
+Version: 0.0.9
+Summary: is a comprehensive Python toolkit built for developers creating intelligent, data-driven web applications. It provides a powerful suite of web utilities including web scraping tools, crawling systems, content extraction pipelines, and search engine components that help developers build fully customizable in-house website search solutions.
+Home-page: https://softhauz.ca
 Author: Karen Urate
 Author-email: karen.urate@softhauz.ca
+License: MIT
 Description-Content-Type: text/markdown
 
 # SofthauzPy

softhauzpy-0.0.7/setup.py

@@ -1,19 +0,0 @@
-from setuptools import setup, find_packages
-
-with open("README.md", "r", encoding="utf-8") as f:
-    description = f.read()
-
-setup(
-    name='softhauzpy',
-    version='0.0.7',
-    author='Karen Urate',
-    author_email='karen.urate@softhauz.ca',
-    packages=find_packages(),
-    install_requires=[
-        'requests>=2.32.3',
-        'beautifulsoup4>=4.12.3',
-        'nltk>=3.9.4'
-    ],
-    long_description=description,
-    long_description_content_type="text/markdown",
-)