abstract-webtools 0.1.6.331__tar.gz → 0.1.6.332__tar.gz

{abstract_webtools-0.1.6.331 → abstract_webtools-0.1.6.332}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract_webtools
-Version: 0.1.6.331
+Version: 0.1.6.332
 Summary: Utilities for fetching/parsing web content with requests/urllib3/BS4 and helpers.
 Home-page: https://github.com/AbstractEndeavors/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.6.331 → abstract_webtools-0.1.6.332}/setup.py

@@ -5,7 +5,7 @@ README = Path("README.md").read_text(encoding="utf-8")
 
 setup(
     name="abstract_webtools",
-    version='0.1.6.331',  # bump once per release
+    version='0.1.6.332',  # bump once per release
     author="putkoff",
     author_email="partners@abstractendeavors.com",
     description="Utilities for fetching/parsing web content with requests/urllib3/BS4 and helpers.",

abstract_webtools-0.1.6.332/src/abstract_webtools/imports/src/__init__.py

@@ -0,0 +1,4 @@
+from .managers import *
+from .functions import *
+from .imports import *
+

abstract_webtools-0.1.6.332/src/abstract_webtools/imports/src/functions/DomainManager.py

@@ -0,0 +1,89 @@
+from ..imports import SingletonMeta,urlparse as _urlparse
+from ..functions import *
+from ..functions.domain_utils import  get_parsed_url as _get_parsed_url
+class DomainManager(metaclass=SingletonMeta):
+    def __init__(self,domain=None):
+        if not hasattr(self, 'initialized'):
+            self.initialized = True
+            self.domains = {}
+        if domain:
+            check_domains(domain)
+    def check_domains(self,url):
+        parsed_url = _get_parsed_url(url)
+        domain_name = parsed_url.get('name')
+        ext = parsed_url.get('ext')
+        if not domain_name in self.domains:
+            self.domains[domain_name] = {}
+        
+        path = parsed_url.get('path')
+        domain = parsed_url.get('domain')
+        http = parsed_url.get('http','https') or 'https'
+        parsed_url['http']=http
+        if ext not in self.domains[domain_name]:
+            self.domains[domain_name][ext] = parsed_url
+        if 'paths' not in self.domains[domain_name][ext]:
+            self.domains[domain_name][ext]['paths'] = []
+            
+        if path not in self.domains[domain_name][ext].get('paths',[]):
+            self.domains[domain_name][ext]['paths'].append(path)
+            del self.domains[domain_name][ext]['path']
+        else:
+            return self.domains[domain_name][ext]    
+        if 'variants' not in self.domains[domain_name][ext]:
+            variants = get_url_variants(domain)                        
+            self.domains[domain_name][ext]['variants'] = variants
+        if 'valid_variants' not in self.domains[domain_name][ext]:
+            variants = self.domains[domain_name][ext]['variants']
+            valid_url_variants = get_valid_url_variants(domain = domain)
+            if valid_url_variants and len(valid_url_variants)>0:
+                valid_url_variant = valid_url_variants[0]
+                http = get_http(valid_url_variant)
+                if http:
+                    self.domains[domain_name][ext]['http']=http
+            self.domains[domain_name][ext]['valid_variants'] =  valid_url_variants                      
+        if 'full_domain' not in self.domains[domain_name][ext]:
+            self.domains[domain_name][ext]['full_domain']=ensure_http(domain)
+        if 'domain_valid' not in self.domains[domain_name][ext]:
+            valid_url_variants = self.domains[domain_name][ext]['valid_variants']
+            self.domains[domain_name][ext]['domain_valid']=False if not valid_url_variants else True
+
+        return self.domains[domain_name][ext]
+    def get_parsed_url(self,url):
+        parsed = self.check_domains(url)
+        parsed_url = parsed.copy()
+        parsed['path'] = get_domain_path(url)
+        return parsed
+    def urlparse(self,url):
+        parsed = self.check_domains(url)
+        full_domain = parsed.get('full_domain')
+        parsed['path'] = get_domain_path(url)
+        domain = get_full_url(parsed_url=parsed)
+        parsed = _urlparse(domain)
+        return parsed
+domain_mgr = DomainManager()
+##def parse_url(*args,url=None,parsed=None,**kwargs):
+##    
+##    for arg in args:
+##        if isinstance(arg,str):
+##            url = arg
+##            parsed = domain_mgr.check_domains(url)
+##        elif isinstance(arg,dict) and parsed == None:
+##            parsed=arg
+##    if not url and parsed:
+##        url = parsed.get('full_url')
+##    parsed_url = domain_mgr.urlparse(url)
+##    parsed = domain_mgr.check_domains(url)
+##    full_domain = parsed.get('full_domain')
+##    parsed['path'] = get_domain_path(url)
+##    full_url = get_full_url(parsed_url=parsed)
+##    parsed['full_url'] = full_url
+##    parsed_url = _parse_url(url=full_url)
+##    parsed_url.update(parsed)
+##    return parsed_url
+def get_parsed_url(url):
+    parsed_url = domain_mgr.get_parsed_url(url)
+    return parsed_url
+def urlparse(url):
+    parsed_url = domain_mgr.urlparse(url)
+    return parsed_url
+parse_url = get_parsed_url

abstract_webtools-0.1.6.332/src/abstract_webtools/imports/src/functions/__init__.py

@@ -0,0 +1,8 @@
+from .domain_utils import *
+from .providers import *
+from .titles import *
+from .DomainManager import *
+from .imports import *
+from .specUrl_utils import *
+from .url_utils import *
+from .utils import *

abstract_webtools-0.1.6.332/src/abstract_webtools/imports/src/functions/domain_utils/__init__.py

@@ -0,0 +1,6 @@
+from .correct_url_utils import *
+from .domain_parse_utils import *
+from .get_strip_utils import *
+
+
+

abstract_webtools-0.1.6.332/src/abstract_webtools/imports/src/functions/domain_utils/correct_url_utils.py

@@ -0,0 +1,37 @@
+from ..imports import *
+def get_correct_url(url=None) -> str:
+    """
+    Try candidates (HEAD request). 
+    Return first that resolves with 200.
+    """
+
+    response = requests.get(url, stream=True, timeout=6, allow_redirects=True)
+    
+    if response.status_code == 200:
+        return url
+
+    return None
+def domain_exists(url=None) -> str:
+    """
+    Try candidates (HEAD request). 
+    Return first that resolves with 200.
+    """
+
+    response = requests.get(url, stream=True, timeout=6, allow_redirects=True)
+    
+    if response.status_code == 200:
+        return True
+    
+    return False
+def get_correct_urls(candidates=None) -> str:
+    """
+    Try candidates (HEAD request). 
+    Return first that resolves with 200.
+    """
+
+    urls = []
+    for candidate in candidates:
+        url  = get_correct_url(url=candidate)
+        if url:
+            urls.append(url)
+    return urls

abstract_webtools-0.1.6.332/src/abstract_webtools/imports/src/functions/domain_utils/domain_parse_utils.py

@@ -0,0 +1,137 @@
+from ..titles import *
+from .get_strip_utils import *
+from .correct_url_utils import *
+def sort_longest_first(values):
+    seen = set()
+    clean = []
+    for v in values:
+        if v and isinstance(v, str):
+            low = v.lower()
+            if low not in seen:
+                clean.append(v)
+                seen.add(low)
+    return sorted(clean, key=lambda s: (-len(s), s.lower()))
+
+def get_parsed_url(domain, **kwargs):
+    parsed_url = dict(kwargs)
+    post_variants = []
+    # http / www
+    http_www = get_http_www(domain)
+    parsed_url.update(http_www)
+
+    http = http_www.get('http')
+
+    # basic domain pieces
+    domain_paths = get_domain_paths(domain, http=http)
+    parsed_url.update(domain_paths)
+
+    domain_name_ext = get_domain_name_ext(domain, http=http)
+    parsed_url.update(domain_name_ext)
+
+    domain_name = parsed_url.get('name')
+    domain = parsed_url.get('domain')
+        # tokenization
+    tokenized_domain = tokenize_domain(domain)
+    parsed_url["tokenized_domain"] = tokenized_domain
+    app_name = " ".join(tokenized_domain)
+    parsed_url["app_name"] = app_name
+    
+
+
+  
+
+
+
+    # author / "i_url"
+    parsed_url["author"] = f"@{domain_name.lower()}"
+    parsed_url["i_url"] = f"{domain_name}://"
+
+    # combine with domain
+    # compute final title
+    title = get_title(parsed_url)
+    post_variants=[title,app_name,domain]
+    variants = title_variants_from_domain(domain)
+    base_variants = list(set([variant for variant in variants if variant not in post_variants]))
+    # update the organized variants
+    parsed_url["title_variants"] = get_all_title_variants(variants=base_variants,page=title,name=app_name,domain=domain)
+
+    parsed_url["title"] = pad_or_trim(
+        "title",
+        string=title,
+        title_variants=parsed_url["title_variants"],
+        page=title,
+        domain=domain,
+        name = app_name
+    )
+
+    # get keywords
+    keywords_info = get_keywords(parsed_url,page=title,domain=domain,name = app_name)
+    parsed_url.update(keywords_info)
+
+    keywords = parsed_url.get("keywords", [])
+
+    # FINAL: longest→shortest list with TITLE first, DOMAIN second
+    final_variants = [title, parsed_url["domain"]]
+
+    # remove title/domain from pool
+    pool = set(keywords + variants)
+    pool.discard(title)
+    pool.discard(parsed_url["domain"])
+
+    # sort longest → shortest
+    final_variants += sort_longest_first(pool)
+
+    parsed_url["title_variants"] = final_variants
+
+    return parsed_url
+
+def get_full_domain(url=None,parsed_url=None):
+    parsed_url = parsed_url or get_parsed_url(url)
+    http = parsed_url.get("http")
+    name = parsed_url.get("name")
+    ext = parsed_url.get("ext")
+    www = "www." if parsed_url.get("www") else ''
+    return f"{http}://{www}{name}{ext}"
+def get_full_url(url=None,parsed_url=None):
+    parsed_url = parsed_url or get_parsed_url(url)
+    full_domain = get_full_domain(url=url,parsed_url=parsed_url)
+    path = parsed_url.get("path")
+    path = f"/{path}" if path else ''
+    return f"{full_domain}{path}"
+def get_url_variants(domain,http=None):
+    parsed_url = get_parsed_url(domain,http=http)
+    stripped_url = parsed_url.copy()
+    http = parsed_url.get('http','https') or 'https'
+    parsed_url['http']=http
+    inverse_http = INVERSE_HTTP.get(http)
+    ensured_parsed_url = parsed_url.copy()
+    ensured_parsed_url['http'] = http
+    
+    www = parsed_url.get('www')
+    inverse_www = INVERSE_BOOL.get(www)
+    parsed_inv_url = ensured_parsed_url.copy()
+    parsed_inv_url['www'] = inverse_www
+    inverse_parsed_url = ensured_parsed_url.copy()
+    inverse_parsed_url['http'] = inverse_http
+    inverse_parsed_inv_url = parsed_inv_url.copy()
+    inverse_parsed_inv_url['http'] = inverse_http
+    url_variants = [ensured_parsed_url,parsed_inv_url,inverse_parsed_url,inverse_parsed_inv_url]
+    for i,url_variant in enumerate(url_variants):
+        http = url_variant.get("http")
+        name = url_variant.get("name")
+        ext = url_variant.get("ext")
+        path = url_variant.get("path")
+        www = "www." if url_variant.get("www") else ''
+        path = f"/{path}" if path else ''
+        url_variants[i] = f"{http}://{www}{name}{ext}{path}"
+    return url_variants
+def get_valid_url_variants(domain=None,http=None,variants=None):
+    url_variants = variants or get_url_variants(domain=domain,http=http)
+    return get_correct_urls(url_variants)
+def ensure_http(domain,http=None):
+    url_variants = get_url_variants(domain=domain,http=http)
+    correct_url = url_variants[0]
+    correct_urls = get_correct_urls(url_variants)
+    if correct_urls and len(correct_urls) >0:
+        correct_url = correct_urls[0]
+    return correct_url

abstract_webtools-0.1.6.332/src/abstract_webtools/imports/src/functions/domain_utils/get_strip_utils.py

@@ -0,0 +1,77 @@
+from ..imports import *
+def get_http(domain):
+    
+    http=None
+    if domain.startswith('http'):
+        http = 'http'
+        if domain.startswith('https'):
+            http = f"{http}s"
+    return http
+def strip_http(domain,http=None):
+    http = http or get_http(domain)
+   
+    domain = domain[len(http or ''):]
+    
+    domain = eatAll(domain,[':','/'])
+    return domain
+
+    
+def get_www(domain,http=None):
+    http = http or get_http(domain)
+    domain = strip_http(domain = domain,http=http)
+    if domain.startswith('www.'):
+        return True
+    return False   
+def strip_www(domain,http=None):
+    http = http or get_http(domain)
+    
+    domain = strip_http(domain = domain,http=http)
+    
+    www = get_www(domain,http=None)
+    if www:
+        domain = domain[len('www.'):]
+    if http:
+        
+        domain = f"{http}://{domain}"
+    return domain
+
+def get_http_www(domain):
+    http = get_http(domain)
+    www = get_www(domain,http=http)
+    return {"http":http,"www":www}
+def strip_http_www(domain,http=None):
+    domain = strip_www(domain,http=http)
+    domain = strip_http(domain,http=http)
+    return domain
+def get_stripped_domain(domain,http=None):
+    domain = strip_http_www(domain,http=http)
+    return domain.split('/')
+def get_domain_paths(domain,http=None):
+    domain = strip_http_www(domain,http=http)
+    domain_paths = domain.split('/')
+    paths = domain_paths[1:] if len(domain_paths)>1 else []
+    return {"domain":domain_paths[0],"path":'/'.join(paths)}
+def get_domain(domain,http=None):
+    domain_ext = get_domain_paths(domain,http=http)
+    return domain_ext.get("domain")
+def get_domain_path(domain,http=None):
+    domain_ext = get_domain_paths(domain,http=http)
+    return domain_ext.get("path")
+def get_domain_name_ext(domain,http=None):
+    domain = get_domain(domain,http=http)
+    domain_ext = os.path.splitext(domain)
+    return {"name":domain_ext[0],"ext":domain_ext[-1]}
+def get_extension(domain=None,http=None,parsed=None,options=None):
+    if parsed:
+        try:
+            domain = parsed.netloc
+            http = parsed.scheme
+        except:
+            pass
+    domain_ext = get_domain_name_ext(domain,http=http)
+    return domain_ext.get("ext")
+def get_domain_name(domain,http=None):
+    domain_ext = get_domain_name_ext(domain,http=http)
+    return domain_ext.get("name")
+
+get_extention = get_extension

abstract_webtools-0.1.6.332/src/abstract_webtools/imports/src/functions/providers/__init__.py

@@ -0,0 +1 @@
+from .main import *

abstract_webtools-0.1.6.332/src/abstract_webtools/imports/src/functions/providers/main.py

@@ -0,0 +1,62 @@
+from urllib.parse import urlparse, parse_qs
+import os
+import mimetypes
+
+# Provider imports
+from .src.youtube import detect_youtube
+from .src.vimeo import detect_vimeo
+from .src.tiktok import detect_tiktok
+from .src.facebook import detect_facebook
+from .src.twitter import detect_twitter
+from .src.instagram import detect_instagram
+from .src.reddit import detect_reddit
+from .src.twitch import detect_twitch
+from .src.imgur import detect_imgur
+from .src.pinterest import detect_pinterest
+from .src.giphy import detect_giphy
+from .src.tenor import detect_tenor
+from .src.direct_file import detect_direct_file
+
+PROVIDERS = [
+    detect_direct_file,   # always check this first
+    detect_youtube,
+    detect_vimeo,
+    detect_tiktok,
+    detect_facebook,
+    detect_twitter,
+    detect_instagram,
+    detect_reddit,
+    detect_twitch,
+    detect_imgur,
+    detect_pinterest,
+    detect_giphy,
+    detect_tenor
+]
+
+def _clean_url(url: str) -> str:
+    if not url:
+        return ""
+    url = url.strip()
+    if url.startswith("//"):
+        url = "https:" + url
+    return url
+
+def get_downloadable_info(url: str):
+    url = _clean_url(url)
+    parsed = urlparse(url)
+    query = parse_qs(parsed.query)
+
+    for provider in PROVIDERS:
+        result = provider(parsed, query) if provider.__code__.co_argcount == 2 else provider(parsed)
+        if result:
+            result["url"] = url
+            return result
+
+    return {
+        "downloadable": False,
+        "kind": None,
+        "provider": None,
+        "id": None,
+        "direct": False,
+        "url": url,
+    }