urlicon 0.1.0__py3-none-any.whl

urlicon/string_cache.py

@@ -0,0 +1,116 @@
+import os
+
+
+class string_cache:
+    cache_folder: str | None = None
+    cache_files_extension: str = "html"
+
+    def __init__(self, cache_folder: str | None = None):
+        if cache_folder is not None:
+            self.cache_folder = cache_folder
+        else:
+            self.cache_folder = self.get_cache_folder()
+        pass
+
+    def safe_cache_id(func):
+        def filter_cache_id(cache_id):
+            cache_id = cache_id.replace('"', "").replace("\\", "")
+            cache_id = f'"{cache_id}"'
+            return cache_id
+
+        def _filter_cache_id_func(*args, **kwargs):
+            if "cache_id" in args:
+                args["cache_id"] = filter_cache_id(cache_id=args["cache_id"])
+
+            if "cache_id" in kwargs:
+                kwargs["cache_id"] = filter_cache_id(cache_id=kwargs["cache_id"])
+
+            return func(*args, **kwargs)
+
+        return _filter_cache_id_func
+
+    @safe_cache_id
+    def set(self, text: str, cache_id: str):
+        cache_folder = self.get_cache_folder()
+        cache_index_file_path = self.get_cache_index_path()
+
+        cache_folder_files = os.listdir(cache_folder)
+        cached_file_index = self.get_index_from_file_index(_safe_cache_id=cache_id)
+        if cached_file_index is not None:
+            new_file_index = cached_file_index
+        else:
+            new_file_index = len(cache_folder_files)
+            with open(cache_index_file_path, "a") as cache_index_file_writer:
+                cache_index_file_writer.write(f"\n{new_file_index}: {cache_id}")
+
+        new_file_name = f"{new_file_index}.{self.cache_files_extension}"
+        new_file_path = os.path.join(cache_folder, new_file_name)
+        with open(new_file_path, "w+") as new_file_writer:
+            new_file_writer.write(text)
+
+    @safe_cache_id
+    def get(self, cache_id: str) -> str:
+        cached_file_index = self.get_index_from_file_index(_safe_cache_id=cache_id)
+        code = self.get_cached_file_by_index(cached_file_index=cached_file_index)
+        return code
+
+    def get_index_from_file_index(self, _safe_cache_id):
+        cache_index_file = self.get_cache_index_file()
+        if cache_index_file.find(_safe_cache_id) < 1:
+            return None
+        cache_index_file = cache_index_file[: cache_index_file.find(_safe_cache_id) - 2]
+        cached_file_index = int(cache_index_file.split("\n")[-1].strip())
+        return cached_file_index
+
+    def get_cache_index_path(
+        self,
+    ) -> str:
+        cache_index_file_name = "cache_index.yaml"
+        cache_folder = self.get_cache_folder()
+        cache_index_file_path = os.path.join(cache_folder, cache_index_file_name)
+
+        if not os.path.exists(cache_index_file_path):
+            with open(cache_index_file_path, "w+") as cache_index_file_writer:
+                cache_index_file_writer.write(f"0: {cache_index_file_name}")
+
+        return cache_index_file_path
+
+    def get_cache_index_file(
+        self,
+    ) -> str:
+        with open(self.get_cache_index_path(), "r") as f:
+            cache_index_file_content = f.read()
+
+        return cache_index_file_content
+
+    def get_cached_file_by_index(self, cached_file_index: int) -> str:
+        code = None
+        cache_folder = self.get_cache_folder()
+        cached_file_name = f"{cached_file_index}.{self.cache_files_extension}"
+        cached_file_path = os.path.join(cache_folder, cached_file_name)
+        if not os.path.exists(cached_file_path):
+            return None
+        with open(cached_file_path, "r") as cached_file_reader:
+            code = cached_file_reader.read()
+        return code
+
+    def clean(
+        self,
+    ):
+        cache_folder = self.get_cache_folder()
+        cache_folder_files = os.listdir(cache_folder)
+        for file in cache_folder_files:
+            file_to_clean = os.path.join(cache_folder, file)
+            if os.path.exists(file_to_clean):
+                os.remove(file_to_clean)
+
+    def get_cache_folder(
+        self,
+    ):
+        import tempfile
+
+        if self.cache_folder is not None:
+            return self.cache_folder
+
+        tmpdirname = tempfile.mkdtemp()
+        return tmpdirname

urlicon/urlicon.py

@@ -0,0 +1,157 @@
+import os
+import re
+import urllib
+
+import requests
+from bs4 import BeautifulSoup
+from dotenv import load_dotenv
+
+from urlicon import urls
+from urlicon.string_cache import string_cache
+
+load_dotenv()
+
+STRING_CACHE_ROOT_DIR = os.getenv("STRING_CACHE_ROOT_DIR", None)
+cache = string_cache(cache_folder=STRING_CACHE_ROOT_DIR)
+
+
+def get_url_icon(url):
+
+    domain = urls.extract_domain_from_url(url=url)
+    if len(domain) == 0:
+        return get_default_img(text=url)
+
+    meta_icon, url_soup = get_meta_icon_from_url(url=url)
+    if meta_icon is not None:
+        return meta_icon
+
+    favicon = get_favicon_from_url(url=url)
+    if favicon is not None:
+        return favicon
+
+    url_first_img = get_first_img_from_url(url=url, url_soup=url_soup)
+    if url_first_img is not None:
+        return urls.ensure_domain(url_first_img, domain)
+
+    domain_first_img = get_first_img_from_url(url=domain)
+    if domain_first_img is not None:
+        return urls.ensure_domain(domain_first_img, domain)
+
+    return get_default_img(text=url)
+
+
+def get_meta_icon_from_url(url, url_soup=None):
+    soup_icons, url_soup = get_soup_icons_from_url(url, url_soup)
+
+    if soup_icons is None or len(soup_icons) == 0:
+        img = None
+        return img, url_soup
+    else:
+        final_icon = soup_icons[0]
+        for soup_icon in soup_icons:
+            if get_soup_icon_size(soup_icon) >= get_soup_icon_size(final_icon):
+                final_icon = soup_icon
+        img = final_icon["href"]
+
+    img = urls.ensure_relative_path(img, url)
+    return img, url_soup
+
+
+def get_soup_icons_from_url(url, url_soup=None):
+    if url_soup is None:
+        try:
+            url_request = requests_get(url)
+            url_soup = BeautifulSoup(url_request, features="html.parser")
+        except:
+            return None, None
+
+    soup_icons = url_soup.find_all(
+        "link",
+        attrs={
+            "rel": [
+                "icon",
+                "apple-touch-icon",
+            ]
+        },
+    )
+    return soup_icons, url_soup
+
+
+def get_soup_icon_size(soup_icon):
+    if not soup_icon.has_attr("sizes"):
+        return 1
+    return int(soup_icon["sizes"].split("x")[0])
+
+
+def get_favicon_from_url(url):
+    domain = urls.extract_domain_from_url(url=url)
+    favicon = domain + "/favicon.ico"
+    try:
+        favicon_request = requests_get(favicon)
+    except:
+        return None
+
+    if favicon_request is None or "html" in favicon_request[:150]:
+        return None
+    else:
+        return favicon
+
+
+def get_first_img_from_url(url, url_soup=None):
+    try:
+        url_request = requests_get(url)
+    except:
+        return None
+
+    if url_request is None:
+        return None
+    if url_soup is None:
+        url_soup = BeautifulSoup(url_request, features="html.parser")
+
+    first_img = url_soup.find("img")
+    if first_img is not None:
+        img = urls.ensure_domain(first_img["src"], url)
+        return img
+    else:
+        return None
+
+
+def get_default_img(text):
+    text = re.sub(r"[^a-zA-Z0-9./+]", "", text)
+    text = text.replace("www", "").replace("https", "")
+    text = text.replace(".", " ").replace("/", " ").strip()
+    text = urllib.parse.quote(text, safe="=?&")
+    return f"https://ui-avatars.com/api/?name={text}"
+
+
+def get_img_from_a_soup_item(soup_item, domain):
+    soup_item["href"] = urls.ensure_domain(url=soup_item["href"], domain=domain)
+
+    use_soup_img = False
+    if len(soup_item.find_all("img")) > 0:
+        if len(soup_item.find("img")["src"]) < 200:
+            use_soup_img = False
+        else:
+            use_soup_img = True
+
+    if use_soup_img:
+        img = soup_item.find("img")["src"]
+    else:
+        img = get_url_icon(soup_item["href"])
+    return img
+
+
+def requests_get(url):
+    cache_prefix = "sniff-urf:"
+    cached_code = cache.get(cache_id=cache_prefix + url)
+
+    if cached_code is not None:
+        return cached_code
+
+    req = requests.get(url=url, timeout=5)
+    if req.status_code != 200:
+        return None
+
+    code = req.text
+    cache.set(text=code, cache_id=cache_prefix + url)
+    return code

urlicon/urls.py

@@ -0,0 +1,85 @@
+import urllib
+
+import requests
+
+
+def ensure_domain(url, domain):
+    if (
+        urllib.parse.urlparse(url).netloc == ""
+        and urllib.parse.urlparse(domain).netloc == ""
+    ):
+        raise ValueError("`url` or `domain` must be a domain")
+
+    if urllib.parse.urlparse(url).netloc == "":
+        parsed_url = urllib.parse.urlparse(url)
+        parsed_domain = urllib.parse.urlparse(domain)
+        url = parsed_url._replace(scheme="https", netloc=parsed_domain.netloc).geturl()
+
+    url = urllib.parse.urlparse(url)._replace(scheme="https").geturl()
+    return url
+
+
+def get_name_from_domain(url):
+    href_parse = urllib.parse.urlparse(url)
+    country_code_len = 2
+
+    if href_parse.netloc == "":
+        raise ValueError("`url` must have a domain")
+    name = href_parse.netloc.split(".")
+
+    if len(name[-1]) == country_code_len:
+        name = name[:-1]
+
+    if name[0] == "www":
+        name = name[1:]
+
+    if len(name) > 2:
+        name = "_".join(name[-3:-1][::-1])
+    elif len(name) == 1:
+        name = name[0]
+    else:
+        name = name[-2]
+    return name
+
+
+def extract_domain_from_url(url):
+    domain = ""
+
+    url_parse = urllib.parse.urlparse(url)
+    if len(url_parse.netloc) > 0:
+        domain = "://".join([url_parse.scheme, url_parse.netloc])
+    return domain
+
+
+def read_from_url_or_path(url_path):
+    if str(url_path).startswith("https://"):
+        try:
+            content = requests.get(url_path).text
+        except:
+            return ""
+    else:
+        url_path = url_path.split(".")[0] + ".html"
+        with open(url_path, "r") as f:
+            content = f.read()
+    return content
+
+
+def ensure_relative_path(path, url):
+    if path[0] == "/":
+        return ensure_domain(url=path, domain=url)
+
+    if len(extract_domain_from_url(path)) > 0:
+        return path
+
+    parsed_url = urllib.parse.urlparse(url)
+    last_url_element = parsed_url.path.split("/")[-1]
+    if last_url_element.find(".") >= 0:
+        parsed_url_path = parsed_url.path.split("/")[:-1]
+    else:
+        parsed_url_path = parsed_url.path.split("/")
+    url = parsed_url._replace(
+        path="/".join(parsed_url_path), query="", params="", fragment=""
+    ).geturl()
+
+    url = f"{url}/{path}"
+    return url

urlicon-0.1.0.dist-info/METADATA

@@ -0,0 +1,66 @@
+Metadata-Version: 2.4
+Name: urlicon
+Version: 0.1.0
+Summary: `URLicon` helps you to discover an possible icon from a URL.
+Author-email: Cesar Cardoso <hello@cesarcardoso.cc>
+License-Expression: MIT
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: bs4>=0.0.2
+Requires-Dist: dotenv>=0.9.9
+Requires-Dist: requests>=2.32.5
+
+# URLicon - v0.1.0
+
+`URLicon` helps you to discover an possible icon from a URL.
+
+We check for the metatag `icons`, `favicons` and, if we don't find, we check the
+first image in the URL html code. Finally, if nothing is found, we use the
+[https://ui-avatars.com/api/](https://ui-avatars.com/api/) to bring you at least
+some avatar-like icon.
+
+### How to install and use
+
+Install with `uv` or `pip`
+```shell
+uv add urlicon
+# or
+pip urlicon
+```
+
+Usage:
+```python
+from urlicon import urlicon
+
+url = "https://this-is.your-url.com/some-path"
+
+icon_url = urlicon.get_url_icon(url)
+
+print("icon:", icon_url)
+# icon: "https://this-is.your-url.com/icon.jpeg"
+```
+
+### Caching
+
+`URLicon` use a simple "cache" method to avoid unecessary URL requests.
+It uses a [temp dir](https://docs.python.org/3/library/tempfile.html) for each
+execution. But you can define a your own directory and use the cache as much as
+you want setting `STRING_CACHE_ROOT_DIR` env var.
+
+```python
+STRING_CACHE_ROOT_DIR = os.getenv("STRING_CACHE_ROOT_DIR", None)
+cache = string_cache(cache_folder=STRING_CACHE_ROOT_DIR)
+```
+
+And you can clean the cache with:
+```python
+urlicon.string_cache.clean()
+```
+
+## See Also
+
+- Github: https://github.com/bouli/urlicon
+- PyPI: https://pypi.org/project/urlicon/
+
+## License
+This package is distributed under the [MIT license](https://opensource.org/license/MIT).

urlicon-0.1.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+urlicon/string_cache.py,sha256=fY4ZtAdZC3PDfdotuHZmuSAcQxQLA4WyJJ6pmOUZZMc,4130
+urlicon/urlicon.py,sha256=zcbevt29l2xbr14z9jFxz64TrvyS8O-fk29mCSQIoqI,4066
+urlicon/urls.py,sha256=ErdlgRva3bxs6HuBX2iHng21PqpWGIF1etZFToJq4jc,2235
+urlicon-0.1.0.dist-info/METADATA,sha256=1SEq8BOcKYjoGQhcFFrgQFscuJoe0Sv0wDcXyq3YBLw,1713
+urlicon-0.1.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+urlicon-0.1.0.dist-info/top_level.txt,sha256=Jts8QbeWp-6xANJ9KVj3CHk6L-8D950AQ_ZKXmF4YGI,8
+urlicon-0.1.0.dist-info/RECORD,,

urlicon-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.10.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

urlicon-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+urlicon