geler-CERTIC 0.1.0__tar.gz

geler_certic-0.1.0/PKG-INFO

@@ -0,0 +1,71 @@
+Metadata-Version: 2.1
+Name: geler-CERTIC
+Version: 0.1.0
+Summary: Help convert dynamic webistes to static ones.
+License: CECILL-C
+Author: Mickaël Desfrênes
+Author-email: mickael.desfrenes@unicaen.fr
+Requires-Python: >=3.9,<4.0
+Classifier: License :: CeCILL-C Free Software License Agreement (CECILL-C)
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: argh (>=0.31.3,<0.32.0)
+Requires-Dist: beautifulsoup4 (>=4.12.3,<5.0.0)
+Requires-Dist: requests (>=2.32.3,<3.0.0)
+Description-Content-Type: text/markdown
+
+# Geler
+
+Help convert dynamic websites to static ones.
+
+## Install
+
+    pip install geler-CERTIC
+
+## Usage
+
+As a library in your own program:
+
+    from geler import freeze
+    freeze("https://acme.tld/", "/path/to/local/dir/")
+
+As a CLI tool:
+    
+    $> geler --help
+    usage: geler [-h] start-url destination-path
+    
+    Freeze given site at start_url to directory at destination_path
+    
+    positional arguments:
+      start-url         -
+      destination-path  -
+    
+    optional arguments:
+      -h, --help        show this help message and exit
+
+
+## Why ?
+
+For [MaX](https://git.unicaen.fr/pdn-certic/MaX) and associated tools, 
+we needed a lightweight, portable, pure Python solution to convert 
+small dynamic websites to static ones.
+
+## Alternatives
+
+This tool has a narrow scope, on purpose. Please turn to these solutions if you need more:
+
+- [wget](https://www.gnu.org/software/wget/)
+- [pywebcopy](https://pypi.org/project/pywebcopy/)
+- [HTTrack](https://www.httrack.com)
+
+## Limitations
+
+- only works with HTTP GET
+- does not submit forms (even with GET method)
+- only considers URLs in `src` or `href` attributes
+- only considers URLs with `http` or `https` schemes
+- only downloads what is in the same [netloc](https://docs.python.org/3/library/urllib.parse.html) (same domain, same port) as the start URL
+- only patches URLs in `*.html` files, not in `*.js`, not in `*.css` (watch out for those `url(...)` in your CSS)

geler_certic-0.1.0/README.md

@@ -0,0 +1,52 @@
+# Geler
+
+Help convert dynamic websites to static ones.
+
+## Install
+
+    pip install geler-CERTIC
+
+## Usage
+
+As a library in your own program:
+
+    from geler import freeze
+    freeze("https://acme.tld/", "/path/to/local/dir/")
+
+As a CLI tool:
+    
+    $> geler --help
+    usage: geler [-h] start-url destination-path
+    
+    Freeze given site at start_url to directory at destination_path
+    
+    positional arguments:
+      start-url         -
+      destination-path  -
+    
+    optional arguments:
+      -h, --help        show this help message and exit
+
+
+## Why ?
+
+For [MaX](https://git.unicaen.fr/pdn-certic/MaX) and associated tools, 
+we needed a lightweight, portable, pure Python solution to convert 
+small dynamic websites to static ones.
+
+## Alternatives
+
+This tool has a narrow scope, on purpose. Please turn to these solutions if you need more:
+
+- [wget](https://www.gnu.org/software/wget/)
+- [pywebcopy](https://pypi.org/project/pywebcopy/)
+- [HTTrack](https://www.httrack.com)
+
+## Limitations
+
+- only works with HTTP GET
+- does not submit forms (even with GET method)
+- only considers URLs in `src` or `href` attributes
+- only considers URLs with `http` or `https` schemes
+- only downloads what is in the same [netloc](https://docs.python.org/3/library/urllib.parse.html) (same domain, same port) as the start URL
+- only patches URLs in `*.html` files, not in `*.js`, not in `*.css` (watch out for those `url(...)` in your CSS)

geler_certic-0.1.0/geler/__init__.py

@@ -0,0 +1,229 @@
+from typing import Union
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor
+from urllib.parse import urlparse, urljoin, ParseResult, parse_qs
+from bs4 import BeautifulSoup
+from requests.exceptions import RequestException
+from functools import cache
+import logging
+import requests
+import os
+import json
+import mimetypes
+from time import sleep
+from base64 import urlsafe_b64encode
+
+logger = logging.Logger(__name__)
+
+
+def url64(s: Union[dict, str]) -> str:
+    if isinstance(s, dict):
+        s = json.dumps(s, sort_keys=True).encode()
+    if isinstance(s, str):
+        s = s.encode()
+    return urlsafe_b64encode(s).decode().rstrip("=")
+
+
+@cache
+def guess_extension(mime_type: str) -> str:
+    extension = mimetypes.guess_extension(mime_type)
+    if extension:
+        return extension
+    else:
+        return ""
+
+
+@cache
+def url_to_path(url: str, mime: str = "") -> str:
+    index = ""
+    parsed_url = cached_urlparse(url)
+    path_no_ext, ext = os.path.splitext(parsed_url.path)
+    guessed_ext = guess_extension(mime)
+    def_path = path_no_ext
+    if guessed_ext and guessed_ext.lower() != ext:
+        def_path = path_no_ext + ext
+    if parsed_url.path.endswith("/"):
+        index = "index"
+    query_digest = ""
+    if parsed_url.query:
+        query_digest = "-" + url64(dict(parse_qs(parsed_url.query)))
+    result = f"{def_path}{index}{query_digest}{guessed_ext}"
+    return result
+
+
+@cache
+def cached_urlparse(url):
+    return urlparse(url)
+
+
+class Freezer:
+    def __init__(
+        self,
+        start_from: str,
+        destination: Union[Path, str],
+        skip_extensions=None,
+        http_get_timeout=30,
+        thread_pool_size=10,
+        user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/112.0",
+    ):
+        self.destination = Path(destination)
+        if not self.destination.exists():
+            os.makedirs(self.destination, exist_ok=True)
+        self.start_from_url = start_from.split("#")[0]  # ignore anchors
+        self.start_from_domain = cached_urlparse(self.start_from_url).netloc
+        self.get_timeout = http_get_timeout
+        self.max_workers = thread_pool_size
+        self.html_mimes = ["text/html", "application/xhtml+xml", "application/xml"]
+        self.schemes = ["http", "https"]
+        self.scrap_executor_pool = ThreadPoolExecutor(max_workers=self.max_workers)
+        self.links_done = {}
+        self.links_to_do = []
+        self.user_agent = user_agent
+        self.skip_extensions = []
+        if skip_extensions is not None:
+            self.skip_extensions = skip_extensions
+
+    @cache
+    def validate_candidate_url(
+        self, url: str, parent_url: str
+    ) -> Union[ParseResult, bool]:
+        if not url.strip():
+            return False
+        # exclude absolute links not in the site
+        if (
+            url.startswith("http://") or url.startswith("https://")
+        ) and not url.startswith(self.start_from_url):
+            return False  # Skip foreign URL
+        url = urljoin(parent_url, url)  # make link absolute
+        url = url.split("#")[0]  # ignore anchors
+        parsed_url = cached_urlparse(url)
+        if parsed_url.scheme not in self.schemes:  # excludes non http protocols
+            return False
+        if parsed_url.netloc != self.start_from_domain:
+            return False
+        _, ext = os.path.splitext(parsed_url.path)
+        if ext.lower() in self.skip_extensions:  # exclude extensions
+            return False
+        # TODO: should make url based on parsed_url here ?
+        if (
+            url not in self.links_done.keys() and url not in self.links_to_do
+        ):  # Not in queue, submit to executor
+            self.scrap_executor_pool.submit(self.scrap_item, url)
+            self.links_to_do.append(url)
+        else:  # Already in queue, nothing to do here
+            pass
+        return parsed_url
+
+    def _rebuild_link(self, parsed_url: ParseResult) -> str:
+        return (
+            parsed_url.path
+            + ("?" + parsed_url.query if parsed_url.query else "")
+            + ("#" + parsed_url.fragment if parsed_url.fragment else "")
+        )
+
+    def scrap_item(self, item_url):
+        anchor = ""
+        parts = item_url.split("#")
+        item_url_no_anchor = parts[0]
+        if len(parts) > 1:
+            anchor = parts[1]
+
+        # add to links in  done list early, avoid race conditions
+        if item_url_no_anchor not in self.links_done.keys():
+            self.links_done[item_url_no_anchor] = ""
+        else:
+            return
+        try:
+            response = requests.get(
+                item_url_no_anchor,
+                allow_redirects=True,
+                timeout=self.get_timeout,
+                headers={"User-Agent": self.user_agent},
+                stream=True,
+            )
+            if not response.ok:
+                return
+            content_type = response.headers.get("Content-Type").split(";")[0].strip()
+            self.links_done[item_url_no_anchor] = url_to_path(
+                item_url_no_anchor, content_type
+            )
+            if content_type in self.html_mimes:
+                soup = BeautifulSoup(response.content, "html.parser")
+                for attrib in ["href", "src"]:
+                    for link in soup.select(f"[{attrib}]"):
+                        validated_link = self.validate_candidate_url(
+                            link.get(attrib), item_url_no_anchor
+                        )
+                        if validated_link:
+                            link[attrib] = self._rebuild_link(validated_link)
+                self._save_item_to_disk(item_url_no_anchor, content_type, soup)
+            else:
+                self._save_item_to_disk(item_url_no_anchor, content_type, response)
+        except RequestException as e:
+            logger.warning(f"RequestException: {e} on URL {item_url_no_anchor}")
+        # whatever happens, remove item from to do list
+        finally:
+            while item_url_no_anchor in self.links_to_do:
+                self.links_to_do.remove(item_url_no_anchor)
+
+    def _save_item_to_disk(
+        self,
+        item_url: str,
+        content_type: str,
+        data: Union[requests.Response, BeautifulSoup],
+    ):
+        convert_url_to_path = url_to_path(item_url, content_type)
+        local_path = Path(self.destination, convert_url_to_path.lstrip("/"))
+        os.makedirs(os.path.dirname(local_path), exist_ok=True)
+        if type(data) is BeautifulSoup:
+            with open(local_path, "w") as f:
+                f.write(str(data))
+
+        if type(data) is requests.Response:
+            with open(local_path, "wb") as f:
+                f.write(data.content)
+
+        # update done list with new URL value
+        self.links_done[item_url] = convert_url_to_path
+
+    def _patch_links(self):
+        parsed_start_url = cached_urlparse(self.start_from_url)
+        trans_table = {}
+        for original_url, local_path in self.links_done.items():
+            url_no_netloc = original_url.split(parsed_start_url.netloc)[1]
+            trans_table[url_no_netloc] = local_path
+        for _, local_path in self.links_done.items():
+            if local_path.endswith(".html"):
+                self._patch_html_file(local_path, trans_table)
+
+    def _patch_html_file(self, local_path: Union[Path, str], trans_table: dict) -> bool:
+        f_path = Path(self.destination, local_path.lstrip("/"))
+        with open(f_path, "r+") as fp:
+            soup = BeautifulSoup(fp, "html.parser")
+            for link in soup.select("[href]"):
+                parts = link.get("href").split("#")
+                if parts[0] in trans_table.keys():
+                    link["href"] = trans_table[link.get("href")] + (
+                        "#" + parts[1] if len(parts) > 1 else ""
+                    )
+            for link in soup.select("[src]"):
+                parts = link.get("src").split("#")
+                if parts[0] in trans_table.keys():
+                    link["src"] = trans_table[link.get("src")] + (
+                        "#" + parts[1] if len(parts) > 1 else ""
+                    )
+            fp.seek(0)
+            fp.write(str(soup))
+            fp.truncate()
+        return True
+
+    def freeze(self):
+        self.scrap_item(self.start_from_url)
+        while len(self.links_to_do) > 0:
+            sleep(1)
+        self.scrap_executor_pool.shutdown(wait=True)
+        self._patch_links()
+
+
+def freeze(start_from_url: str, save_to_path: Union[Path, str]):
+    Freezer(start_from_url, save_to_path).freeze()

geler_certic-0.1.0/geler/__main__.py

@@ -0,0 +1,21 @@
+import argh
+from geler import Freezer
+
+
+def freeze(start_url: str, destination_path: str):
+    """
+    Freeze given site at start-url to directory at destination-path
+    """
+    f = Freezer(start_url, destination_path)
+    f.freeze()
+
+
+def run_cli():
+    # parser = argh.ArghParser()
+    # parser.add_commands([freeze])
+    # parser.dispatch()
+    argh.dispatch_command(freeze)
+
+
+if __name__ == "__main__":
+    run_cli()

geler_certic-0.1.0/pyproject.toml

@@ -0,0 +1,24 @@
+[tool.poetry]
+name = "geler-CERTIC"
+version = "0.1.0"
+description = "Help convert dynamic webistes to static ones."
+authors = ["Mickaël Desfrênes <mickael.desfrenes@unicaen.fr>"]
+license = "CECILL-C"
+readme = "README.md"
+packages = [{include = "geler"}]
+
+[tool.poetry.dependencies]
+python = "^3.9"
+argh = "^0.31.3"
+beautifulsoup4 = "^4.12.3"
+requests = "^2.32.3"
+
+[tool.poetry.group.dev.dependencies]
+black = "^24.10.0"
+
+[tool.poetry.scripts]
+geler = "geler.__main__:run_cli"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"