structables 0.2.2__tar.gz → 0.3.0__tar.gz

{structables-0.2.2 → structables-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: structables
-Version: 0.2.2
+Version: 0.3.0
 Summary: A simple frontend for Instructables
 Project-URL: Homepage, https://git.private.coffee/privatecoffee/structables
 Project-URL: Bug Tracker, https://git.private.coffee/privatecoffee/structables/issues

{structables-0.2.2 → structables-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "structables"
-version = "0.2.2"
+version = "0.3.0"
 authors = [
   { name="Private.coffee Team", email="support@private.coffee" },
 ]

structables-0.3.0/src/structables/config.py

@@ -0,0 +1,13 @@
+import os
+
+class Config:
+    DEBUG = os.environ.get("FLASK_DEBUG", os.environ.get("STRUCTABLES_DEBUG", False))
+    PORT = int(os.environ.get("STRUCTABLES_PORT", 8002))
+    LISTEN_HOST = os.environ.get("STRUCTABLES_LISTEN_HOST", "127.0.0.1")
+    INVIDIOUS = os.environ.get("STRUCTABLES_INVIDIOUS")
+    UNSAFE = os.environ.get("STRUCTABLES_UNSAFE", False)
+    PRIVACY_FILE = os.environ.get("STRUCTABLES_PRIVACY_FILE", "privacy.txt")
+
+    @staticmethod
+    def init_app(app):
+        pass

structables-0.3.0/src/structables/main.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+from flask import Flask
+
+from .config import Config
+from .routes import init_routes
+from .utils.data import update_data
+from .utils.helpers import get_typesense_api_key
+
+app = Flask(__name__, template_folder="templates", static_folder="static")
+app.config.from_object(Config)
+app.typesense_api_key = get_typesense_api_key()
+
+init_routes(app)
+
+if __name__ == "__main__":
+    app.run(port=app.config['PORT'], host=app.config['LISTEN_HOST'], debug=app.config['DEBUG'])
+
+# Initialize data when the server starts
+update_data(app)

structables-0.3.0/src/structables/routes/__init__.py

@@ -0,0 +1,12 @@
+from .main import init_main_routes
+from .category import init_category_routes
+from .member import init_member_routes
+from .proxy import init_proxy_routes
+from .contest import init_contest_routes
+
+def init_routes(app):
+    init_main_routes(app)
+    init_category_routes(app)
+    init_member_routes(app)
+    init_proxy_routes(app)
+    init_contest_routes(app)

structables-0.3.0/src/structables/routes/category.py

@@ -0,0 +1,76 @@
+from flask import redirect
+from werkzeug.exceptions import NotFound
+from ..utils.helpers import project_list, category_page
+
+
+def init_category_routes(app):
+    @app.route("/<category>/<channel>/projects/")
+    def route_channel_projects(category, channel):
+        return project_list(app, channel.title())
+
+    @app.route("/<category>/<channel>/projects/<sort>/")
+    def route_channel_projects_sort(category, channel, sort):
+        return project_list(
+            app,
+            channel.title(),
+            " Sorted by " + sort.title(),
+        )
+
+    @app.route("/<category>/projects/")
+    def route_category_projects(category):
+        return project_list(app, category.title())
+
+    @app.route("/<category>/projects/<sort>/")
+    def route_category_projects_sort(category, sort):
+        return project_list(app, category.title(), " Sorted by " + sort.title())
+
+    @app.route("/projects/")
+    def route_projects():
+        return project_list(app, "")
+
+    @app.route("/projects/<sort>/")
+    def route_projects_sort(sort):
+        return project_list(app, "", " Sorted by " + sort.title())
+
+    @app.route("/circuits/")
+    def route_circuits():
+        return category_page(app, "Circuits")
+
+    @app.route("/workshop/")
+    def route_workshop():
+        return category_page(app, "Workshop")
+
+    @app.route("/craft/")
+    def route_craft():
+        return category_page(app, "Craft")
+
+    @app.route("/cooking/")
+    def route_cooking():
+        return category_page(app, "Cooking")
+
+    @app.route("/living/")
+    def route_living():
+        return category_page(app, "Living")
+
+    @app.route("/outside/")
+    def route_outside():
+        return category_page(app, "Outside")
+
+    @app.route("/teachers/")
+    def route_teachers():
+        return category_page(app, "Teachers", True)
+
+    @app.route("/<category>/<channel>/")
+    def route_channel_redirect(category, channel):
+        if (
+            category == "circuits"
+            or category == "workshop"
+            or category == "craft"
+            or category == "cooking"
+            or category == "living"
+            or category == "outside"
+            or category == "teachers"
+        ):
+            return redirect(f"/{category}/{channel}/projects/", 307)
+        else:
+            raise NotFound()

structables-0.3.0/src/structables/routes/contest.py

@@ -0,0 +1,177 @@
+from flask import render_template, request, abort
+from urllib.request import urlopen
+from urllib.error import HTTPError
+from ..utils.helpers import proxy
+from bs4 import BeautifulSoup
+
+def init_contest_routes(app):
+    @app.route("/contest/archive/")
+    def route_contest_archive():
+        page = 1
+        if request.args.get("page") is not None:
+            page = request.args.get("page")
+
+        try:
+            data = urlopen(f"https://www.instructables.com/contest/archive/?page={page}")
+        except HTTPError as e:
+            abort(e.code)
+
+        soup = BeautifulSoup(data.read().decode(), "html.parser")
+
+        main = soup.select("div#contest-archive-wrapper")[0]
+
+        contest_count = main.select("p.contest-count")[0].text
+
+        contest_list = []
+        for index, year in enumerate(main.select("div.contest-archive-list h2")):
+            year_list = main.select(
+                "div.contest-archive-list div.contest-archive-list-year"
+            )[index]
+            year_name = year.text
+            month_list = []
+            for month in year_list.select("div.contest-archive-list-month"):
+                month_name = month.select("h3")[0].text
+                month_contest_list = []
+                for p in month.select("p"):
+                    date = p.select("span")[0].text
+                    link = p.select("a")[0].get("href")
+                    title = p.select("a")[0].text
+                    month_contest_list.append([date, link, title])
+                month_list.append([month_name, month_contest_list])
+            contest_list.append([year_name, month_list])
+
+        pagination = main.select("nav.pagination ul.pagination")[0]
+
+        return render_template(
+            "archives.html",
+            title=f"Contest Archives (Page {page})",
+            page=page,
+            contest_count=contest_count,
+            pagination=pagination,
+            contest_list=contest_list,
+        )
+
+    @app.route("/contest/<contest>/")
+    def route_contest(contest):
+        try:
+            data = urlopen(f"https://www.instructables.com/contest/{contest}/")
+        except HTTPError as e:
+            abort(e.code)
+
+        soup = BeautifulSoup(data.read().decode(), "html.parser")
+
+        title = soup.select('meta[property="og:title"]')[0].get("content")
+
+        body = soup.select("div#contest-wrapper")[0]
+
+        img = proxy(body.select("div#contest-masthead img")[0].get("src"))
+
+        entry_count = body.select("li.entries-nav-btn")[0].text.split(" ")[0]
+        prizes = body.select("li.prizes-nav-btn")[0].text.split(" ")[0]
+
+        info = body.select("div.contest-body-column-left")[0]
+        info.select("div#site-announcements-page")[0].decompose()
+        info.select("h3")[0].decompose()
+        info.select("div#contest-body-nav")[0].decompose()
+        info = str(info).replace("https://www.instructables.com", "/")
+
+        body.select("span.contest-entity-count")[0].text
+
+        entry_list = []
+        for entry in body.select("div.contest-entries-list div.contest-entries-list-ible"):
+            link = entry.a["href"]
+            entry_img = proxy(entry.select("a noscript img")[0].get("src"))
+            entry_title = entry.select("a.ible-title")[0].text
+            author = entry.select("div span.ible-author a")[0].text
+        author_link = entry.select("div span.ible-author a")[0].get("href")
+        channel = entry.select("div span.ible-channel a")[0].text
+        channel_link = entry.select("div span.ible-channel a")[0].get("href")
+        views = entry.select(".ible-views")[0].text
+
+        entry_list.append(
+            {
+                "link": link,
+                "entry_img": entry_img,
+                "entry_title": entry_title,
+                "author": author,
+                "author_link": author_link,
+                "channel": channel,
+                "channel_link": channel_link,
+                "views": views,
+            }
+        )
+
+        return render_template(
+            "contest.html",
+            title=title,
+            img=img,
+            entry_count=entry_count,
+            prizes=prizes,
+            info=info,
+            entry_list=entry_list,
+        )
+
+    @app.route("/contest/")
+    def route_contests():
+        try:
+            data = urlopen("https://www.instructables.com/contest/")
+        except HTTPError as e:
+            abort(e.code)
+
+        soup = BeautifulSoup(data.read().decode(), "html.parser")
+
+        contest_count = str(soup.select("p.contest-count")[0])
+
+        contests = []
+        for contest in soup.select("div#cur-contests div.row-fluid div.contest-banner"):
+            link = contest.select("div.contest-banner-inner a")[0].get("href")
+            img = proxy(contest.select("div.contest-banner-inner a img")[0].get("src"))
+            alt = contest.select("div.contest-banner-inner a img")[0].get("alt")
+            deadline = contest.select("span.contest-meta-deadline")[0].get("data-deadline")
+            prizes = contest.select("span.contest-meta-count")[0].text
+            entries = contest.select("span.contest-meta-count")[1].text
+
+            contests.append(
+                {
+                    "link": link,
+                    "img": img,
+                    "alt": alt,
+                    "deadline": deadline,
+                    "prizes": prizes,
+                    "entries": entries,
+                }
+            )
+
+        closed = []
+        for display in soup.select("div.contest-winner-display"):
+            link = display.select("div.contest-banner-inner a")[0].get("href")
+            img = proxy(display.select("div.contest-banner-inner a img")[0].get("src"))
+            alt = display.select("div.contest-banner-inner a img")[0].get("alt")
+            featured_items = []
+            for featured_item in display.select("ul.featured-items li"):
+                item_link = featured_item.select("div.ible-thumb a")[0].get("href")
+                item_img = proxy(featured_item.select("div.ible-thumb a img")[0].get("src"))
+                item_title = featured_item.select("a.title")[0].text
+                item_author = featured_item.select("a.author")[0].text
+                item_author_link = featured_item.select("a.author")[0].get("href")
+
+                featured_items.append(
+                    {
+                        "link": item_link,
+                        "img": item_img,
+                        "title": item_title,
+                        "author": item_author,
+                        "author_link": item_author_link,
+                    }
+                )
+            closed.append(
+                {"link": link, "img": img, "alt": alt, "featured_items": featured_items}
+            )
+
+        return render_template(
+            "contests.html",
+            title="Contests",
+            contest_count=contest_count,
+            contests=contests,
+            closed=closed,
+        )

structables-0.3.0/src/structables/routes/main.py

@@ -0,0 +1,362 @@
+from flask import render_template, abort
+from urllib.request import urlopen
+from urllib.error import HTTPError
+from bs4 import BeautifulSoup
+from urllib.parse import quote
+from werkzeug.exceptions import InternalServerError
+from traceback import print_exc
+import pathlib
+import json
+
+from ..utils.data import update_data
+from ..utils.helpers import explore_lists, proxy
+from .category import project_list
+
+
+def init_main_routes(app):
+    @app.route("/")
+    def route_explore():
+        try:
+            data = urlopen("https://www.instructables.com/")
+        except HTTPError as e:
+            abort(e.code)
+
+        soup = BeautifulSoup(data.read().decode(), "html.parser")
+
+        explore = soup.select(".home-content-explore-wrap")[0]
+
+        title = explore.select("h2")[0].text
+
+        circuits = explore_lists(
+            explore.select(".home-content-explore-category-circuits")[0]
+        )
+        workshop = explore_lists(
+            explore.select(".home-content-explore-category-workshop")[0]
+        )
+        craft = explore_lists(explore.select(".home-content-explore-category-craft")[0])
+        cooking = explore_lists(
+            explore.select(".home-content-explore-category-cooking")[0]
+        )
+        living = explore_lists(
+            explore.select(".home-content-explore-category-living")[0]
+        )
+        outside = explore_lists(
+            explore.select(".home-content-explore-category-outside")[0]
+        )
+        teachers = explore_lists(
+            explore.select(".home-content-explore-category-teachers")[0]
+        )
+
+        return render_template(
+            "index.html",
+            title=title,
+            sections=[
+                ("Circuits", "/circuits", circuits),
+                ("Workshop", "/workshop", workshop),
+                ("Craft", "/craft", craft),
+                ("Cooking", "/cooking", cooking),
+                ("Living", "/living", living),
+                ("Outside", "/outside", outside),
+                ("Teachers", "/teachers", teachers),
+            ],
+        )
+
+    @app.route("/sitemap/")
+    @app.route("/sitemap/<path:path>")
+    def route_sitemap(path=""):
+        try:
+            data = urlopen("https://www.instructables.com/sitemap/" + path)
+        except HTTPError as e:
+            abort(e.code)
+
+        soup = BeautifulSoup(data.read().decode(), "html.parser")
+
+        main = soup.select("div.sitemap-content")[0]
+
+        group_section = main.select("div.group-section")
+
+        if group_section:
+            groups = []
+            for group in group_section:
+                category = group.select("h2 a")[0].text
+                category_link = group.select("h2 a")[0].get("href")
+                channels = []
+                for li in group.select("ul.sitemap-listing li"):
+                    channel = li.a.text
+                    channel_link = li.a["href"]
+                    channels.append([channel, channel_link])
+                groups.append([category, category_link, channels])
+
+        else:
+            groups = []
+            channels = []
+            for li in main.select("ul.sitemap-listing li"):
+                channel = li.a.text
+                channel_link = li.a["href"]
+
+                if channel_link.startswith("https://"):
+                    channel_link = f'/{"/".join(channel_link.split("/")[3:])}'
+
+                channels.append([channel, channel_link])
+            groups.append(["", "", channels])
+
+        return render_template("sitemap.html", title="Sitemap", groups=groups)
+
+    @app.route("/<article>/")
+    def route_article(article):
+        try:
+            data = urlopen(
+                f"https://www.instructables.com/json-api/showInstructableModel?urlString={article}"
+            )
+            data = json.loads(data.read().decode())
+        except HTTPError as e:
+            abort(e.code)
+
+        try:
+            title = data["title"]
+            author = data["author"]["screenName"]
+            author_link = f"/member/{author}"
+            category = data["classifications"][0]["title"]
+            category_slug = data["classifications"][0]["name"]
+            category_link = f"/{category_slug}/"
+            channel = data["classifications"][0]["channels"][0]["title"]
+            channel_slug = data["classifications"][0]["channels"][0]["name"]
+            channel_link = f"/{category_slug}/{channel_slug}/"
+
+            views = data["views"]
+            favorites = data["favorites"]
+
+            if "steps" in data:
+                steps = []
+
+                if "supplies" in data:
+                    supplies = data["supplies"]
+
+                    supplies_files = []
+
+                    if "suppliesFiles" in data:
+                        supplies_files = data["suppliesFiles"]
+
+                    data["steps"].insert(
+                        1,
+                        {
+                            "title": "Supplies",
+                            "body": supplies,
+                            "files": supplies_files,
+                        },
+                    )
+
+                for step in data["steps"]:
+                    step_title = step["title"]
+
+                    step_imgs = []
+                    step_videos = []  # TODO: Check if this is still required
+                    step_iframes = []
+                    step_downloads = []
+
+                    for file in step["files"]:
+                        if file["image"] and "embedType" not in "file":
+                            step_imgs.append(
+                                {"src": proxy(file["downloadUrl"]), "alt": file["name"]}
+                            )
+
+                        elif not file["image"]:
+                            step_downloads.append(
+                                {
+                                    "src": proxy(file["downloadUrl"]),
+                                    "name": file["name"],
+                                }
+                            )
+
+                        else:  # Leaves us with embeds
+                            embed_code = file["embedHtmlCode"]
+                            soup = BeautifulSoup(embed_code, "html.parser")
+
+                            iframe = soup.select("iframe")[0]
+
+                            src = iframe.get("src")
+
+                            if src.startswith("https://content.instructables.com"):
+                                src = src.replace(
+                                    "https://content.instructables.com",
+                                    f"/proxy/?url={src}",
+                                )
+
+                            elif app.config["INVIDIOUS"] and src.startswith(
+                                "https://www.youtube.com"
+                            ):
+                                src = src.replace(
+                                    "https://www.youtube.com", app.config["INVIDIOUS"]
+                                )
+
+                            elif not app.config["UNSAFE"]:
+                                src = "/iframe/?url=" + quote(src)
+
+                            step_iframes.append(
+                                {
+                                    "src": src,
+                                    "width": file.get("width"),
+                                    "height": file.get("height"),
+                                }
+                            )
+
+                    step_text = step["body"]
+                    step_text = step_text.replace(
+                        "https://content.instructables.com",
+                        "/proxy/?url=https://content.instructables.com",
+                    )
+                    steps.append(
+                        {
+                            "title": step_title,
+                            "imgs": step_imgs,
+                            "text": step_text,
+                            "videos": step_videos,
+                            "iframes": step_iframes,
+                            "downloads": step_downloads,
+                        }
+                    )
+
+                comments_list = []
+                comment_count = 0
+
+                # TODO: Fix comments
+
+                # comments = body.select("section.discussion")[0]
+
+                # comment_count = comments.select("h2")[0].text
+                # comment_list = comments.select("div.posts")
+
+                # if comment_list != []:
+                #     comment_list = comment_list[0]
+                #     comments_list = []
+                #     replies_used = 0
+                #     for comment in comment_list.select(".post.js-comment:not(.reply)"):
+                #         comment_votes = comment.select(".votes")[0].text
+                #         comment_author_img_src = proxy(comment.select(".avatar a noscript img")[0].get("src"))
+                #         comment_author_img_alt = comment.select(".avatar a noscript img")[0].get("alt")
+                #         comment_author = comment.select(".posted-by a")[0].text
+                #         comment_author_link = comment.select(".posted-by a")[0].get("href")
+                #         comment_date = comment.select(".posted-by p.posted-date")[0].text
+                #         comment_text = comment.select("div.text p")[0]
+                #         comment_reply_count = comment.select("button.js-show-replies")
+                #         if comment_reply_count != []:
+                #             comment_reply_count = comment_reply_count[0].get("data-num-hidden")
+                #         else:
+                #             comment_reply_count = 0
+                #         reply_list = []
+                #         for index, reply in enumerate(comment_list.select(".post.js-comment:not(.reply) ~ .post.js-comment.reply.hide:has(~.post.js-comment:not(.reply))")[replies_used:int(comment_reply_count) + replies_used]):
+                #             reply_votes = reply.select(".votes")[0].text
+                #             reply_author_img_src = proxy(reply.select(".avatar a noscript img")[0].get("src"))
+                #             reply_author_img_alt = reply.select(".avatar a noscript img")[0].get("alt")
+                #             reply_author = reply.select(".posted-by a")[0].text
+                #             reply_author_link = reply.select(".posted-by a")[0].get("href")
+                #             reply_date = reply.select(".posted-by p.posted-date")[0].text
+                #             reply_text = reply.select("div.text p")[0]
+
+                #             reply_list.append([reply_votes, reply_author_img_src, reply_author_img_alt, reply_author, reply_author_link, reply_date, reply_text])
+                #         replies_used += 1
+
+                #         comments_list.append([comment_votes, comment_author_img_src, comment_author_img_alt, comment_author, comment_author_link, comment_date, comment_text, comment_reply_count, reply_list])
+                return render_template(
+                    "article.html",
+                    title=title,
+                    author=author,
+                    author_link=author_link,
+                    category=category,
+                    category_link=category_link,
+                    channel=channel,
+                    channel_link=channel_link,
+                    views=views,
+                    favorites=favorites,
+                    steps=steps,
+                    comment_count=comment_count,
+                    comments_list=comments_list,
+                    enumerate=enumerate,
+                )
+            else:
+                ## Collections
+                thumbnails = []
+                for thumbnail in data["instructables"]:
+                    text = thumbnail["title"]
+                    link = thumbnail["showUrl"]
+                    img = proxy(thumbnail["downloadUrl"])
+                    thumbnail_title = thumbnail["title"]
+                    thumbnail_author = thumbnail["author"]["screenName"]
+                    thumbnail_author_link = f"/member/{thumbnail_author}"
+                    thumbnail_channel = thumbnail["classifications"][0]["channels"][0][
+                        "title"
+                    ]
+                    thumbnail_category = thumbnail["classifications"][0]["title"]
+                    thumbnail_channel_link = (
+                        f"/{thumbnail_category}/{thumbnail_channel}"
+                    )
+
+                    thumbnails.append(
+                        {
+                            "text": text,
+                            "link": link,
+                            "img": img,
+                            "title": thumbnail_title,
+                            "author": thumbnail_author,
+                            "author_link": thumbnail_author_link,
+                            "channel": thumbnail_channel,
+                            "channel_link": thumbnail_channel_link,
+                        }
+                    )
+
+                return render_template(
+                    "collection.html",
+                    title=title,
+                    author=author,
+                    author_link=author_link,
+                    category=category,
+                    category_link=category_link,
+                    channel=channel,
+                    channel_link=channel_link,
+                    views=views,
+                    favorites=favorites,
+                    thumbnails=thumbnails,
+                )
+
+        except Exception:
+            print_exc()
+            raise InternalServerError()
+
+    @app.route("/search", methods=["POST", "GET"])
+    def route_search():
+        return project_list(app, "Search")
+
+    @app.route("/cron/")
+    def cron():
+        update_data(app)
+        return "OK"
+
+    @app.route("/privacypolicy/")
+    def privacypolicy():
+        content = "No privacy policy found."
+
+        try:
+            with (pathlib.Path(__file__).parent / "privacy.txt").open() as f:
+                content = f.read()
+        except OSError:
+            pass
+
+        return render_template(
+            "privacypolicy.html", title="Privacy Policy", content=content
+        )
+
+    @app.errorhandler(404)
+    def not_found(e):
+        return render_template("404.html"), 404
+
+    @app.errorhandler(400)
+    def bad_request(e):
+        return render_template("400.html"), 400
+
+    @app.errorhandler(429)
+    def too_many_requests(e):
+        return render_template("429.html"), 429
+
+    @app.errorhandler(500)
+    def internal_server_error(e):
+        return render_template("500.html"), 500