wikimore 0.1.0__tar.gz

wikimore-0.1.0/.gitignore

@@ -0,0 +1,4 @@
+venv/
+.venv/
+__pycache__/
+*.pyc

wikimore-0.1.0/.vscode/launch.json

@@ -0,0 +1,27 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python Debugger: Flask",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "flask",
+            "env": {
+                "FLASK_DEBUG": "1",
+            },
+            "args": [
+                "--app",
+                "wikimore",
+                "run",
+                "--no-debugger",
+                "--no-reload",
+                "--port=8109"
+            ],
+            "jinja": true,
+            "autoStartBrowser": false
+        }
+    ]
+}

wikimore-0.1.0/.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "files.associations": {
+        "*.html": "jinja-html"
+    }
+}

wikimore-0.1.0/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2024 Private.coffee Team <support@private.coffee>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

wikimore-0.1.0/PKG-INFO

@@ -0,0 +1,121 @@
+Metadata-Version: 2.3
+Name: wikimore
+Version: 0.1.0
+Summary: A simple frontend for Wikimedia wikis
+Project-URL: Homepage, https://git.private.coffee/privatecoffee/wikimore
+Project-URL: Bug Tracker, https://git.private.coffee/privatecoffee/wikimore/issues
+Project-URL: Source Code, https://git.private.coffee/privatecoffee/wikimore
+Author-email: "Private.coffee Team" <support@private.coffee>
+License: Copyright (c) 2024 Private.coffee Team <support@private.coffee>
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Classifier: License :: OSI Approved :: GNU Affero General Public License v3
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Requires-Dist: bs4
+Requires-Dist: flask
+Description-Content-Type: text/markdown
+
+# Wikimore - A simple frontend for Wikimedia projects
+
+[![Support Private.coffee!](https://shields.private.coffee/badge/private.coffee-support%20us!-pink?logo=coffeescript)](https://private.coffee)
+[![Matrix](https://shields.private.coffee/badge/Matrix-join%20us!-blue?logo=matrix)](https://matrix.pcof.fi/#/#wikimore:private.coffee)
+[![PyPI](https://shields.private.coffee/pypi/v/wikimore)](https://pypi.org/project/wikimore/)
+[![PyPI - Python Version](https://shields.private.coffee/pypi/pyversions/wikimore)](https://pypi.org/project/wikimore/)
+[![PyPI - License](https://shields.private.coffee/pypi/l/wikimore)](https://pypi.org/project/wikimore/)
+[![Latest Git Commit](https://shields.private.coffee/gitea/last-commit/privatecoffee/wikimore?gitea_url=https://git.private.coffee)](https://git.private.coffee/privatecoffee/wikimore)
+
+Wikimore is a simple frontend for Wikimedia projects. It uses the MediaWiki API to fetch data from Wikimedia projects and display it in a user-friendly way. It is built using Flask.
+
+This project is still in development and more features will be added in the future. It is useful for anyone who wants to access Wikimedia projects with a more basic frontend, or to provide access to Wikimedia projects to users who cannot access them directly, for example due to state censorship.
+
+## Features
+
+- Supports all Wikimedia projects in all languages
+- Search functionality
+- Proxy support for Wikimedia images
+
+## Instances
+
+| URL                                                         | Provided by                               | Country | Comments |
+| ----------------------------------------------------------- | ----------------------------------------- | ------- | -------- |
+| [wikimore.private.coffee](https://wikimore.private.coffee/) | [Private.coffee](https://private.coffee/) | Austria |          |
+
+If you operate a public instance of Wikimore and would like to have it listed here, please open an issue or a pull request.
+
+## Installation
+
+### Production
+
+1. Create a virtual environment and activate it
+
+```bash
+python3 -m venv venv
+source venv/bin/activate
+```
+
+2. Install the package from PyPI
+
+```bash
+pip install wikimore
+```
+
+3. Run the application
+
+```bash
+wikimore
+```
+
+4. Open your browser and navigate to `http://localhost:8109`
+
+## Development
+
+1. Clone the repository
+
+```bash
+git clone https://git.private.coffee/privatecoffee/wikimore.git
+cd wikimore
+```
+
+2. Create a virtual environment and activate it
+
+```bash
+python3 -m venv venv
+source venv/bin/activate
+```
+
+3. Install the package in editable mode
+
+```bash
+pip install -e .
+```
+
+4. Run the application
+
+```bash
+flask --app wikimore run
+```
+
+5. Open your browser and navigate to `http://localhost:5000`
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

wikimore-0.1.0/README.md

@@ -0,0 +1,85 @@
+# Wikimore - A simple frontend for Wikimedia projects
+
+[![Support Private.coffee!](https://shields.private.coffee/badge/private.coffee-support%20us!-pink?logo=coffeescript)](https://private.coffee)
+[![Matrix](https://shields.private.coffee/badge/Matrix-join%20us!-blue?logo=matrix)](https://matrix.pcof.fi/#/#wikimore:private.coffee)
+[![PyPI](https://shields.private.coffee/pypi/v/wikimore)](https://pypi.org/project/wikimore/)
+[![PyPI - Python Version](https://shields.private.coffee/pypi/pyversions/wikimore)](https://pypi.org/project/wikimore/)
+[![PyPI - License](https://shields.private.coffee/pypi/l/wikimore)](https://pypi.org/project/wikimore/)
+[![Latest Git Commit](https://shields.private.coffee/gitea/last-commit/privatecoffee/wikimore?gitea_url=https://git.private.coffee)](https://git.private.coffee/privatecoffee/wikimore)
+
+Wikimore is a simple frontend for Wikimedia projects. It uses the MediaWiki API to fetch data from Wikimedia projects and display it in a user-friendly way. It is built using Flask.
+
+This project is still in development and more features will be added in the future. It is useful for anyone who wants to access Wikimedia projects with a more basic frontend, or to provide access to Wikimedia projects to users who cannot access them directly, for example due to state censorship.
+
+## Features
+
+- Supports all Wikimedia projects in all languages
+- Search functionality
+- Proxy support for Wikimedia images
+
+## Instances
+
+| URL                                                         | Provided by                               | Country | Comments |
+| ----------------------------------------------------------- | ----------------------------------------- | ------- | -------- |
+| [wikimore.private.coffee](https://wikimore.private.coffee/) | [Private.coffee](https://private.coffee/) | Austria |          |
+
+If you operate a public instance of Wikimore and would like to have it listed here, please open an issue or a pull request.
+
+## Installation
+
+### Production
+
+1. Create a virtual environment and activate it
+
+```bash
+python3 -m venv venv
+source venv/bin/activate
+```
+
+2. Install the package from PyPI
+
+```bash
+pip install wikimore
+```
+
+3. Run the application
+
+```bash
+wikimore
+```
+
+4. Open your browser and navigate to `http://localhost:8109`
+
+## Development
+
+1. Clone the repository
+
+```bash
+git clone https://git.private.coffee/privatecoffee/wikimore.git
+cd wikimore
+```
+
+2. Create a virtual environment and activate it
+
+```bash
+python3 -m venv venv
+source venv/bin/activate
+```
+
+3. Install the package in editable mode
+
+```bash
+pip install -e .
+```
+
+4. Run the application
+
+```bash
+flask --app wikimore run
+```
+
+5. Open your browser and navigate to `http://localhost:5000`
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

wikimore-0.1.0/pyproject.toml

@@ -0,0 +1,26 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "wikimore"
+version = "0.1.0"
+authors = [{ name = "Private.coffee Team", email = "support@private.coffee" }]
+description = "A simple frontend for Wikimedia wikis"
+readme = "README.md"
+license = { file = "LICENSE" }
+requires-python = ">=3.10"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: GNU Affero General Public License v3",
+    "Operating System :: OS Independent",
+]
+dependencies = ["flask", "bs4"]
+
+[project.scripts]
+wikimore = "wikimore.app:main"
+
+[project.urls]
+"Homepage" = "https://git.private.coffee/privatecoffee/wikimore"
+"Bug Tracker" = "https://git.private.coffee/privatecoffee/wikimore/issues"
+"Source Code" = "https://git.private.coffee/privatecoffee/wikimore"

wikimore-0.1.0/requirements.txt

@@ -0,0 +1,2 @@
+flask
+bs4

wikimore-0.1.0/src/wikimore/__init__.py

@@ -0,0 +1,3 @@
+from .app import app
+
+__all__ = ["app"]

wikimore-0.1.0/src/wikimore/app.py

@@ -0,0 +1,223 @@
+from flask import Flask, render_template, request, redirect, url_for
+import urllib.request
+from urllib.parse import urlencode, urlparse
+from html import escape
+import json
+import os
+import logging
+from bs4 import BeautifulSoup
+
+app = Flask(__name__)
+
+logging.basicConfig(level=logging.DEBUG)
+
+logger = logging.getLogger(__name__)
+
+handler = logging.StreamHandler()
+handler.setLevel(logging.DEBUG)
+logger.addHandler(handler)
+
+formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+handler.setFormatter(formatter)
+
+
+def get_wikimedia_projects():
+    url = "https://meta.wikimedia.org/w/api.php?action=sitematrix&format=json"
+    with urllib.request.urlopen(url) as response:
+        data = json.loads(response.read().decode())
+
+    projects = {}
+    languages = {}
+
+    for key, value in data["sitematrix"].items():
+        if key.isdigit():
+            language = value["name"]
+            language_code = value["code"]
+            language_projects = {}
+
+            for site in value["site"]:
+                language_projects[site["code"]] = site["url"]
+
+                if language_code == "en":
+                    projects[site["code"]] = site["sitename"]
+
+            if language_projects:
+                languages[language_code] = {
+                    "projects": language_projects,
+                    "name": language,
+                }
+
+    return projects, languages
+
+
+def get_proxy_url(url):
+    if url.startswith("//"):
+        url = "https:" + url
+
+    if not url.startswith("https://upload.wikimedia.org/"):
+        logger.debug(f"Not generating proxy URL for {url}")
+        return url
+
+    logger.debug(f"Generating proxy URL for {url}")
+    return f"/proxy?{urlencode({'url': url})}"
+
+
+@app.route("/proxy")
+def proxy():
+    url = request.args.get("url")
+
+    if not url or not url.startswith("https://upload.wikimedia.org/"):
+        logger.error(f"Invalid URL for proxying: {url}")
+        return "Invalid URL"
+
+    logger.debug(f"Proxying {url}")
+
+    with urllib.request.urlopen(url) as response:
+        data = response.read()
+    return data
+
+
+@app.route("/")
+def home():
+    return render_template(
+        "home.html",
+        wikimedia_projects=app.wikimedia_projects,
+        languages=app.languages,
+    )
+
+
+@app.route("/search", methods=["GET", "POST"])
+def search():
+    if request.method == "POST":
+        query = request.form["query"]
+        lang = request.form["lang"]
+        project = request.form["project"]
+        return redirect(
+            url_for("search_results", project=project, lang=lang, query=query)
+        )
+    return render_template(
+        "search.html",
+        wikimedia_projects=app.wikimedia_projects,
+        languages=app.languages,
+    )
+
+
+@app.route("/<project>/<lang>/wiki/<title>")
+def wiki_article(project, lang, title):
+    language_projects = app.languages.get(lang, {}).get("projects", {})
+    base_url = language_projects.get(project)
+
+    if not base_url:
+        return "Invalid language or project"
+
+    logger.debug(f"Fetching {title} from {base_url}")
+
+    url = f"{base_url}/w/api.php?action=query&format=json&titles={escape(title.replace(" ", "_"), True)}&prop=revisions&rvprop=content&rvparse=1"
+    with urllib.request.urlopen(url) as response:
+        data = json.loads(response.read().decode())
+    pages = data["query"]["pages"]
+    article_html = next(iter(pages.values()))["revisions"][0]["*"]
+
+    soup = BeautifulSoup(article_html, "html.parser")
+
+    for a in soup.find_all("a", href=True):
+        href = a["href"]
+
+        if href.startswith("/wiki/"):
+            a["href"] = f"/{project}/{lang}{href}"
+
+        elif href.startswith("//") or href.startswith("https://"):
+            parts = urlparse(href)
+
+            target_domain = parts.netloc
+            path_parts = parts.path.split("/")
+
+            if len(path_parts) > 4:
+                target_title = "/".join(path_parts[4:])
+                target_lang = target_domain.split(".")[0]
+
+                found = False
+                for language, language_projects in app.languages.items():
+                    for project_name, project_url in language_projects.items():
+                        if target_domain == project_url:
+                            a["href"] = (
+                                f"/{project_name}/{target_lang}/wiki/{target_title}"
+                            )
+                            found = True
+                            break
+                    if found:
+                        break
+
+    for span in soup.find_all("span", class_="mw-editsection"):
+        span.decompose()
+
+    for style in soup.find_all("style"):
+        style.decompose()
+
+    for img in soup.find_all("img"):
+        img["src"] = get_proxy_url(img["src"])
+
+    for source in soup.find_all("source"):
+        source["src"] = get_proxy_url(source["src"])
+
+    for video in soup.find_all("video"):
+        video["poster"] = get_proxy_url(video["poster"])
+
+    for li in soup.find_all("li"):
+        if any(cls in li.get("class", []) for cls in ["nv-view", "nv-talk", "nv-edit"]):
+            li.decompose()
+
+    processed_html = str(soup)
+    return render_template(
+        "article.html",
+        title=title.replace("_", " "),
+        content=processed_html,
+        wikimedia_projects=app.wikimedia_projects,
+        languages=app.languages,
+    )
+
+
+@app.route("/<project>/<lang>/search/<query>")
+def search_results(project, lang, query):
+    language_projects = app.languages.get(lang, {}).get("projects", {})
+    base_url = language_projects.get(project)
+
+    if not base_url:
+        return "Invalid language or project"
+
+    logger.debug(f"Searching {base_url} for {query}")
+
+    url = f"{base_url}/w/api.php?action=query&format=json&list=search&srsearch={query}"
+    with urllib.request.urlopen(url) as response:
+        data = json.loads(response.read().decode())
+    search_results = data["query"]["search"]
+    return render_template(
+        "search_results.html",
+        query=query,
+        search_results=search_results,
+        project=project,
+        lang=lang,
+        wikimedia_projects=app.wikimedia_projects,
+        languages=app.languages,
+    )
+
+
+@app.route("/<project>/<lang>/wiki/Special:Search/<query>")
+def search_redirect(project, lang, query):
+    return redirect(url_for("search_results", project=project, lang=lang, query=query))
+
+
+app.wikimedia_projects, app.languages = get_wikimedia_projects()
+
+logger.debug(
+    f"Loaded {len(app.wikimedia_projects)} Wikimedia projects and {len(app.languages)} languages"
+)
+
+def main():
+    port = int(os.environ.get("PORT", 8109))
+    host = os.environ.get("HOST", "0.0.0.0")
+    debug = os.environ.get("DEBUG", False)
+    app.run(port=port, host=host, debug=debug)
+
+if __name__ == "__main__":
+    main()

wikimore-0.1.0/src/wikimore/templates/article.html

@@ -0,0 +1,6 @@
+{% extends "base.html" %}
+
+{% block content %}
+    <h1>{{ title }}</h1>
+    <p>{{ content|safe }}</p>
+{% endblock %}

wikimore-0.1.0/src/wikimore/templates/base.html

@@ -0,0 +1,110 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>{{ title }}{% if title %} &dash; {% endif %}Wikimore</title>
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            margin: 0;
+            padding: 0;
+            background-color: #f8f9fa;
+        }
+        #header {
+            background-color: #ffffff;
+            border-bottom: 1px solid #a2a9b1;
+            padding: 10px;
+            display: flex;
+            align-items: center;
+            justify-content: space-between;
+        }
+        #header h1 {
+            margin: 0;
+            font-size: 1.5em;
+        }
+        #header h1 a {
+            text-decoration: none;
+            color: #333;
+        }
+        #search-form {
+            margin: 0;
+            padding: 0;
+            display: flex;
+            align-items: center;
+        }
+        #search-form input[type="text"] {
+            width: 300px;
+            padding: 5px;
+        }
+        #search-form select {
+            padding: 5px;
+        }
+        #search-form button {
+            padding: 5px 10px;
+        }
+        #content {
+            padding: 20px;
+        }
+        h1 {
+            font-size: 2em;
+            padding-bottom: 10px;
+            margin-bottom: 20px;
+        }
+        .sidebar {
+            float: right;
+            width: 250px;
+            margin-left: 20px;
+            background-color: #ffffff;
+            border: 1px solid #a2a9b1;
+            padding: 10px;
+            box-sizing: border-box;
+        }
+        .toc {
+            list-style-type: none;
+            padding: 0;
+        }
+        .toc li {
+            margin-bottom: 5px;
+        }
+        .toc a {
+            text-decoration: none;
+            color: #333;
+        }
+        .toc a:hover {
+            text-decoration: underline;
+        }
+        #footer {
+            background-color: #ffffff;
+            border-top: 1px solid #a2a9b1;
+            padding: 10px;
+            text-align: center;
+        }
+    </style>
+</head>
+<body>
+    <div id="header">
+        <h1><a href="/">Wikimore</a></h1>
+        <form id="search-form" action="{{ url_for('search') }}" method="post">
+            <select name="project" id="project">
+            {% for key, value in wikimedia_projects.items() %}
+                <option value="{{ key }}">{{ value }}</option>
+            {% endfor %}
+            </select>
+            <select name="lang" id="lang">
+            {% for key, value in languages.items() %}
+                <option {% if key == "en" %}selected="selected"{% endif %} value="{{ key }}">{{ value["name"] }}</option>
+            {% endfor %}
+            </select>
+            <input type="text" name="query" id="query" placeholder="Search Wikipedia" required>
+            <button type="submit">Search</button>
+        </form>
+    </div>
+    <div id="content">
+        {% block content %}{% endblock %}
+    </div>
+    <div id="footer">
+        <p>Brought to you by <a href="https://git.private.coffee/privatecoffee/wikimore">Wikimore</a>, a <a href="https://private.coffee">Private.coffee</a> project</p>
+    </div>
+</body>
+</html>

wikimore-0.1.0/src/wikimore/templates/home.html

@@ -0,0 +1,6 @@
+{% extends "base.html" %}
+
+{% block content %}
+    <h1>Welcome to Wikimore</h1>
+    <p>Use the search form above to find articles on Wikipedia.</p>
+{% endblock %}

wikimore-0.1.0/src/wikimore/templates/search_results.html

@@ -0,0 +1,13 @@
+{% extends "base.html" %}
+
+{% block content %}
+    <h1>Search Results for "{{ query }}"</h1>
+    <ul>
+        {% for result in search_results %}
+        <li>
+            <a href="{{ url_for('wiki_article', project=project, lang=lang, title=result['title']) }}">{{ result['title'] }}</a>
+            <p>{{ result['snippet']|safe }}</p>
+        </li>
+        {% endfor %}
+    </ul>
+{% endblock %}