qcanvas 0.0.5.6a0__py3-none-any.whl → 1.0.3.post0__py3-none-any.whl

qcanvas/util/course_indexer/resource_helpers.py

@@ -1,191 +0,0 @@
-import asyncio
-import logging
-from dataclasses import dataclass
-from typing import Sequence
-
-from bs4 import Tag, BeautifulSoup
-from sqlalchemy.ext.asyncio import AsyncSession
-
-import qcanvas.db as db
-from qcanvas.util.link_scanner import ResourceScanner
-from qcanvas.util.progress_reporter import ProgressReporter
-from qcanvas.util.task_pool import TaskPool
-
-_logger = logging.getLogger(__name__)
-
-resource_elements = ["a", "iframe", "img"]
-
-
-# todo could probably just use the database types directly now
-@dataclass
-class TransientResourceToPageLink:
-    """
-    Represents a temporary link between a page and a resource that will be added to the database soon.
-    """
-    page_id: str
-    resource_id: str
-
-    def __hash__(self):
-        return hash(self.page_id) ^ hash(self.resource_id)
-
-
-async def create_module_item_resource_relations(relations: Sequence[TransientResourceToPageLink],
-                                                session: AsyncSession):
-    """
-    Creates a link between module items/pages and resources found on those pages
-    """
-    for relation in relations:
-        existing_relation = await session.get(
-            db.ResourceToModuleItemAssociation,
-            (relation.page_id, relation.resource_id)
-        )
-
-        if existing_relation is None:
-            session.add(
-                db.ResourceToModuleItemAssociation(
-                    module_item_id=relation.page_id,
-                    resource_id=relation.resource_id
-                )
-            )
-
-
-async def create_assignment_resource_relations(relations: Sequence[TransientResourceToPageLink], session: AsyncSession):
-    """
-    Turns temporary TransientResourceToPageLink into a persistent relation in the database
-    """
-    for relation in relations:
-        if await session.get(db.ResourceToAssignmentAssociation, (relation.page_id, relation.resource_id)) is None:
-            session.add(
-                db.ResourceToAssignmentAssociation(
-                    assignment_id=relation.page_id,
-                    resource_id=relation.resource_id
-                )
-            )
-
-
-# todo change resource system to think of resources as links on a page with a shallow id (that may be the same as the deep id) which links to one or more deep ids
-async def find_resources_in_pages(link_scanners: Sequence[ResourceScanner], resource_pool: TaskPool[db.Resource],
-                                  items: Sequence[db.PageLike], progress_reporter: ProgressReporter) -> list[
-    TransientResourceToPageLink]:
-    """
-    Produce a list of resource to page links from resources extracted from the specified `items` using `link_scanners`.
-    Extracted resources will be added to `resource_pool`
-    """
-    progress = progress_reporter.section("Indexing resources", len(items))
-    tasks = []
-
-    for item in items:
-        # Assignment descriptions may be null. Avoid creating extra tasks by checking here
-        if item.content is None:
-            progress.increment_progress()
-            continue
-
-        # extract resources from the page
-        task = asyncio.create_task(_extract_resources_from_page(link_scanners, resource_pool, item))
-        task.add_done_callback(progress.increment_progress)
-        tasks.append(task)
-
-    if len(tasks) > 0:
-        # Wait for all tasks to complete
-        await asyncio.wait(tasks)
-
-        result = []
-        # Flatten the array of results
-        for task in tasks:
-            result.extend(task.result())
-
-        return result
-    else:
-        return []
-
-
-async def _extract_resources_from_page(link_scanners: Sequence[ResourceScanner], resource_pool: TaskPool[db.Resource],
-                                       page: db.PageLike) -> list[TransientResourceToPageLink]:
-    """
-    Extracts any detected resource links from the specified page and then uses `link_scanners` to extract information
-    about which is then added to the `resource_pool`.
-
-    Returns
-    -------
-    list
-        A list of resource to page links for any resources found on this page.
-    """
-    _logger.debug(f"Scanning %s %s for files", page.id, page.name)
-    tasks = []
-
-    # Extract iframes, hyperlinks, etc from the page
-    for link in _scan_page_for_links(page):
-        tasks.append(asyncio.create_task(_process_link(link_scanners, resource_pool, link, page.course_id)))
-
-    if len(tasks) > 0:
-        # Wait for all tasks to complete
-        await asyncio.wait(tasks)
-
-        task_results = [task.result() for task in tasks]
-
-        # Convert every non-null result in the task results to a resource page link and return it
-        return [
-            TransientResourceToPageLink(page_id=page.id, resource_id=result.id)
-            for result in task_results if result is not None
-        ]
-    else:
-        return []
-
-
-def _scan_page_for_links(page: db.PageLike) -> list[Tag]:
-    """
-    Extracts (potential) resource elements from a PageLike object
-    """
-    soup = BeautifulSoup(page.content, 'html.parser')
-    return list(soup.find_all(resource_elements))
-
-
-async def _process_link(link_scanners: Sequence[ResourceScanner], resource_pool: TaskPool[db.Resource], link: Tag,
-                        course_id: str) -> db.Resource | None:
-    """
-    Iterates over `link_scanners` to find one that will accept `link`, then uses it to fetch resource information and
-     adds it to the `resource_pool`.
-    If no scanner accepts the link then None is returned.
-    """
-    for scanner in link_scanners:
-        if scanner.accepts_link(link):
-            resource_id = scanner.extract_id(link)
-
-            return await resource_pool.submit(
-                f"{scanner.name}:{resource_id}",  # match the format used by the resource id
-                lambda: _extract_file_info(link, scanner, resource_id, course_id)
-            )
-
-    return None
-
-
-async def _extract_file_info(link: Tag, scanner: ResourceScanner, resource_id: str,
-                             course_id: str) -> db.Resource | None:
-    """
-    Extracts file info from `link` using `scanner` and assigns the course_id to the resulting resource.
-
-    Parameters
-    ----------
-    link
-        The html element to scan
-    scanner
-        The scanner to process the link with
-    course_id
-        The id of the course the file belongs to
-    Returns
-    -------
-    db.Resource
-        The resource if the link was processed successfully.
-    None
-        If processing failed
-    """
-    try:
-        _logger.debug(f"Fetching info for file %s with scanner %s", scanner.extract_id(link), scanner.name)
-
-        result = await scanner.extract_resource(link, resource_id)
-        result.id = f"{scanner.name}:{result.id}"  # Prefix the scanner name to prevent resources from different sites potentially clashing
-        result.course_id = course_id
-        return result
-    except BaseException as e:
-        _logger.error(f"Failed to retrieve info for file id %s: %s", f"{scanner.name}:{resource_id}", str(e))
-        return None

qcanvas/util/download_pool.py

@@ -1,58 +0,0 @@
-import asyncio
-from typing import Callable, Any
-
-from PySide6.QtCore import QObject, Signal
-
-from qcanvas.util.task_pool import TaskPool
-
-
-class DownloadPool(TaskPool[None], QObject):
-    """
-    Like the TaskPool, but uses a queue (a channel in kotlin terms) in the task to report download progress.
-    When using .submit(), it is EXPECTED that the lambda returns a function that uses yield to signal progress, e.g:
-    ```
-    pool.submit(my_task_id, lambda: my_func())
-
-    async def my_func(channel: asyncio.Queue):
-        for i in range(0, 10):
-            await slow_thing()
-            yield (i / 10) # can be whatever format/type you want
-    ```
-    """
-    download_progress_updated = Signal(object, Any)
-    download_failed = Signal(object)
-    download_finished = Signal(object)
-
-    def __init__(self, parent: QObject | None = None):
-        QObject.__init__(self, parent)
-        TaskPool.__init__(self)
-
-    async def _handle_task(self, func: Callable, task_id: object, event: asyncio.Event, func_args: dict):
-        sem = self._semaphore
-
-        try:
-            # Consume progress updates 'yield'ed from the function
-            async for progress in func(**func_args):
-                # Fire the download progress update signal
-                self.download_progress_updated.emit(task_id, progress)
-        except BaseException as e:
-            # Try to maintain integrity when a task fails
-            async with sem:
-                # Release anything else waiting for this task
-                event.set()
-                # Remove the record from the results map
-                del self._results[task_id]
-
-            # Emit failure signal and rethrow
-            self.download_failed.emit(task_id)
-            raise e
-
-        async with sem:
-            self._logger.debug("Task %s finished.", task_id)
-            # Record this task as done
-            self._results[task_id] = None
-            event.set()
-
-        self.download_finished.emit(task_id)
-
-        return None

qcanvas/util/helpers/canvas_sanitiser.py

@@ -1,47 +0,0 @@
-"""
-I dedicate this file to removing the random crap canvas puts in its data.
-Like random NBSPs. Gotta love those fuckers.
-Thanks instructure btw for you state of the art WYSIWYG dogshit editor.
-"""
-from bs4 import BeautifulSoup
-
-NBSP = " "
-
-
-def remove_garbage_from_title(smelly_canvas_title: str) -> str:
-    """
-    Removes trailing tabs, spaces and NBSPs from smelly canvas titles.
-    Parameters
-    ----------
-    smelly_canvas_title
-
-    Returns
-    -------
-    str
-        Clean title that is not smelly and has no NBSPs.
-    """
-    return (smelly_canvas_title
-            .strip(f"\t {NBSP}")  # remove trailing garbage
-            .replace(NBSP, " ")  # remove any other NBSPs
-            )
-
-
-def remove_stylesheets_from_html(smelly_html: str) -> str:
-    """
-    Removes all stylesheet links from `smelly_html`.
-
-    Parameters
-    ----------
-    smelly_html
-        The html to remove style sheets from
-    Returns
-    -------
-        The non-smelly html with all stylesheet links removed
-    """
-    bs = BeautifulSoup(smelly_html, "html.parser")
-
-    # remove links
-    for ele in bs.find_all("link", {"rel": "stylesheet"}):
-        ele.decompose()
-
-    return str(bs)

qcanvas/util/helpers/file_icon_helper.py

@@ -1,34 +0,0 @@
-from PySide6.QtCore import QMimeDatabase
-from PySide6.QtGui import QIcon
-from PySide6.QtWidgets import QApplication, QStyle
-
-_mime_database = QMimeDatabase()
-_default_icon = None
-
-
-def icon_for_filename(file_name: str) -> QIcon:
-    """
-    Gets the icon for a filename, based on its extension
-    Parameters
-    ----------
-    file_name
-        The name of the file
-    Returns
-    -------
-    QIcon
-        The icon for the file
-    """
-    global _default_icon
-
-    for mime_type in _mime_database.mimeTypesForFileName(file_name):
-        icon = QIcon.fromTheme(mime_type.iconName())
-
-        # Return the appropriate icon if it's found
-        if not icon.isNull():
-            return icon
-
-    # Cache the default icon, used when the icon for a file is unknown/not found
-    if _default_icon is None:
-        _default_icon = QApplication.style().standardIcon(QStyle.StandardPixmap.SP_FileIcon)
-
-    return _default_icon

qcanvas/util/helpers/qaction_helper.py

@@ -1,25 +0,0 @@
-from typing import Any
-
-from PySide6.QtGui import QAction, QKeySequence
-
-
-def create_qaction(name: str, shortcut: QKeySequence | None = None, parent: Any = None, triggered: Any = None,
-                   checkable: bool | None = None, checked: bool | None = None) -> QAction:
-    action = QAction(name)
-
-    if shortcut is not None:
-        action.setShortcut(shortcut)
-
-    if parent is not None:
-        action.setParent(parent)
-
-    if triggered is not None:
-        action.triggered.connect(triggered)
-
-    if checkable is not None:
-        action.setCheckable(checkable)
-
-        if checked is not None:
-            action.setChecked(checked)
-
-    return action

qcanvas/util/helpers/theme_helper.py

@@ -1,45 +0,0 @@
-from datetime import datetime
-
-import qdarktheme
-
-from qcanvas.util.app_settings import settings
-
-
-def apply_selected_theme() -> None:
-    """
-    Applies the selected theme from the app's settings
-    """
-    if settings.theme != "native":
-        qdarktheme.setup_theme(
-            settings.theme,
-            custom_colors=_get_colours()
-        )
-
-
-red_theme = {
-    "primary": "e21d31",
-    "[light]": {"foreground": "480910", "background": "fcf8f8"},
-    "[dark]": {"foreground": "fbdfe2", "background": "231f1f"}
-}
-
-
-def _get_colours() -> dict:
-    now = datetime.now()
-
-    if now.year >= 2025:
-        print("I certainly hope not")
-
-    if now.month == 3 and now.day == 17:
-        # And this is on the weekend...
-        return {"primary": "08ff00"}
-    elif now.month == 2 and now.day == 14:
-        print("Why are you looking at canvas? Don't you have something better to do?")
-
-        # Nobody will ever see this because uni starts around the 20th
-        # Too bad, I kinda liked the theme
-        return red_theme
-    elif now.month == 8 and now.day == 20:
-        # Some random day... I just wanted to see the red theme
-        return red_theme
-    else:
-        return {"primary": "FF804F"}

qcanvas/util/link_scanner/__init__.py

@@ -1,2 +0,0 @@
-from .canvas_link_scanner import CanvasFileScanner
-from .resource_scanner import ResourceScanner

qcanvas/util/link_scanner/canvas_link_scanner.py

@@ -1,41 +0,0 @@
-from bs4 import Tag
-from httpx import URL
-
-from qcanvas import db as db
-from qcanvas.net.canvas import CanvasClient
-from qcanvas.util.link_scanner.resource_scanner import ResourceScanner
-
-canvas_resource_id_prefix = "canvas_file"
-
-
-class CanvasFileScanner(ResourceScanner):
-    _canvas_client: CanvasClient
-
-    def __init__(self, canvas_client: CanvasClient):
-        self._canvas_client = canvas_client
-
-    def accepts_link(self, link: Tag) -> bool:
-        if link.name not in ["a", "img"]:
-            return False
-
-        return "data-api-returntype" in link.attrs.keys() and link["data-api-returntype"] == "File"
-
-    async def extract_resource(self, link: Tag, file_id: str) -> db.Resource:
-        return db.convert_legacy_file(await self._canvas_client.get_file_from_endpoint(link["data-api-endpoint"]))
-
-    def extract_id(self, link: Tag) -> str:
-        # https://canvas.newcastle.edu.au/courses/27716/files/5975585/...
-        # --------------------------------- Extract this part ^^^^^^^
-        return URL(link["data-api-endpoint"]).path.rsplit('/', 2)[-1]
-
-    async def download(self, resource):
-        path = resource.download_location
-        path.parent.mkdir(parents=True, exist_ok=True)
-
-        with open(path, "wb") as file:
-            async for progress in self._canvas_client.download_file(resource, file):
-                yield progress
-
-    @property
-    def name(self) -> str:
-        return canvas_resource_id_prefix

qcanvas/util/link_scanner/canvas_media_object_scanner.py

@@ -1,60 +0,0 @@
-import json
-from typing import Any
-
-from bs4 import Tag, BeautifulSoup
-from httpx import AsyncClient
-
-from qcanvas import db as db
-from qcanvas.util.link_scanner import ResourceScanner
-
-
-class CanvasMediaObjectScanner(ResourceScanner):
-
-    def __init__(self, client: AsyncClient):
-        self.client = client
-
-    @property
-    def name(self) -> str:
-        return "canvas_media_object"
-
-    def accepts_link(self, link: Tag) -> bool:
-        return (
-                link.name == "iframe"
-                and "data-media-type" in link.attrs.keys()
-                and link.attrs["data-media-type"] == "video"
-        )
-
-    async def extract_resource(self, link: Tag, file_id: str) -> db.Resource:
-        # Get the page for the embedded player (I could not find another way to get the needed data from canvas)
-        response = (await self.client.get(link.attrs["src"])).text
-        # Parse the HTML response
-        doc = BeautifulSoup(response, "html.parser")
-        media_data: None | dict[str, Any] = None
-
-        # Find all script tags (one of them has the data we are interested in)
-        for script_tag in doc.find_all("script", {}):
-            body = script_tag.text.strip()
-
-            # If the tag content starts with this then it has the data we want
-            if "INST = {" in body:
-                # Find the data that we are interested in (is on a line that starts with "ENV = ")
-                line: str = next(filter(lambda x: x.strip().startswith("ENV ="), script_tag.text.splitlines()))
-                # Parse the json embedded in the script tag
-                media_data = json.loads(line.lstrip("ENV = ").rstrip(";"))["media_object"]
-                break
-
-        if media_data is None:
-            raise Exception("Could not extract media info")
-
-        # The highest quality stream is the first
-        media_source = media_data["media_sources"][0]
-
-        return db.Resource(
-            id=file_id,
-            url=media_source["src"],
-            file_name=media_data["title"],
-            file_size=int(media_source["size"]) * 1024  # Seems to be recorded in KiB, not bytes
-        )
-
-    def extract_id(self, link: Tag) -> str:
-        return link.attrs["data-media-id"]

qcanvas/util/link_scanner/dropbox_scanner.py

@@ -1,68 +0,0 @@
-import httpx
-from bs4 import Tag
-from httpx import URL
-
-from qcanvas import db as db
-from qcanvas.util.link_scanner import ResourceScanner
-
-
-# from httpx import URL
-
-
-def parse_content_disposition(header: str) -> dict[str, str | None]:
-    bad_chars = "\" \t"
-    result = {}
-
-    for statement in header.split(";"):
-        split = statement.split("=", 2)
-
-        result[split[0].strip(bad_chars)] = None if len(split) == 1 else split[1].strip(bad_chars)
-
-    return result
-
-
-class DropboxScanner(ResourceScanner):
-    def __init__(self, client: httpx.AsyncClient):
-        self.client = client
-
-    def accepts_link(self, link: Tag) -> bool:
-        if link.name != "a":
-            return False
-
-        if "href" in link.attrs:
-            url = URL(link.attrs["href"])
-
-            return url.host == "www.dropbox.com" and url.path.split("/", 2)[1] == "s"
-        else:
-            return False
-
-    async def extract_resource(self, link: Tag, file_id: str) -> db.Resource:
-        url = URL(link.attrs["href"]).copy_set_param("dl", 1)
-
-        req = self.client.build_request(
-            method="GET",
-            url=url
-        )
-
-        # The following code essentially starts downloading the file, reads the headers and then stops downloading it,
-        # just to ge the size of the file
-        resp = await self.client.send(req, follow_redirects=True, stream=True)
-
-        try:
-            resp.raise_for_status()
-
-            filename = parse_content_disposition(resp.headers["content-disposition"])["filename"]
-            size = int(resp.headers["content-length"])
-
-            return db.Resource(id=file_id, url=str(url), file_name=filename, file_size=size)
-        finally:
-            await resp.aclose()
-
-    def extract_id(self, link: Tag) -> str:
-        # https://www.dropbox.com/s/vwk48ajl9nw6pqh/Lab1_ENGG1500_robot.pdf?dl=0
-        # ------- Extract this part ^^^^^^^^^^^^^^^
-        return URL(link.attrs["href"]).path.split("/", 3)[2]
-
-    @property
-    def name(self) -> str:
-        return "dropbox"

qcanvas/util/link_scanner/resource_scanner.py

@@ -1,69 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import AsyncIterator
-
-import httpx
-from bs4 import Tag
-
-import qcanvas.db as db
-
-
-class ResourceScanner(ABC):
-    """
-    A resource scanner extracts resources from canvas pages.
-    The resource may be an embedded video, a file or anything that will be of use to the user.
-    Each scanner should be responsible for only 1 type of resource.
-    """
-
-    @property
-    @abstractmethod
-    def name(self) -> str:
-        """
-        The name of the resource scanner.
-        Will be attached to the resource id externally.
-        """
-        ...
-
-    @abstractmethod
-    def accepts_link(self, link: Tag) -> bool:
-        """
-        Whether this resource scanner accepts the specified link
-        """
-        ...
-
-    @abstractmethod
-    async def extract_resource(self, link: Tag, file_id: str) -> db.Resource:
-        """
-        Extract information about the resource in the specified tag
-        Parameters
-        ----------
-        link
-            The element that links to the resource
-        file_id
-            The id of the file (as produced from `extract_id`)
-        Returns
-        -------
-            The resource
-        """
-        ...
-
-    @abstractmethod
-    def extract_id(self, link: Tag) -> str:
-        """
-        Extracts a unique id from a file link
-        """
-        ...
-
-    async def download(self, resource: db.Resource) -> AsyncIterator[int]:
-        yield 0
-
-        download_destination = resource.download_location
-        download_destination.parent.mkdir(parents=True, exist_ok=True)
-
-        with open(download_destination, "wb") as file:
-            async with httpx.AsyncClient(follow_redirects=True) as client:
-                async with client.stream(method='get', url=resource.url) as resp:
-                    resp.raise_for_status()
-
-                    async for chunk in resp.aiter_bytes():
-                        file.write(chunk)
-                        yield resp.num_bytes_downloaded