qcanvas 0.0.5.7a0__py3-none-any.whl → 1.0.3.post1__py3-none-any.whl

qcanvas/util/course_indexer/data_manager.py

@@ -1,447 +0,0 @@
-import asyncio
-import logging
-import traceback
-from asyncio import Task
-from dataclasses import dataclass
-from typing import Sequence
-
-from gql import gql
-from sqlalchemy import select
-from sqlalchemy.dialects.sqlite import insert as sqlite_upsert
-from sqlalchemy.ext.asyncio.session import async_sessionmaker as AsyncSessionMaker, AsyncSession
-from sqlalchemy.orm import selectin_polymorphic, selectinload
-
-import qcanvas.db as db
-import qcanvas.queries as queries
-import qcanvas.util.course_indexer.conversion_helpers as conv_helper
-import qcanvas.util.course_indexer.resource_helpers as resource_helper
-from qcanvas.net.canvas import CanvasClient
-from qcanvas.util.download_pool import DownloadPool
-from qcanvas.util.link_scanner.canvas_link_scanner import canvas_resource_id_prefix
-from qcanvas.util.link_scanner.resource_scanner import ResourceScanner
-from qcanvas.util.progress_reporter import ProgressReporter, noop_reporter
-from qcanvas.util.task_pool import TaskPool
-
-_logger = logging.getLogger("course_loader")
-
-
-@dataclass
-class TransientModulePage:
-    page: queries.Page | queries.File
-    course_id: str
-    module_id: str
-    position: int
-
-
-def _prepare_out_of_date_pages_for_loading(g_courses: Sequence[queries.Course], pages: Sequence[db.ModuleItem]) -> list[
-    TransientModulePage]:
-    """
-    Removes pages that are up-to-date from the pages list by comparing the last update time of the pages from the query
-    to the last update time of the pages in the database.
-
-    Parameters
-    ----------
-    g_courses
-        The list of courses (with module items) to check for an update.
-    pages
-        The list of pages already existing in the database.
-    Returns
-    -------
-    list[TransientModulePage]
-        A list of pages that have had an update to them.
-    """
-    pages_id_mapped = {x.id: x for x in pages}
-
-    result: list[TransientModulePage] = []
-
-    for g_course in g_courses:
-        for g_module in g_course.modules_connection.nodes:
-            for item_position, g_module_item in enumerate(g_module.module_items):
-                content = g_module_item.content
-
-                if isinstance(content, (queries.File, queries.Page)):
-                    # todo need to decide how to only rescan old pages or only rescan new pages without fetching content of old pages again for no good reason
-                    if (
-                            content.m_id not in pages_id_mapped
-                            or content.updated_at.replace(tzinfo=None) > pages_id_mapped[content.m_id].updated_at
-                    ):
-                        result.append(TransientModulePage(content, g_course.m_id, g_module.q_id, item_position))
-                    else:
-                        _logger.debug("Page %s is already up to date", content.m_id)
-
-    return result
-
-
-# todo make this reusable and add some way of refreshing only a list of pages or one page or one course or something
-# todo use logger instead of print and put some signals around the place for useful things, e.g. indexing progress
-class DataManager:
-    """
-    Responsible for storing all data pulled from canvas or other websites in the database.
-    Provides functions for synchronizing with canvas and downloading files.
-    """
-    def __init__(self,
-                 client: CanvasClient,
-                 sessionmaker: AsyncSessionMaker,
-                 link_scanners: Sequence[ResourceScanner]):
-
-        self.client = client
-        self._link_scanners = link_scanners
-        self._session_maker = sessionmaker
-
-        self._resource_pool = TaskPool[db.Resource]()
-        # todo just remove this thing it does nothing
-        self._moduleitem_pool = TaskPool[db.ModuleItem]()
-        self.download_pool = DownloadPool()
-
-        # Map all the scanners we have to their own name
-        self._scanner_name_map = {scanner.name: scanner for scanner in self._link_scanners}
-
-        self._init_called = False
-
-    async def init(self):
-        """
-        Load existing pages and resources from the database, so they don't have to be fetched from canvas again
-        """
-        self._init_called = True
-
-        async with self._session_maker.begin() as session:
-            # Load existing pages and resources from the database
-            existing_pages = (await session.execute(
-                select(db.ModuleItem)
-                .options(selectin_polymorphic(db.ModuleItem, [db.ModulePage]))
-            )).scalars().all()
-
-            existing_resources = (await session.execute(
-                select(db.Resource)
-            )).scalars().all()
-
-            # Add the existing items to the relevant taskpools
-            self._add_resources_and_pages_to_taskpool(existing_pages=existing_pages,
-                                                      existing_resources=existing_resources)
-
-    async def _download_resource_helper(self, link_handler: ResourceScanner, resource: db.Resource):
-        try:
-            async for progress in link_handler.download(resource):
-                yield progress
-
-            # Do this here because this function will only be called once for this resource
-            async with self._session_maker.begin() as session:
-                session.add(resource)
-                resource.state = db.ResourceState.DOWNLOADED
-        except BaseException as e:
-            # Something went wrong, record the failure in the database
-            async with self._session_maker.begin() as session:
-                session.add(resource)
-                resource.state = db.ResourceState.FAILED
-                resource.fail_message = str(e)
-
-            raise e
-
-    async def download_resource(self, resource: db.Resource):
-        if not self._init_called:
-            raise Exception("Init was not called")
-
-        # Resource ids look like this: "canvas_file:387837", and we just want the "canvas_file" part
-        scanner_name: str = resource.id.split(':', 2)[0]
-        # Find the scanner that will deal with this resource
-        scanner = self._scanner_name_map[scanner_name]
-
-        await self.download_pool.submit(resource.id, lambda: self._download_resource_helper(scanner, resource))
-
-    async def update_item(self, item: db.Base):
-        async with self._session_maker.begin() as session:
-            await session.merge(item)
-
-    async def get_data(self):
-        """
-        Loads all the course data
-        """
-        async with self._session_maker.begin() as session:
-            module_items_load = selectinload(db.Course.modules).joinedload(db.Module.items)
-
-            # Eagerly load fucking everything
-            options = [
-                selectinload(db.Course.modules)
-                .joinedload(db.Module.course),
-
-                module_items_load.selectin_polymorphic([db.ModulePage, db.ModuleFile])
-                .joinedload(db.ModuleItem.module),
-
-                module_items_load.joinedload(db.ModuleItem.resources),
-                selectinload(db.Course.assignments)
-                .joinedload(db.Assignment.course),
-
-                selectinload(db.Course.assignments)
-                .joinedload(db.Assignment.resources),
-
-                selectinload(db.Course.term),
-
-                selectinload(db.Course.module_items)
-                .joinedload(db.ModuleItem.course),
-
-                selectinload(db.Course.preferences)
-                .joinedload(db.CoursePreferences.course),
-
-                selectinload(db.Course.resources)
-                .joinedload(db.Resource.course)
-            ]
-
-            return (await session.execute(select(db.Course).options(*options))).scalars().all()
-
-    async def synchronize_with_canvas(self, progress_reporter: ProgressReporter = noop_reporter):
-        section = progress_reporter.section("Loading index", 0)
-        raw_query = (await self.client.do_graphql_query(gql(queries.all_courses.DEFINITION), detailed=True))
-        section.increment_progress()
-
-        await self.load_courses_data(queries.AllCoursesQueryData(**raw_query).all_courses, progress_reporter)
-
-    async def load_courses_data(self, g_courses: Sequence[queries.Course], progress_reporter: ProgressReporter):
-        """
-        Loads data for all specified courses, including loading module pages and scanning for resources.
-        """
-
-        if not self._init_called:
-            raise Exception("Init was not called")
-
-        try:
-            async with self._session_maker.begin() as session:
-                # Load module pages/files for the courses
-                await self._load_module_items(g_courses, session, progress_reporter)
-
-                # Collect assignments from the courses
-                assignments = []
-
-                for g_course in g_courses:
-                    # Create needed data in the session
-                    term = await conv_helper.create_term(g_course, session)
-                    await conv_helper.create_course(g_course, session, term)
-                    await conv_helper.create_modules(g_course, session)
-
-                    # Add course assignments to the list
-                    assignments.extend(await conv_helper.create_assignments(g_course, session))
-
-                # Scan assignments for resources
-                await self._scan_assignments_for_resources(assignments, session, progress_reporter)
-
-                # Add all resources back into the session
-                session.add_all(self._resource_pool.results())
-                progress_reporter.finished()
-        except BaseException as e:
-            traceback.print_exc()
-            progress_reporter.errored(e)
-
-    async def _scan_assignments_for_resources(self, assignments: Sequence[db.Assignment], session: AsyncSession,
-                                              progress_reporter: ProgressReporter):
-        """
-        Scans assignments for resources
-        """
-
-        # Link the resources found to each page in the database
-        await resource_helper.create_assignment_resource_relations(
-            # Find all the resources in each assignment description
-            await resource_helper.find_resources_in_pages(
-                link_scanners=self._link_scanners,
-                resource_pool=self._resource_pool,
-                items=assignments,
-                progress_reporter=progress_reporter
-            ),
-            session
-        )
-
-    async def _load_module_items(self, g_courses: Sequence[queries.Course], session: AsyncSession,
-                                 progress_reporter: ProgressReporter):
-        # Get the ids of all the courses we are going to index/load
-        course_ids = [g_course.m_id for g_course in g_courses]
-
-        # Prepare pages for loading
-        existing_pages = (
-            await session.execute(
-                select(db.ModulePage)
-                .where(db.ModuleItem.course_id.in_(course_ids))
-            )).scalars().all()
-
-        # Filter out pages that don't need updating
-        pages_to_update = _prepare_out_of_date_pages_for_loading(g_courses, existing_pages)
-
-        if len(pages_to_update) == 0:
-            return
-
-        # Load the content for all the pages that need updating
-        module_items: list[db.ModuleItem] = await self._load_content_for_pages(pages_to_update, progress_reporter)
-        module_pages = [item for item in module_items if isinstance(item, db.ModulePage)]
-
-        # Link the resources found to the pages they were found on and add them to the database
-        await resource_helper.create_module_item_resource_relations(
-            # Find all the resources in each page
-            await resource_helper.find_resources_in_pages(
-                link_scanners=self._link_scanners,
-                resource_pool=self._resource_pool,
-                progress_reporter=progress_reporter,
-                # Collect just the module pages for scanning
-                items=module_pages
-            ),
-            session
-        )
-
-        # empty inserts/upserts causes an sql error. don't do them
-        if len(module_pages) > 0:
-            # Add all the module items to the session
-            # shitty bandaid fix
-            upsert_item = sqlite_upsert(db.ModuleItem).values([self.moduleitem_dict(item) for item in module_pages])
-            upsert_item = upsert_item.on_conflict_do_update(
-                index_elements=[db.ModuleItem.id],
-                set_=dict(name=upsert_item.excluded.name, updated_at=upsert_item.excluded.updated_at,
-                          position=upsert_item.excluded.position),
-
-            )
-
-            upsert_page = sqlite_upsert(db.ModulePage).values([self.page_dict(item) for item in module_pages])
-            upsert_page = upsert_page.on_conflict_do_update(
-                index_elements=[db.ModulePage.id],
-                set_=dict(content=upsert_page.excluded.content)
-            )
-
-            await session.execute(upsert_item)
-            await session.execute(upsert_page)
-
-        session.add_all([item for item in module_items if isinstance(item, db.ModuleFile)])
-
-    @staticmethod
-    def page_dict(page: db.ModulePage) -> dict[str, object]:
-        return {"id": page.id, "content": page.content}
-
-    @staticmethod
-    def moduleitem_dict(page: db.ModuleItem) -> dict[str, object]:
-        return {"id": page.id, "name": page.name, "updated_at": page.updated_at, "position": page.position,
-                "module_id": page.module_id, "course_id": page.course_id, "type": page.type,
-                "created_at": page.created_at}
-
-    def _add_resources_and_pages_to_taskpool(self, existing_pages: Sequence[db.ModuleItem],
-                                             existing_resources: Sequence[db.Resource]):
-        self._moduleitem_pool.add_values({page.id: page for page in existing_pages})
-        self._resource_pool.add_values({resource.id: resource for resource in existing_resources})
-        # Add downloaded resources to the resource pool so we don't download them again
-        self.download_pool.add_values(
-            {resource.id: None for resource in existing_resources if resource.state == db.ResourceState.DOWNLOADED})
-
-    async def _load_content_for_pages(self, pages: Sequence[TransientModulePage],
-                                      progress_reporter: ProgressReporter) -> list[db.ModuleItem]:
-        """
-        Loads the page content for the specified pages
-        Parameters
-        ----------
-        pages
-            The pages to load
-        Returns
-        -------
-        list
-            The list of complete pages with page content loaded.
-        """
-        progress = progress_reporter.section("Loading page content", len(pages))
-        tasks: list[Task[db.ModuleItem | None]] = []
-
-        for page in pages:
-            content = page.page
-
-            # Load the content for the pages
-            if isinstance(content, queries.File):
-                task = asyncio.create_task(
-                    self._load_module_file(content, page.course_id, page.module_id, page.position))
-                task.add_done_callback(progress.increment_progress)
-                tasks.append(task)
-            elif isinstance(content, queries.Page):
-                task = asyncio.create_task(
-                    self.load_module_page(content, page.course_id, page.module_id, page.position))
-                task.add_done_callback(progress.increment_progress)
-                tasks.append(task)
-
-        if len(tasks) > 0:
-            await asyncio.wait(tasks)
-
-            # Collect results and filter out nulls
-            return [task.result() for task in tasks if task.result() is not None]
-        else:
-            return []
-
-    async def _load_module_file(self, g_file: queries.File, course_id: str, module_id: str,
-                                position: int) -> db.ModuleFile:
-        """
-        Fetches resource information for the module file and converts it into a module item
-        """
-        _logger.debug(f"Loading module file %s %s", g_file.m_id, g_file.display_name)
-
-        resource = await self._resource_pool.submit(
-            f"{canvas_resource_id_prefix}:{g_file.m_id}",  # to match the format used by canvas link extractor
-            lambda: self._fetch_module_file_resource(g_file, course_id)
-        )
-
-        return await self._moduleitem_pool.submit(
-            g_file.m_id,
-            lambda: self._fetch_module_file_page(g_file, resource, course_id, module_id, position)
-        )
-
-    async def _fetch_module_file_resource(self, file: queries.File, course_id: str) -> db.Resource:
-        """
-        Fetches information about the specified file from canvas
-        """
-        _logger.debug(f"Fetching file (for module file) %s %s", file.m_id, file.display_name)
-        result = await self.client.get_file(file.m_id, course_id)
-        resource = db.convert_file(file, result.size)
-        resource.id = f"{canvas_resource_id_prefix}:{resource.id}"
-        resource.course_id = course_id
-
-        return resource
-
-    async def load_module_page(self, g_page: queries.Page, course_id: str, module_id: str,
-                               position: int) -> db.ModulePage | None:
-        """
-        Creates task for loading the specified module page
-        """
-        return await self._fetch_module_item_page(g_page, course_id, module_id, position)
-
-    async def _fetch_module_item_page(self, page: queries.Page, course_id: str, module_id: str,
-                                      position: int) -> db.ModulePage | None:
-        """
-        Fetches module page content from canvas. Returns None if the page could not be loaded.
-        """
-        _logger.debug("Fetching module page %s %s", page.m_id, page.title)
-
-        try:
-            # Get the page
-            result = await self.client.get_page(page.m_id, course_id)
-        except BaseException as e:
-            # Handle any errors
-            _logger.error(e)
-            traceback.print_exc()
-            return None
-
-        if result.locked_for_user:
-            _logger.error("Page %s %s is locked", page.m_id, page.title)
-            return None
-
-        page = db.convert_page(page, result.body)
-        page.module_id = module_id
-        page.course_id = course_id
-        page.position = position
-
-        return page
-
-    @staticmethod
-    async def _fetch_module_file_page(file: queries.File, resource: db.Resource, course_id: str,
-                                      module_id: str, position: int) -> db.ModuleFile:
-        """
-        Converts module file information into the database format
-        """
-        _logger.debug(f"Creating page for module file %s %s", file.m_id, file.display_name)
-
-        page = db.convert_file_page(file)
-        page.module_id = module_id
-        page.course_id = course_id
-        page.position = position
-        page.resources.append(resource)
-
-        return page
-
-    @property
-    def link_scanners(self):
-        return self._link_scanners

qcanvas/util/course_indexer/resource_helpers.py

@@ -1,191 +0,0 @@
-import asyncio
-import logging
-from dataclasses import dataclass
-from typing import Sequence
-
-from bs4 import Tag, BeautifulSoup
-from sqlalchemy.ext.asyncio import AsyncSession
-
-import qcanvas.db as db
-from qcanvas.util.link_scanner import ResourceScanner
-from qcanvas.util.progress_reporter import ProgressReporter
-from qcanvas.util.task_pool import TaskPool
-
-_logger = logging.getLogger(__name__)
-
-resource_elements = ["a", "iframe", "img"]
-
-
-# todo could probably just use the database types directly now
-@dataclass
-class TransientResourceToPageLink:
-    """
-    Represents a temporary link between a page and a resource that will be added to the database soon.
-    """
-    page_id: str
-    resource_id: str
-
-    def __hash__(self):
-        return hash(self.page_id) ^ hash(self.resource_id)
-
-
-async def create_module_item_resource_relations(relations: Sequence[TransientResourceToPageLink],
-                                                session: AsyncSession):
-    """
-    Creates a link between module items/pages and resources found on those pages
-    """
-    for relation in relations:
-        existing_relation = await session.get(
-            db.ResourceToModuleItemAssociation,
-            (relation.page_id, relation.resource_id)
-        )
-
-        if existing_relation is None:
-            session.add(
-                db.ResourceToModuleItemAssociation(
-                    module_item_id=relation.page_id,
-                    resource_id=relation.resource_id
-                )
-            )
-
-
-async def create_assignment_resource_relations(relations: Sequence[TransientResourceToPageLink], session: AsyncSession):
-    """
-    Turns temporary TransientResourceToPageLink into a persistent relation in the database
-    """
-    for relation in relations:
-        if await session.get(db.ResourceToAssignmentAssociation, (relation.page_id, relation.resource_id)) is None:
-            session.add(
-                db.ResourceToAssignmentAssociation(
-                    assignment_id=relation.page_id,
-                    resource_id=relation.resource_id
-                )
-            )
-
-
-# todo change resource system to think of resources as links on a page with a shallow id (that may be the same as the deep id) which links to one or more deep ids
-async def find_resources_in_pages(link_scanners: Sequence[ResourceScanner], resource_pool: TaskPool[db.Resource],
-                                  items: Sequence[db.PageLike], progress_reporter: ProgressReporter) -> list[
-    TransientResourceToPageLink]:
-    """
-    Produce a list of resource to page links from resources extracted from the specified `items` using `link_scanners`.
-    Extracted resources will be added to `resource_pool`
-    """
-    progress = progress_reporter.section("Indexing resources", len(items))
-    tasks = []
-
-    for item in items:
-        # Assignment descriptions may be null. Avoid creating extra tasks by checking here
-        if item.content is None:
-            progress.increment_progress()
-            continue
-
-        # extract resources from the page
-        task = asyncio.create_task(_extract_resources_from_page(link_scanners, resource_pool, item))
-        task.add_done_callback(progress.increment_progress)
-        tasks.append(task)
-
-    if len(tasks) > 0:
-        # Wait for all tasks to complete
-        await asyncio.wait(tasks)
-
-        result = []
-        # Flatten the array of results
-        for task in tasks:
-            result.extend(task.result())
-
-        return result
-    else:
-        return []
-
-
-async def _extract_resources_from_page(link_scanners: Sequence[ResourceScanner], resource_pool: TaskPool[db.Resource],
-                                       page: db.PageLike) -> list[TransientResourceToPageLink]:
-    """
-    Extracts any detected resource links from the specified page and then uses `link_scanners` to extract information
-    about which is then added to the `resource_pool`.
-
-    Returns
-    -------
-    list
-        A list of resource to page links for any resources found on this page.
-    """
-    _logger.debug(f"Scanning %s %s for files", page.id, page.name)
-    tasks = []
-
-    # Extract iframes, hyperlinks, etc from the page
-    for link in _scan_page_for_links(page):
-        tasks.append(asyncio.create_task(_process_link(link_scanners, resource_pool, link, page.course_id)))
-
-    if len(tasks) > 0:
-        # Wait for all tasks to complete
-        await asyncio.wait(tasks)
-
-        task_results = [task.result() for task in tasks]
-
-        # Convert every non-null result in the task results to a resource page link and return it
-        return [
-            TransientResourceToPageLink(page_id=page.id, resource_id=result.id)
-            for result in task_results if result is not None
-        ]
-    else:
-        return []
-
-
-def _scan_page_for_links(page: db.PageLike) -> list[Tag]:
-    """
-    Extracts (potential) resource elements from a PageLike object
-    """
-    soup = BeautifulSoup(page.content, 'html.parser')
-    return list(soup.find_all(resource_elements))
-
-
-async def _process_link(link_scanners: Sequence[ResourceScanner], resource_pool: TaskPool[db.Resource], link: Tag,
-                        course_id: str) -> db.Resource | None:
-    """
-    Iterates over `link_scanners` to find one that will accept `link`, then uses it to fetch resource information and
-     adds it to the `resource_pool`.
-    If no scanner accepts the link then None is returned.
-    """
-    for scanner in link_scanners:
-        if scanner.accepts_link(link):
-            resource_id = scanner.extract_id(link)
-
-            return await resource_pool.submit(
-                f"{scanner.name}:{resource_id}",  # match the format used by the resource id
-                lambda: _extract_file_info(link, scanner, resource_id, course_id)
-            )
-
-    return None
-
-
-async def _extract_file_info(link: Tag, scanner: ResourceScanner, resource_id: str,
-                             course_id: str) -> db.Resource | None:
-    """
-    Extracts file info from `link` using `scanner` and assigns the course_id to the resulting resource.
-
-    Parameters
-    ----------
-    link
-        The html element to scan
-    scanner
-        The scanner to process the link with
-    course_id
-        The id of the course the file belongs to
-    Returns
-    -------
-    db.Resource
-        The resource if the link was processed successfully.
-    None
-        If processing failed
-    """
-    try:
-        _logger.debug(f"Fetching info for file %s with scanner %s", scanner.extract_id(link), scanner.name)
-
-        result = await scanner.extract_resource(link, resource_id)
-        result.id = f"{scanner.name}:{result.id}"  # Prefix the scanner name to prevent resources from different sites potentially clashing
-        result.course_id = course_id
-        return result
-    except BaseException as e:
-        _logger.error(f"Failed to retrieve info for file id %s: %s", f"{scanner.name}:{resource_id}", str(e))
-        return None

qcanvas/util/download_pool.py

@@ -1,58 +0,0 @@
-import asyncio
-from typing import Callable, Any
-
-from PySide6.QtCore import QObject, Signal
-
-from qcanvas.util.task_pool import TaskPool
-
-
-class DownloadPool(TaskPool[None], QObject):
-    """
-    Like the TaskPool, but uses a queue (a channel in kotlin terms) in the task to report download progress.
-    When using .submit(), it is EXPECTED that the lambda returns a function that uses yield to signal progress, e.g:
-    ```
-    pool.submit(my_task_id, lambda: my_func())
-
-    async def my_func(channel: asyncio.Queue):
-        for i in range(0, 10):
-            await slow_thing()
-            yield (i / 10) # can be whatever format/type you want
-    ```
-    """
-    download_progress_updated = Signal(object, Any)
-    download_failed = Signal(object)
-    download_finished = Signal(object)
-
-    def __init__(self, parent: QObject | None = None):
-        QObject.__init__(self, parent)
-        TaskPool.__init__(self)
-
-    async def _handle_task(self, func: Callable, task_id: object, event: asyncio.Event, func_args: dict):
-        sem = self._semaphore
-
-        try:
-            # Consume progress updates 'yield'ed from the function
-            async for progress in func(**func_args):
-                # Fire the download progress update signal
-                self.download_progress_updated.emit(task_id, progress)
-        except BaseException as e:
-            # Try to maintain integrity when a task fails
-            async with sem:
-                # Release anything else waiting for this task
-                event.set()
-                # Remove the record from the results map
-                del self._results[task_id]
-
-            # Emit failure signal and rethrow
-            self.download_failed.emit(task_id)
-            raise e
-
-        async with sem:
-            self._logger.debug("Task %s finished.", task_id)
-            # Record this task as done
-            self._results[task_id] = None
-            event.set()
-
-        self.download_finished.emit(task_id)
-
-        return None