oqtopus 0.2.1__tar.gz → 1.0.0__tar.gz

{oqtopus-0.2.1 → oqtopus-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oqtopus
-Version: 0.2.1
+Version: 1.0.0
 Summary: oQtopus : A QGIS module manager that helps you deploy, manage and upgrade your QGIS projects, plugins and associated postgreSQL / PostGIS data model implementations
 Author-email: Damiano Lombardi <damiano@opengis.ch>
 License:                     GNU GENERAL PUBLIC LICENSE

oqtopus-1.0.0/oqtopus/core/module.py

@@ -0,0 +1,262 @@
+import json
+import os
+import re
+import time
+
+from qgis.PyQt.QtCore import (
+    QByteArray,
+    QObject,
+    QTimer,
+    QUrl,
+    pyqtSignal,
+)
+from qgis.PyQt.QtNetwork import QNetworkAccessManager, QNetworkReply, QNetworkRequest
+
+from ..utils.plugin_utils import PluginUtils, logger
+from .module_package import ModulePackage
+
+# Cache duration in seconds (1 hour)
+CACHE_DURATION = 3600
+
+
+class Module(QObject):
+    signal_versionsLoaded = pyqtSignal(str)
+    signal_developmentVersionsLoaded = pyqtSignal(str)
+
+    def __init__(
+        self,
+        name: str,
+        id: str,
+        organisation: str,
+        repository: str,
+        exclude_releases: str | None = None,
+        parent=None,
+    ):
+        super().__init__(parent)
+        self.name = name
+        self.id = id
+        self.organisation = organisation
+        self.repository = repository
+        self.exclude_releases = exclude_releases
+        self.versions = []
+        self.development_versions = []
+        self.latest_version = None
+        self.network_manager = QNetworkAccessManager(self)
+
+    def __repr__(self):
+        return f"Module(name={self.name}, organisation={self.organisation}, repository={self.repository})"
+
+    def __get_cache_dir(self):
+        """Get the cache directory for GitHub API responses."""
+        cache_dir = os.path.join(
+            PluginUtils.plugin_cache_path(), "github_api", self.organisation, self.repository
+        )
+        os.makedirs(cache_dir, exist_ok=True)
+        return cache_dir
+
+    def __get_cache_file(self, cache_type):
+        """Get the cache file path for a specific type (releases or pulls)."""
+        cache_file = os.path.join(self.__get_cache_dir(), f"{cache_type}.json")
+        return cache_file
+
+    def __read_cache(self, cache_type):
+        """Read cached data if it exists and is not expired."""
+        cache_file = self.__get_cache_file(cache_type)
+        if not os.path.exists(cache_file):
+            return None
+
+        # Check if cache is expired
+        file_age = time.time() - os.path.getmtime(cache_file)
+        if file_age > CACHE_DURATION:
+            return None
+
+        try:
+            with open(cache_file, encoding="utf-8") as f:
+                data = json.load(f)
+                logger.info(
+                    f"Using cached {cache_type} data (age: {file_age:.0f}s, {len(data)} items)"
+                )
+                return data
+        except Exception as e:
+            logger.warning(f"Failed to read cache for {cache_type}: {e}")
+            return None
+
+    def __write_cache(self, cache_type, data):
+        """Write data to cache file."""
+        cache_file = self.__get_cache_file(cache_type)
+        try:
+            with open(cache_file, "w", encoding="utf-8") as f:
+                json.dump(data, f)
+        except Exception as e:
+            logger.warning(f"Failed to write cache for {cache_type}: {e}")
+
+    def start_load_versions(self):
+        # Read cache asynchronously to avoid blocking UI
+        QTimer.singleShot(0, self.__async_load_versions)
+
+    def __async_load_versions(self):
+        """Load versions asynchronously from cache or API."""
+        # Try to load from cache first
+        cached_data = self.__read_cache("releases")
+        if cached_data is not None:
+            try:
+                self._process_versions_data(cached_data)
+                self.signal_versionsLoaded.emit("")
+                return
+            except Exception as e:
+                logger.warning(f"Failed to process cached releases: {e}")
+
+        # Cache miss or invalid - fetch from API
+        url = f"https://api.github.com/repos/{self.organisation}/{self.repository}/releases"
+        logger.info(f"Loading versions from '{url}'...")
+        request = QNetworkRequest(QUrl(url))
+        headers = PluginUtils.get_github_headers()
+        for key, value in headers.items():
+            request.setRawHeader(QByteArray(key.encode()), QByteArray(value.encode()))
+        reply = self.network_manager.get(request)
+        reply.finished.connect(lambda: self._on_versions_reply(reply))
+
+    def _on_versions_reply(self, reply):
+        if reply.error() != QNetworkReply.NetworkError.NoError:
+            self.signal_versionsLoaded.emit(reply.errorString())
+            reply.deleteLater()
+            return
+        try:
+            data = reply.readAll().data()
+            json_versions = json.loads(data.decode())
+
+            # Cache the response
+            self.__write_cache("releases", json_versions)
+
+            self._process_versions_data(json_versions)
+            self.signal_versionsLoaded.emit("")
+        except Exception as e:
+            self.signal_versionsLoaded.emit(str(e))
+        reply.deleteLater()
+
+    def _process_versions_data(self, json_versions):
+        """Process versions data from cache or API response."""
+        self.versions = []
+        self.latest_version = None
+
+        # Compile exclude pattern if specified
+        exclude_pattern = None
+        if self.exclude_releases:
+            try:
+                exclude_pattern = re.compile(self.exclude_releases)
+            except re.error as e:
+                logger.warning(f"Invalid exclude_releases pattern '{self.exclude_releases}': {e}")
+
+        for json_version in json_versions:
+            # Check if this release should be excluded
+            tag_name = json_version.get("tag_name", "")
+            if exclude_pattern and exclude_pattern.search(tag_name):
+                continue
+
+            module_package = ModulePackage(
+                module=self,
+                organisation=self.organisation,
+                repository=self.repository,
+                json_payload=json_version,
+                type=ModulePackage.Type.RELEASE,
+            )
+            self.versions.append(module_package)
+
+            # Latest version -> most recent commit date for non prerelease
+            if module_package.prerelease is True:
+                continue
+
+            if self.latest_version is None:
+                self.latest_version = module_package
+                continue
+
+            if module_package.created_at > self.latest_version.created_at:
+                self.latest_version = module_package
+
+    def start_load_development_versions(self):
+        self.development_versions = []
+
+        # Add pre-releases from already loaded versions
+        for version in self.versions:
+            if version.prerelease is True:
+                self.development_versions.append(version)
+
+        # Create version for the main branch
+        mainVersion = ModulePackage(
+            module=self,
+            organisation=self.organisation,
+            repository=self.repository,
+            json_payload="",
+            type=ModulePackage.Type.BRANCH,
+            name="main",
+            branch="main",
+        )
+        # Fetch the latest commit SHA for caching (async to avoid blocking UI)
+        QTimer.singleShot(0, lambda: mainVersion.fetch_commit_sha())
+        self.development_versions.append(mainVersion)
+
+        # Try to load pull requests from cache first
+        cached_data = self.__read_cache("pulls")
+        if cached_data is not None:
+            # Process cache asynchronously to keep UI responsive
+            QTimer.singleShot(0, lambda: self._process_cached_pulls(cached_data))
+            return
+
+        # Cache miss or invalid - fetch from API
+        url = f"https://api.github.com/repos/{self.organisation}/{self.repository}/pulls"
+        logger.info(f"Loading pre-releases and development versions from '{url}'...")
+
+        request = QNetworkRequest(QUrl(url))
+        headers = PluginUtils.get_github_headers()
+        for key, value in headers.items():
+            request.setRawHeader(QByteArray(key.encode()), QByteArray(value.encode()))
+        reply = self.network_manager.get(request)
+        reply.finished.connect(lambda: self._on_development_versions_reply(reply))
+
+    def _process_cached_pulls(self, cached_data):
+        """Process cached pull requests data asynchronously."""
+        try:
+            self._process_pull_requests_data(cached_data)
+            self.signal_developmentVersionsLoaded.emit("")
+        except Exception as e:
+            logger.warning(f"Failed to process cached pull requests: {e}")
+            # On error, continue with API call
+            url = f"https://api.github.com/repos/{self.organisation}/{self.repository}/pulls"
+            logger.info(f"Loading pre-releases and development versions from '{url}'...")
+            request = QNetworkRequest(QUrl(url))
+            headers = PluginUtils.get_github_headers()
+            for key, value in headers.items():
+                request.setRawHeader(QByteArray(key.encode()), QByteArray(value.encode()))
+            reply = self.network_manager.get(request)
+            reply.finished.connect(lambda: self._on_development_versions_reply(reply))
+
+    def _on_development_versions_reply(self, reply):
+        if reply.error() != QNetworkReply.NetworkError.NoError:
+            self.signal_developmentVersionsLoaded.emit(reply.errorString())
+            reply.deleteLater()
+            return
+
+        try:
+            data = reply.readAll().data()
+            json_versions = json.loads(data.decode())
+
+            # Cache the response
+            self.__write_cache("pulls", json_versions)
+
+            self._process_pull_requests_data(json_versions)
+            self.signal_developmentVersionsLoaded.emit("")
+        except Exception as e:
+            self.signal_developmentVersionsLoaded.emit(str(e))
+        reply.deleteLater()
+
+    def _process_pull_requests_data(self, json_versions):
+        """Process pull requests data from cache or API response."""
+        for json_version in json_versions:
+            module_package = ModulePackage(
+                module=self,
+                organisation=self.organisation,
+                repository=self.repository,
+                json_payload=json_version,
+                type=ModulePackage.Type.PULL_REQUEST,
+            )
+            self.development_versions.append(module_package)

oqtopus-1.0.0/oqtopus/core/module_operation_task.py

@@ -0,0 +1,234 @@
+"""Background task for module install/upgrade/uninstall operations."""
+
+import psycopg
+from qgis.PyQt.QtCore import QThread, pyqtSignal
+
+from ..libs.pum.feedback import Feedback
+from ..libs.pum.pum_config import PumConfig
+from ..libs.pum.upgrader import Upgrader
+from ..utils.plugin_utils import logger
+
+
+class ModuleOperationCanceled(Exception):
+    """Exception raised when module operation is canceled."""
+
+
+class ModuleOperationTask(QThread):
+    """
+    Background task for running module install/upgrade/uninstall operations.
+    This allows the UI to remain responsive and show progress during long operations.
+    """
+
+    signalProgress = pyqtSignal(str, int, int)  # message, current, total
+    signalFinished = pyqtSignal(bool, str)  # success, error_message
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+
+        self.__pum_config = None
+        self.__connection = None
+        self.__operation = None  # 'install', 'upgrade', 'uninstall'
+        self.__parameters = None
+        self.__options = {}
+
+        self.__feedback = None
+        self.__canceled = False
+        self.__error_message = None
+
+    def start_install(
+        self, pum_config: PumConfig, connection: psycopg.Connection, parameters: dict, **options
+    ):
+        """Start an install operation."""
+        self.__pum_config = pum_config
+        self.__connection = connection
+        self.__operation = "install"
+        self.__parameters = parameters
+        self.__options = options
+        self.__canceled = False
+        self.__error_message = None
+        self.start()
+
+    def start_upgrade(
+        self, pum_config: PumConfig, connection: psycopg.Connection, parameters: dict, **options
+    ):
+        """Start an upgrade operation."""
+        self.__pum_config = pum_config
+        self.__connection = connection
+        self.__operation = "upgrade"
+        self.__parameters = parameters
+        self.__options = options
+        self.__canceled = False
+        self.__error_message = None
+        self.start()
+
+    def start_uninstall(
+        self, pum_config: PumConfig, connection: psycopg.Connection, parameters: dict, **options
+    ):
+        """Start an uninstall operation."""
+        self.__pum_config = pum_config
+        self.__connection = connection
+        self.__operation = "uninstall"
+        self.__parameters = parameters
+        self.__options = options
+        self.__canceled = False
+        self.__error_message = None
+        self.start()
+
+    def start_roles(
+        self, pum_config: PumConfig, connection: psycopg.Connection, parameters: dict, **options
+    ):
+        """Start a create and grant roles operation."""
+        self.__pum_config = pum_config
+        self.__connection = connection
+        self.__operation = "roles"
+        self.__parameters = parameters
+        self.__options = options
+        self.__canceled = False
+        self.__error_message = None
+        self.start()
+
+    def cancel(self):
+        """Cancel the current operation."""
+        self.__canceled = True
+        if self.__feedback:
+            self.__feedback.cancel()
+
+    def run(self):
+        """Execute the operation in a background thread."""
+        try:
+            # Create feedback instance that emits signals
+            self.__feedback = self._create_feedback()
+
+            upgrader = Upgrader(config=self.__pum_config)
+
+            if self.__operation == "install":
+                self._run_install(upgrader)
+            elif self.__operation == "upgrade":
+                self._run_upgrade(upgrader)
+            elif self.__operation == "uninstall":
+                self._run_uninstall(upgrader)
+            elif self.__operation == "roles":
+                self._run_roles()
+            else:
+                raise Exception(f"Unknown operation: {self.__operation}")
+
+            # Commit if successful and not canceled
+            if not self.__canceled and self.__options.get("commit", True):
+                logger.info("Committing changes to database...")
+                self.__connection.commit()
+                logger.info("Changes committed to the database.")
+
+            logger.info(f"Operation '{self.__operation}' completed successfully")
+            self.signalFinished.emit(True, "")
+
+        except Exception as e:
+            logger.critical(f"Module operation error in '{self.__operation}': {e}")
+            logger.exception("Full traceback:")  # Log full stack trace
+            self.__error_message = str(e)
+            # Rollback on error
+            try:
+                logger.info("Rolling back transaction...")
+                self.__connection.rollback()
+                logger.info("Transaction rolled back")
+            except Exception as rollback_error:
+                logger.error(f"Rollback failed: {rollback_error}")
+            self.signalFinished.emit(False, self.__error_message)
+
+    def _run_install(self, upgrader: Upgrader):
+        """Run install operation."""
+        # Extract options that should not be passed to install()
+        install_demo_data = self.__options.pop("install_demo_data", False)
+        demo_data_name = self.__options.pop("demo_data_name", None)
+
+        upgrader.install(
+            connection=self.__connection,
+            parameters=self.__parameters,
+            feedback=self.__feedback,
+            commit=False,
+            **self.__options,
+        )
+
+        # Install demo data if requested
+        if install_demo_data and demo_data_name:
+            upgrader.install_demo_data(
+                connection=self.__connection,
+                name=demo_data_name,
+                parameters=self.__parameters,
+            )
+
+    def _run_upgrade(self, upgrader: Upgrader):
+        """Run upgrade operation."""
+        upgrader.upgrade(
+            connection=self.__connection,
+            parameters=self.__parameters,
+            feedback=self.__feedback,
+            **self.__options,
+        )
+
+    def _run_uninstall(self, upgrader: Upgrader):
+        """Run uninstall operation."""
+        logger.info("Starting uninstall operation...")
+        logger.debug(f"Parameters: {self.__parameters}")
+        logger.debug(f"Options: {self.__options}")
+
+        upgrader.uninstall(
+            connection=self.__connection,
+            parameters=self.__parameters,
+            feedback=self.__feedback,
+            commit=False,
+        )
+
+        logger.info("Uninstall operation completed")
+
+    def _run_roles(self):
+        """Run create and grant roles operation."""
+        logger.info("Starting create and grant roles operation...")
+
+        role_manager = self.__pum_config.role_manager()
+
+        if not role_manager.roles:
+            logger.warning("No roles defined in the configuration")
+            return
+
+        # Create roles with grant=True to also grant permissions
+        role_manager.create_roles(
+            connection=self.__connection,
+            grant=True,
+            feedback=self.__feedback,
+        )
+
+        logger.info("Create and grant roles operation completed")
+
+    def _create_feedback(self):
+        """Create a Feedback instance that emits Qt signals."""
+
+        class QtFeedback(Feedback):
+            """Feedback implementation that emits Qt signals."""
+
+            def __init__(self, task):
+                super().__init__()
+                self.task = task
+
+            def report_progress(self, message: str, current: int = 0, total: int = 0):
+                """Report progress via Qt signal.
+
+                If current and total are provided (non-zero), use those.
+                Otherwise, use the internal step counter.
+                """
+                # Use provided values if available, otherwise use internal counter
+                if current > 0 or total > 0:
+                    actual_current = current
+                    actual_total = total
+                else:
+                    actual_current, actual_total = self.get_progress()
+
+                logger.info(
+                    f"[{actual_current}/{actual_total}] {message}" if actual_total > 0 else message
+                )
+                self.task.signalProgress.emit(message, actual_current, actual_total)
+
+            def is_cancelled(self):
+                """Check if operation is cancelled."""
+                return self.task._ModuleOperationTask__canceled
+
+        return QtFeedback(self)

{oqtopus-0.2.1 → oqtopus-1.0.0}/oqtopus/core/module_package.py

@@ -23,6 +23,7 @@ class ModulePackage:
         type=Type.RELEASE,
         name=None,
         branch=None,
+        commit_sha=None,
     ):
         self.module = module
         self.organisation = organisation
@@ -30,6 +31,7 @@ class ModulePackage:
         self.type = type
         self.name = name
         self.branch = branch
+        self.commit_sha = commit_sha
         self.created_at = None
         self.prerelease = False
         self.html_url = None
@@ -63,6 +65,26 @@ class ModulePackage:
 
         return self.name
 
+    def fetch_commit_sha(self):
+        """Fetch the latest commit SHA for the branch from GitHub API."""
+        if self.type not in (ModulePackage.Type.BRANCH, ModulePackage.Type.PULL_REQUEST):
+            return
+
+        try:
+            # For branches: use refs/heads/{branch}
+            # For PRs: use refs/heads/{branch} from the head repo
+            url = f"https://api.github.com/repos/{self.organisation}/{self.repository}/commits/{self.branch}"
+            r = requests.get(url, headers=PluginUtils.get_github_headers(), timeout=10)
+            r.raise_for_status()
+            commit_data = r.json()
+            self.commit_sha = commit_data["sha"]
+        except Exception as e:
+            # If we can't fetch the commit SHA, we'll fall back to not caching
+            from ..utils.plugin_utils import logger
+
+            logger.warning(f"Failed to fetch commit SHA for branch '{self.branch}': {e}")
+            self.commit_sha = None
+
     def __parse_release(self, json_payload: dict):
         if self.name is None:
             self.name = json_payload["name"]
@@ -75,17 +97,11 @@ class ModulePackage:
         self.prerelease = json_payload["prerelease"]
         self.html_url = json_payload["html_url"]
 
-        self.__parse_release_assets(json_payload["assets_url"])
-
-    def __parse_release_assets(self, assets_url: str):
-
-        # Load assets
-        r = requests.get(assets_url, headers=PluginUtils.get_github_headers())
-
-        # Raise an exception in case of http errors
-        r.raise_for_status()
+        # Use assets directly from the release payload (already included in releases API response)
+        self.__parse_release_assets(json_payload.get("assets", []))
 
-        json_assets = r.json()
+    def __parse_release_assets(self, json_assets: list):
+        """Parse release assets from the already-fetched release data."""
         for json_asset in json_assets:
             asset = ModuleAsset(
                 name=json_asset["name"],
@@ -113,6 +129,7 @@ class ModulePackage:
         if self.name is None:
             self.name = f"#{json_payload['number']} {json_payload['title']}"
         self.branch = json_payload["head"]["ref"]
+        self.commit_sha = json_payload["head"]["sha"]
         self.created_at = QDateTime.fromString(json_payload["created_at"], Qt.DateFormat.ISODate)
         self.prerelease = False
         self.html_url = json_payload["html_url"]

{oqtopus-0.2.1 → oqtopus-1.0.0}/oqtopus/core/modules_config.py

@@ -3,8 +3,10 @@ from pydantic import BaseModel
 
 class ModuleConfig(BaseModel):
     name: str
+    id: str
     organisation: str
     repository: str
+    exclude_releases: str | None = None  # Regexp pattern to exclude releases
 
 
 class ModulesConfig(BaseModel):