oqtopus 0.2.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

oqtopus/core/package_prepare_task.py

@@ -1,10 +1,12 @@
 import os
+import re
 import shutil
 import zipfile
 
 import requests
 from qgis.PyQt.QtCore import QThread, pyqtSignal
 
+from ..core.module_package import ModulePackage
 from ..utils.plugin_utils import PluginUtils, logger
 
 
@@ -12,13 +14,36 @@ class PackagePrepareTaskCanceled(Exception):
     pass
 
 
+def sanitize_filename(name: str) -> str:
+    """Sanitize a string to be safe for use as a filename/directory name.
+
+    Replaces characters that are problematic on Windows or other filesystems.
+    For PR names like "#909 some title", extracts just the number to keep paths short.
+    """
+    # For PR-style names starting with #, extract just the number
+    pr_match = re.match(r"#?(\d+)", name)
+    if pr_match:
+        return f"PR_{pr_match.group(1)}"
+
+    # Replace characters that are invalid on Windows: < > : " / \\ | ? * #
+    # Also replace spaces to avoid path issues
+    sanitized = re.sub(r'[<>:"/\\|?*#\s]+', "_", name)
+    # Remove leading/trailing underscores and dots
+    sanitized = sanitized.strip("_.")
+    # Limit length to avoid Windows MAX_PATH issues (260 char limit)
+    # Keep it very short since zip contents add more nested paths
+    if len(sanitized) > 40:
+        sanitized = sanitized[:40]
+    return sanitized
+
+
 class PackagePrepareTask(QThread):
     """
     This class is responsible for preparing the package for the oQtopus module management tool.
     It inherits from QThread to run the preparation process in a separate thread.
     """
 
-    signalPackagingProgress = pyqtSignal(float)
+    signalPackagingProgress = pyqtSignal(float, int)  # progress_percent, bytes_downloaded
 
     def __init__(self, parent=None):
         super().__init__(parent)
@@ -31,6 +56,11 @@ class PackagePrepareTask(QThread):
         self.__canceled = False
         self.lastError = None
 
+        # Track download progress across all assets
+        self.__download_total_expected = 0
+        self.__download_total_received = 0
+        self.__last_emitted_percent = None
+
     def startFromZip(self, module_package, zip_file: str):
         self.module_package = module_package
         self.from_zip_file = zip_file
@@ -60,6 +90,11 @@ class PackagePrepareTask(QThread):
             self.__destination_directory = self.__prepare_destination_directory()
             logger.info(f"Destination directory: {self.__destination_directory}")
 
+            # Reset progress tracking
+            self.__download_total_expected = 0
+            self.__download_total_received = 0
+            self.__last_emitted_percent = None
+
             self.__prepare_module_assets(self.module_package)
             self.lastError = None
 
@@ -71,24 +106,30 @@ class PackagePrepareTask(QThread):
     def __prepare_destination_directory(self):
         """
         Prepare the destination directory for the module package.
-        This method creates a temporary directory for the package.
+        This method creates a cache directory for the package downloads.
         """
-        temp_dir = PluginUtils.plugin_temp_path()
-        destination_directory = os.path.join(
-            temp_dir,
+        # Sanitize the package name to avoid filesystem issues (especially on Windows)
+        # PR names can contain special characters like # and spaces
+        safe_name = sanitize_filename(self.module_package.name)
+        cache_dir = os.path.join(
+            PluginUtils.plugin_cache_path(),
+            "pkgs",
             self.module_package.organisation,
             self.module_package.repository,
-            self.module_package.name,
+            safe_name,
         )
-        os.makedirs(destination_directory, exist_ok=True)
+        os.makedirs(cache_dir, exist_ok=True)
 
-        return destination_directory
+        return cache_dir
 
     def __prepare_module_assets(self, module_package):
 
+        # Pre-fetch all file sizes to calculate accurate total progress
+        self.__prefetch_download_sizes(module_package)
+
         # Download the source or use from zip
         zip_file = self.from_zip_file or self.__download_module_asset(
-            module_package.download_url, "source.zip"
+            module_package.download_url, "source.zip", module_package
         )
 
         module_package.source_package_zip = zip_file
@@ -101,6 +142,7 @@ class PackagePrepareTask(QThread):
             zip_file = self.__download_module_asset(
                 module_package.asset_project.download_url,
                 module_package.asset_project.type.value + ".zip",
+                module_package,
             )
             package_dir = self.__extract_zip_file(zip_file)
             module_package.asset_project.package_zip = zip_file
@@ -111,52 +153,225 @@ class PackagePrepareTask(QThread):
             zip_file = self.__download_module_asset(
                 module_package.asset_plugin.download_url,
                 module_package.asset_plugin.type.value + ".zip",
+                module_package,
             )
             package_dir = self.__extract_zip_file(zip_file)
             module_package.asset_plugin.package_zip = zip_file
             module_package.asset_plugin.package_dir = package_dir
 
-    def __download_module_asset(self, url: str, filename: str):
+    def __prefetch_download_sizes(self, module_package):
+        """Pre-fetch Content-Length for all files to be downloaded for accurate progress."""
+        urls_to_check = []
 
-        zip_file = os.path.join(self.__destination_directory, filename)
+        # Check source if not from zip
+        if self.from_zip_file is None:
+            # Only check if not already cached
+            cache_filename = self.__get_cache_filename("source.zip", module_package)
+            zip_file = os.path.join(self.__destination_directory, cache_filename)
+            if not self.__is_cached_and_valid(zip_file):
+                urls_to_check.append((module_package.download_url, "source.zip"))
+
+        # Check project asset
+        if module_package.asset_project is not None:
+            cache_filename = self.__get_cache_filename(
+                module_package.asset_project.type.value + ".zip", module_package
+            )
+            zip_file = os.path.join(self.__destination_directory, cache_filename)
+            if not self.__is_cached_and_valid(zip_file):
+                urls_to_check.append(
+                    (
+                        module_package.asset_project.download_url,
+                        module_package.asset_project.type.value + ".zip",
+                    )
+                )
+
+        # Check plugin asset
+        if module_package.asset_plugin is not None:
+            cache_filename = self.__get_cache_filename(
+                module_package.asset_plugin.type.value + ".zip", module_package
+            )
+            zip_file = os.path.join(self.__destination_directory, cache_filename)
+            if not self.__is_cached_and_valid(zip_file):
+                urls_to_check.append(
+                    (
+                        module_package.asset_plugin.download_url,
+                        module_package.asset_plugin.type.value + ".zip",
+                    )
+                )
+
+        # If everything is cached, skip size checking entirely
+        if not urls_to_check:
+            self.__download_total_expected = 0
+            return
+
+        total_size = 0
+        for url, filename in urls_to_check:
+            # Get Content-Length via HEAD request
+            try:
+                response = requests.head(url, allow_redirects=True, timeout=10)
+                content_length = response.headers.get("content-length")
+                if content_length:
+                    file_size = int(content_length)
+                    total_size += file_size
+                else:
+                    logger.warning(
+                        f"No Content-Length for '{filename}', progress may be inaccurate"
+                    )
+            except Exception as e:
+                logger.warning(f"Failed to get size for '{filename}': {e}")
+
+        self.__download_total_expected = total_size
+        if total_size > 0:
+            logger.info(f"Total download size: {total_size / (1024 * 1024):.1f} MB")
+
+    def __is_cached_and_valid(self, zip_file):
+        """Check if a zip file is cached and valid (quick check)."""
+        if not os.path.exists(zip_file):
+            return False
+        # Quick check: file exists and has reasonable size
+        try:
+            size = os.path.getsize(zip_file)
+            if size < 100:  # Too small to be valid
+                return False
+            # Just check if it opens as a valid zip, don't read entire contents
+            with zipfile.ZipFile(zip_file, "r") as zip_test:
+                # Quick check - just verify we can read the file list
+                _ = zip_test.namelist()
+            return True
+        except (zipfile.BadZipFile, OSError, Exception):
+            return False
+
+    def __get_cache_filename(self, base_filename: str, module_package):
+        """Generate cache filename, including commit SHA for branches/PRs if available."""
+        if module_package.type in (ModulePackage.Type.BRANCH, ModulePackage.Type.PULL_REQUEST):
+            if module_package.commit_sha:
+                # Include commit SHA in filename for cache invalidation
+                name, ext = os.path.splitext(base_filename)
+                return f"{name}-{module_package.commit_sha[:8]}{ext}"
+            else:
+                # No commit SHA available, don't cache (use unique name)
+                import time
+
+                name, ext = os.path.splitext(base_filename)
+                return f"{name}-{int(time.time())}{ext}"
+        return base_filename
+
+    def __download_module_asset(self, url: str, filename: str, module_package):
+
+        cache_filename = self.__get_cache_filename(filename, module_package)
+        zip_file = os.path.join(self.__destination_directory, cache_filename)
+
+        # Check if file already exists and is valid
+        if os.path.exists(zip_file):
+            try:
+                # Quick validation - just check if it's a valid zip structure
+                file_size = os.path.getsize(zip_file)
+                if file_size > 100:  # Has reasonable size
+                    with zipfile.ZipFile(zip_file, "r") as zip_test:
+                        # Just verify we can read file list, don't test entire contents
+                        _ = zip_test.namelist()
+                    logger.info(f"Using cached: {os.path.basename(zip_file)}")
+                    # Still emit some progress to show we're not stuck
+                    self.signalPackagingProgress.emit(-1.0, 0)
+                    return zip_file
+            except (zipfile.BadZipFile, OSError, Exception) as e:
+                logger.warning(f"Existing file '{zip_file}' is invalid ({e}), will re-download")
+                try:
+                    os.remove(zip_file)
+                except OSError:
+                    pass
 
         # Streaming, so we can iterate over the response.
-        response = requests.get(url, allow_redirects=True, stream=True)
+        timeout = 60
+        logger.info(f"Downloading: {os.path.basename(zip_file)}")
+
+        try:
+            response = requests.get(url, allow_redirects=True, stream=True, timeout=timeout)
+        except Exception as e:
+            logger.error(f"HTTP request failed: {e}")
+            raise
 
         # Raise an exception in case of http errors
         response.raise_for_status()
 
         self.__checkForCanceled()
 
-        logger.info(f"Downloading from '{url}' to '{zip_file}'")
-        data_size = 0
+        # Get total file size from headers
+        content_length = response.headers.get("content-length")
+        file_size = int(content_length) if content_length else 0
+
+        if file_size == 0:
+            # Emit indeterminate progress
+            self.signalPackagingProgress.emit(-1.0, 0)
+
+        downloaded_size = 0
         with open(zip_file, "wb") as file:
-            next_emit_threshold = 10 * 1024 * 1024  # 10MB threshold
-            for data in response.iter_content(chunk_size=None):
+            chunk_size = 256 * 1024  # 256KB chunks
+            for data in response.iter_content(chunk_size=chunk_size, decode_unicode=False):
                 file.write(data)
 
                 self.__checkForCanceled()
 
-                data_size += len(data)
-                if data_size >= next_emit_threshold:  # Emit signal when threshold is exceeded
-                    self.signalPackagingProgress.emit(data_size)
-                    next_emit_threshold += 10 * 1024 * 1024  # Update to the next threshold
+                chunk_len = len(data)
+                downloaded_size += chunk_len
+                self.__download_total_received += chunk_len
+
+                # Emit progress on percentage change
+                self.__emit_progress()
+
+        # Ensure final progress reflects completion
+        self.__emit_progress(force=True)
 
         return zip_file
 
+    def __emit_progress(self, force: bool = False):
+        """Emit download progress as percentage (0-100) or -1 for indeterminate."""
+        if self.__download_total_expected <= 0:
+            # Size unknown, emit indeterminate progress with bytes downloaded
+            self.signalPackagingProgress.emit(-1.0, self.__download_total_received)
+            return
+
+        percent = int((self.__download_total_received * 100) / self.__download_total_expected)
+        percent = max(0, min(100, percent))  # Clamp to 0-100
+
+        if force or self.__last_emitted_percent != percent:
+            self.__last_emitted_percent = percent
+            self.signalPackagingProgress.emit(float(percent), self.__download_total_received)
+
     def __extract_zip_file(self, zip_file):
         # Unzip the file to plugin temp dir
+        # Don't set indeterminate here - it confuses the progress when downloading multiple files
+
         try:
             with zipfile.ZipFile(zip_file, "r") as zip_ref:
-                # Find the top-level directory
+                # Find the top-level directory in the zip
                 zip_dirname = zip_ref.namelist()[0].split("/")[0]
-                package_dir = os.path.join(self.__destination_directory, zip_dirname)
-
-                if os.path.exists(package_dir):
-                    shutil.rmtree(package_dir)
 
+                # Use short "src" name to avoid Windows MAX_PATH issues
+                package_dir = os.path.join(self.__destination_directory, "src")
+
+                # Check if already extracted and valid
+                if os.path.exists(package_dir) and os.path.isdir(package_dir):
+                    # Verify it's not empty and has some expected content
+                    if os.listdir(package_dir):
+                        logger.info(
+                            f"Directory '{package_dir}' already extracted - skipping extraction"
+                        )
+                        return package_dir
+                    else:
+                        logger.warning(f"Directory '{package_dir}' is empty, will re-extract")
+                        shutil.rmtree(package_dir)
+
+                logger.info(f"Extracting '{zip_file}'...")
                 zip_ref.extractall(self.__destination_directory)
 
+                # Rename extracted dir to "src" to shorten paths
+                extracted_dir = os.path.join(self.__destination_directory, zip_dirname)
+                if extracted_dir != package_dir:
+                    shutil.move(extracted_dir, package_dir)
+
+                logger.info(f"Extraction complete: '{package_dir}'")
+
         except zipfile.BadZipFile:
             raise Exception(self.tr(f"The selected file '{zip_file}' is not a valid zip archive."))
 

oqtopus/gui/database_connection_widget.py

@@ -1,11 +1,11 @@
 import psycopg
-from pgserviceparser import conf_path as pgserviceparser_conf_path
-from pgserviceparser import service_config as pgserviceparser_service_config
-from pgserviceparser import service_names as pgserviceparser_service_names
 from qgis.PyQt.QtCore import pyqtSignal
 from qgis.PyQt.QtGui import QAction
 from qgis.PyQt.QtWidgets import QDialog, QMenu, QWidget
 
+from ..libs.pgserviceparser import conf_path as pgserviceparser_conf_path
+from ..libs.pgserviceparser import service_config as pgserviceparser_service_config
+from ..libs.pgserviceparser import service_names as pgserviceparser_service_names
 from ..utils.plugin_utils import PluginUtils, logger
 from ..utils.qt_utils import CriticalMessageBox, QtUtils
 from .database_create_dialog import DatabaseCreateDialog
@@ -78,8 +78,14 @@ class DatabaseConnectionWidget(QWidget, DIALOG_UI):
         self.db_services_comboBox.clear()
 
         try:
+            self.db_services_comboBox.addItem(self.tr("Please select a service"), None)
+            # Disable the placeholder item
+            model = self.db_services_comboBox.model()
+            item = model.item(0)
+            item.setEnabled(False)
+
             for service_name in pgserviceparser_service_names():
-                self.db_services_comboBox.addItem(service_name)
+                self.db_services_comboBox.addItem(service_name, service_name)
         except Exception as exception:
             CriticalMessageBox(
                 self.tr("Error"), self.tr("Can't load database services:"), exception, self
@@ -87,7 +93,8 @@ class DatabaseConnectionWidget(QWidget, DIALOG_UI):
             return
 
     def __serviceChanged(self, index=None):
-        if self.db_services_comboBox.currentText() == "":
+        # Check if placeholder is selected (currentData is None)
+        if self.db_services_comboBox.currentData() is None:
             self.db_database_label.setText(self.tr("No database"))
             QtUtils.setForegroundColor(self.db_database_label, PluginUtils.COLOR_WARNING)
             QtUtils.setFontItalic(self.db_database_label, True)

oqtopus/gui/database_create_dialog.py

@@ -23,12 +23,12 @@
 # ---------------------------------------------------------------------
 
 import psycopg
-from pgserviceparser import service_config as pgserviceparser_service_config
-from pgserviceparser import service_names as pgserviceparser_service_names
-from pgserviceparser import write_service as pgserviceparser_write_service
 from qgis.PyQt.QtCore import Qt
 from qgis.PyQt.QtWidgets import QDialog, QMessageBox
 
+from ..libs.pgserviceparser import service_config as pgserviceparser_service_config
+from ..libs.pgserviceparser import service_names as pgserviceparser_service_names
+from ..libs.pgserviceparser import write_service as pgserviceparser_write_service
 from ..utils.plugin_utils import PluginUtils, logger
 from ..utils.qt_utils import OverrideCursor
 

oqtopus/gui/database_duplicate_dialog.py

@@ -23,12 +23,12 @@
 # ---------------------------------------------------------------------
 
 import psycopg
-from pgserviceparser import full_config as pgserviceparser_full_config
-from pgserviceparser import service_config as pgserviceparser_service_config
-from pgserviceparser import write_service as pgserviceparser_write_service
 from qgis.PyQt.QtCore import Qt
 from qgis.PyQt.QtWidgets import QDialog, QMessageBox
 
+from ..libs.pgserviceparser import full_config as pgserviceparser_full_config
+from ..libs.pgserviceparser import service_config as pgserviceparser_service_config
+from ..libs.pgserviceparser import write_service as pgserviceparser_write_service
 from ..utils.plugin_utils import PluginUtils, logger
 from ..utils.qt_utils import OverrideCursor
 
@@ -90,7 +90,7 @@ class DatabaseDuplicateDialog(QDialog, DIALOG_UI):
             with OverrideCursor(Qt.CursorShape.WaitCursor):
                 with database_connection.cursor() as cursor:
                     cursor.execute(
-                        f"CREATE DATABASE {new_database_name} TEMPLATE {self.__existing_service_config.get('dbname')}"
+                        f'CREATE DATABASE "{new_database_name}" TEMPLATE "{self.__existing_service_config.get("dbname")}"'
                     )
         except psycopg.Error as e:
             errorText = self.tr(f"Error duplicating database:\n{e}.")

oqtopus/gui/logs_widget.py

@@ -1,13 +1,25 @@
 import logging
+from datetime import datetime
 
 from qgis.PyQt.QtCore import QAbstractItemModel, QModelIndex, QSortFilterProxyModel, Qt
-from qgis.PyQt.QtWidgets import QAbstractItemView, QApplication, QStyle, QWidget
-
+from qgis.PyQt.QtGui import QKeySequence
+from qgis.PyQt.QtWidgets import (
+    QAbstractItemView,
+    QApplication,
+    QShortcut,
+    QStyle,
+    QWidget,
+)
+
+# Import and register SQL logging level from pum
+from ..libs.pum import SQL
 from ..utils.plugin_utils import LoggingBridge, PluginUtils
 
-DIALOG_UI = PluginUtils.get_ui_class("logs_widget.ui")
+logging.addLevelName(SQL, "SQL")
 
 
+DIALOG_UI = PluginUtils.get_ui_class("logs_widget.ui")
+
 COLUMNS = ["Level", "Module", "Message"]
 
 
@@ -33,7 +45,7 @@ class LogModel(QAbstractItemModel):
         return len(COLUMNS)
 
     def data(self, index: QModelIndex, role: Qt.ItemDataRole = None):
-        if not index.isValid() or role != Qt.ItemDataRole.DisplayRole:
+        if not index.isValid():
             return None
         if (
             index.row() < 0
@@ -42,8 +54,18 @@ class LogModel(QAbstractItemModel):
             or index.column() >= len(COLUMNS)
         ):
             return None
+
         log = self.logs[index.row()]
-        return log[COLUMNS[index.column()]]
+        col_name = COLUMNS[index.column()]
+        value = log[col_name]
+
+        if role == Qt.ItemDataRole.DisplayRole:
+            return value
+        elif role == Qt.ItemDataRole.ToolTipRole:
+            # Show full text in tooltip, especially useful for long messages
+            if col_name == "Message":
+                return value
+        return None
 
     def index(self, row: int, column: int, parent=None):
         if row < 0 or row >= len(self.logs) or column < 0 or column >= len(COLUMNS):
@@ -69,7 +91,7 @@ class LogModel(QAbstractItemModel):
 
 
 class LogFilterProxyModel(QSortFilterProxyModel):
-    LEVELS = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
+    LEVELS = ["SQL", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
 
     def __init__(self, parent=None):
         super().__init__(parent)
@@ -123,13 +145,30 @@ class LogsWidget(QWidget, DIALOG_UI):
         self.logs_treeView.setModel(self.proxy_model)
         self.logs_treeView.setAlternatingRowColors(True)
         self.logs_treeView.setSelectionBehavior(QAbstractItemView.SelectionBehavior.SelectRows)
-        self.logs_treeView.setSelectionMode(QAbstractItemView.SelectionMode.SingleSelection)
+        self.logs_treeView.setSelectionMode(QAbstractItemView.SelectionMode.ExtendedSelection)
         self.logs_treeView.setEditTriggers(QAbstractItemView.EditTrigger.NoEditTriggers)
+
+        # Enable word wrapping for better readability
+        self.logs_treeView.setWordWrap(True)
+        self.logs_treeView.setTextElideMode(Qt.TextElideMode.ElideNone)
+
+        # Configure column widths
+        header = self.logs_treeView.header()
+        header.setStretchLastSection(True)  # Message column stretches to fill space
+        header.resizeSection(0, 100)  # Level column - fixed width
+        header.resizeSection(1, 150)  # Module column - fixed width
+        # Message column will take remaining space due to setStretchLastSection
+
+        # Enable automatic row height adjustment
+        self.logs_treeView.setUniformRowHeights(False)
+        header.setDefaultSectionSize(100)
+
         self.loggingBridge.loggedLine.connect(self.__logged_line)
         logging.getLogger().addHandler(self.loggingBridge)
 
         self.logs_level_comboBox.addItems(
             [
+                "SQL",
                 "DEBUG",
                 "INFO",
                 "WARNING",
@@ -154,13 +193,20 @@ class LogsWidget(QWidget, DIALOG_UI):
         self.logs_clear_toolButton.clicked.connect(self.__logsClearClicked)
         self.logs_filter_LineEdit.textChanged.connect(self.proxy_model.setFilterFixedString)
 
+        # Add copy shortcut (Ctrl+C)
+        self.copy_shortcut = QShortcut(QKeySequence.Copy, self.logs_treeView)
+        self.copy_shortcut.activated.connect(self.__copySelectedRows)
+
     def close(self):
         # uninstall the logging bridge
         logging.getLogger().removeHandler(self.loggingBridge)
 
     def __logged_line(self, record, line):
+        # Convert timestamp from record.created (epoch time) to readable format
+        timestamp = datetime.fromtimestamp(record.created).strftime("%Y-%m-%d %H:%M:%S")
 
         log_entry = {
+            "Timestamp": timestamp,
             "Level": record.levelname,
             "Module": record.name,
             "Message": record.msg,
@@ -180,3 +226,44 @@ class LogsWidget(QWidget, DIALOG_UI):
 
     def __logsClearClicked(self):
         self.logs_model.clear()
+
+    def __copySelectedRows(self):
+        """Copy selected rows to clipboard in CSV format."""
+        selection_model = self.logs_treeView.selectionModel()
+        selected_indexes = selection_model.selectedRows()
+
+        if not selected_indexes:
+            return
+
+        # Sort by row number to maintain order
+        selected_indexes.sort(key=lambda idx: idx.row())
+
+        # Build CSV content with header
+        csv_lines = ["Timestamp,Level,Module,Message"]
+
+        for proxy_index in selected_indexes:
+            # Map proxy index to source model index
+            source_index = self.proxy_model.mapToSource(proxy_index)
+            row = source_index.row()
+            log_entry = self.logs_model.logs[row]
+
+            # Escape fields that might contain commas or quotes
+            def escape_csv(value):
+                value = str(value)
+                if "," in value or '"' in value or "\n" in value:
+                    return '"' + value.replace('"', '""') + '"'
+                return value
+
+            csv_line = ",".join(
+                [
+                    escape_csv(log_entry["Timestamp"]),
+                    escape_csv(log_entry["Level"]),
+                    escape_csv(log_entry["Module"]),
+                    escape_csv(log_entry["Message"]),
+                ]
+            )
+            csv_lines.append(csv_line)
+
+        # Copy to clipboard
+        clipboard = QApplication.clipboard()
+        clipboard.setText("\n".join(csv_lines))