gns3-server 3.0.1__py3-none-any.whl → 3.0.3__py3-none-any.whl

gns3server/controller/__init__.py

@@ -28,10 +28,10 @@ try:
 except ImportError:
     from importlib import resources as importlib_resources
 
-
 from ..config import Config
 from ..utils import parse_version, md5sum
 from ..utils.images import default_images_directory
+from ..utils.asyncio import wait_run_in_executor
 
 from .project import Project
 from .appliance import Appliance
@@ -43,6 +43,7 @@ from .topology import load_topology
 from .gns3vm import GNS3VM
 from .gns3vm.gns3_vm_error import GNS3VMError
 from .controller_error import ControllerError, ControllerNotFoundError
+from ..db.tasks import update_disk_checksums
 from ..version import __version__
 
 import logging
@@ -72,8 +73,11 @@ class Controller:
     async def start(self, computes=None):
 
         log.info("Controller is starting")
-        self._install_base_configs()
-        self._install_builtin_disks()
+        await self._install_base_configs()
+        installed_disks = await self._install_builtin_disks()
+        if installed_disks:
+            await update_disk_checksums(installed_disks)
+
         server_config = Config.instance().settings.Server
         Config.instance().listen_for_config_changes(self._update_config)
         name = server_config.name
@@ -86,7 +90,7 @@ class Controller:
         if host == "0.0.0.0":
             host = "127.0.0.1"
 
-        self._load_controller_vars()
+        await self._load_controller_vars()
 
         if server_config.enable_ssl:
             self._ssl_context = self._create_ssl_context(server_config)
@@ -190,7 +194,7 @@ class Controller:
     async def reload(self):
 
         log.info("Controller is reloading")
-        self._load_controller_vars()
+        await self._load_controller_vars()
 
         # remove all projects deleted from disk.
         for project in self._projects.copy().values():
@@ -234,7 +238,7 @@ class Controller:
         except OSError as e:
             log.error(f"Cannot write controller vars file '{self._vars_file}': {e}")
 
-    def _load_controller_vars(self):
+    async def _load_controller_vars(self):
         """
         Reload the controller vars from disk
         """
@@ -274,9 +278,9 @@ class Controller:
             builtin_appliances_path = self._appliance_manager.builtin_appliances_path()
             if not previous_version or \
                     parse_version(__version__.split("+")[0]) > parse_version(previous_version.split("+")[0]):
-                self._appliance_manager.install_builtin_appliances()
+                await self._appliance_manager.install_builtin_appliances()
             elif not os.listdir(builtin_appliances_path):
-                self._appliance_manager.install_builtin_appliances()
+                await self._appliance_manager.install_builtin_appliances()
             else:
                 log.info(f"Built-in appliances are installed in '{builtin_appliances_path}'")
 
@@ -307,18 +311,21 @@ class Controller:
 
 
     @staticmethod
-    def install_resource_files(dst_path, resource_name, upgrade_resources=True):
+    async def install_resource_files(dst_path, resource_name, upgrade_resources=True):
         """
         Install files from resources to user's file system
         """
 
-        def should_copy(src, dst, upgrade_resources):
+        installed_resources = []
+        async def should_copy(src, dst, upgrade_resources):
             if not os.path.exists(dst):
                 return True
             if upgrade_resources is False:
                 return False
             # copy the resource if it is different
-            return md5sum(src) != md5sum(dst)
+            src_md5 = await wait_run_in_executor(md5sum, src)
+            dst_md5 = await wait_run_in_executor(md5sum, dst)
+            return src_md5 != dst_md5
 
         if hasattr(sys, "frozen") and sys.platform.startswith("win"):
             resource_path = os.path.normpath(os.path.join(os.path.dirname(sys.executable), resource_name))
@@ -328,14 +335,16 @@ class Controller:
         else:
             for entry in importlib_resources.files('gns3server').joinpath(resource_name).iterdir():
                 full_path = os.path.join(dst_path, entry.name)
-                if entry.is_file() and should_copy(str(entry), full_path, upgrade_resources):
+                if entry.is_file() and await should_copy(str(entry), full_path, upgrade_resources):
                     log.debug(f'Installing {resource_name} resource file "{entry.name}" to "{full_path}"')
-                    shutil.copy(str(entry), os.path.join(dst_path, entry.name))
+                    shutil.copy(str(entry), os.path.join(full_path))
+                    installed_resources.append(full_path)
                 elif entry.is_dir():
                     os.makedirs(full_path, exist_ok=True)
-                    Controller.install_resource_files(full_path, os.path.join(resource_name, entry.name))
+                    await Controller.install_resource_files(full_path, os.path.join(resource_name, entry.name))
+        return installed_resources
 
-    def _install_base_configs(self):
+    async def _install_base_configs(self):
         """
         At startup we copy base configs to the user location to allow
         them to customize it
@@ -345,11 +354,11 @@ class Controller:
         log.info(f"Installing base configs in '{dst_path}'")
         try:
             # do not overwrite base configs because they may have been customized by the user
-            Controller.install_resource_files(dst_path, "configs", upgrade_resources=False)
+            await Controller.install_resource_files(dst_path, "configs", upgrade_resources=False)
         except OSError as e:
             log.error(f"Could not install base config files to {dst_path}: {e}")
 
-    def _install_builtin_disks(self):
+    async def _install_builtin_disks(self):
         """
         At startup we copy built-in Qemu disks to the user location to allow
         them to use with appliances
@@ -358,7 +367,7 @@ class Controller:
         dst_path = self.disks_path()
         log.info(f"Installing built-in disks in '{dst_path}'")
         try:
-            Controller.install_resource_files(dst_path, "disks")
+            return await Controller.install_resource_files(dst_path, "disks")
         except OSError as e:
             log.error(f"Could not install disk files to {dst_path}: {e}")
 

gns3server/controller/appliance_manager.py

@@ -110,7 +110,7 @@ class ApplianceManager:
         os.makedirs(appliances_dir, exist_ok=True)
         return appliances_dir
 
-    def install_builtin_appliances(self):
+    async def install_builtin_appliances(self):
         """
         At startup we copy the built-in appliances files.
         """
@@ -119,7 +119,7 @@ class ApplianceManager:
         log.info(f"Installing built-in appliances in '{dst_path}'")
         from . import Controller
         try:
-            Controller.instance().install_resource_files(dst_path, "appliances")
+            await Controller.instance().install_resource_files(dst_path, "appliances")
         except OSError as e:
             log.error(f"Could not install built-in appliance files to {dst_path}: {e}")
 

gns3server/controller/compute.py

@@ -18,14 +18,19 @@
 import ipaddress
 import aiohttp
 import asyncio
-import async_timeout
 import socket
 import json
 import sys
 import io
+
 from fastapi import HTTPException
 from aiohttp import web
 
+if sys.version_info >= (3, 11):
+    from asyncio import timeout as asynctimeout
+else:
+    from async_timeout import timeout as asynctimeout
+
 from ..utils import parse_version
 from ..utils.asyncio import locking
 from ..controller.controller_error import (
@@ -503,7 +508,7 @@ class Compute:
         return self._getUrl(path)
 
     async def _run_http_query(self, method, path, data=None, timeout=120, raw=False):
-        async with async_timeout.timeout(delay=timeout):
+        async with asynctimeout(delay=timeout):
             url = self._getUrl(path)
             headers = {"content-type": "application/json"}
             chunked = None

gns3server/crash_report.py

@@ -58,7 +58,7 @@ class CrashReport:
     Report crash to a third party service
     """
 
-    DSN = "https://847198b87dbd50ef8962901641918a08@o19455.ingest.us.sentry.io/38482"
+    DSN = "https://2c96fa0280f82c48108f122b87cd902c@o19455.ingest.us.sentry.io/38482"
     _instance = None
 
     def __init__(self):

gns3server/db/repositories/images.py

@@ -18,7 +18,7 @@
 import os
 
 from typing import Optional, List
-from sqlalchemy import select, delete
+from sqlalchemy import select, delete, update
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from .base import BaseRepository
@@ -103,6 +103,22 @@ class ImagesRepository(BaseRepository):
         await self._db_session.refresh(db_image)
         return db_image
 
+    async def update_image(self, image_path: str, checksum: str, checksum_algorithm: str) -> models.Image:
+        """
+        Update an image.
+        """
+
+        query = update(models.Image).\
+            where(models.Image.path == image_path).\
+            values(checksum=checksum, checksum_algorithm=checksum_algorithm)
+
+        await self._db_session.execute(query)
+        await self._db_session.commit()
+        image_db = await self.get_image_by_checksum(checksum)
+        if image_db:
+            await self._db_session.refresh(image_db)  # force refresh of updated_at value
+        return image_db
+
     async def delete_image(self, image_path: str) -> bool:
         """
         Delete an image.
@@ -119,7 +135,7 @@ class ImagesRepository(BaseRepository):
         await self._db_session.commit()
         return result.rowcount > 0
 
-    async def prune_images(self) -> int:
+    async def prune_images(self, skip_images: list[str] = None) -> int:
         """
         Prune images not attached to any template.
         """
@@ -130,12 +146,15 @@ class ImagesRepository(BaseRepository):
         images = result.scalars().all()
         images_deleted = 0
         for image in images:
+            if skip_images and image.filename in skip_images:
+                log.debug(f"Skipping image '{image.path}' for pruning")
+                continue
             try:
                 log.debug(f"Deleting image '{image.path}'")
                 os.remove(image.path)
             except OSError:
                 log.warning(f"Could not delete image file {image.path}")
-            if await self.delete_image(image.filename):
+            if await self.delete_image(image.path):
                 images_deleted += 1
         log.info(f"{images_deleted} image(s) have been deleted")
         return images_deleted

gns3server/db/repositories/templates.py

@@ -170,3 +170,14 @@ class TemplatesRepository(BaseRepository):
             await self._db_session.commit()
             await self._db_session.refresh(template_in_db)
         return template_in_db
+
+    async def get_template_images(self, template_id: UUID) -> List[models.Image]:
+        """
+        Return all images attached to a template.
+        """
+
+        query = select(models.Image).\
+            join(models.Image.templates).\
+            filter(models.Template.template_id == template_id)
+        result = await self._db_session.execute(query)
+        return result.scalars().all()

gns3server/db/tasks.py

@@ -16,13 +16,11 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import asyncio
-import signal
+import time
 import os
 
 from fastapi import FastAPI
 from pydantic import ValidationError
-from watchfiles import awatch, Change
-
 from typing import List
 from sqlalchemy import event
 from sqlalchemy.engine import Engine
@@ -32,10 +30,13 @@ from alembic import command, config
 from alembic.script import ScriptDirectory
 from alembic.runtime.migration import MigrationContext
 from alembic.util.exc import CommandError
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEvent, PatternMatchingEventHandler
 
 from gns3server.db.repositories.computes import ComputesRepository
 from gns3server.db.repositories.images import ImagesRepository
-from gns3server.utils.images import discover_images, check_valid_image_header, read_image_info, default_images_directory, InvalidImageError
+from gns3server.utils.images import md5sum, discover_images, read_image_info, InvalidImageError
+from gns3server.utils.asyncio import wait_run_in_executor
 from gns3server import schemas
 
 from .models import Base
@@ -130,81 +131,7 @@ async def get_computes(app: FastAPI) -> List[dict]:
     return computes
 
 
-def image_filter(change: Change, path: str) -> bool:
-
-    if change == Change.added and os.path.isfile(path):
-        if path.endswith(".tmp") or path.endswith(".md5sum") or path.startswith("."):
-            return False
-        if "/lib/" in path or "/lib64/" in path:
-            # ignore custom IOU libraries
-            return False
-        header_magic_len = 7
-        with open(path, "rb") as f:
-            image_header = f.read(header_magic_len)  # read the first 7 bytes of the file
-            if len(image_header) >= header_magic_len:
-                try:
-                    check_valid_image_header(image_header)
-                except InvalidImageError as e:
-                    log.debug(f"New image '{path}': {e}")
-                    return False
-            else:
-                log.debug(f"New image '{path}': size is too small to be valid")
-                return False
-        return True
-    # FIXME: should we support image deletion?
-    # elif change == Change.deleted:
-    #     return True
-    return False
-
-
-async def monitor_images_on_filesystem(app: FastAPI):
-
-    directories_to_monitor = []
-    for image_type in ("qemu", "ios", "iou"):
-        image_dir = default_images_directory(image_type)
-        if os.path.isdir(image_dir):
-            log.debug(f"Monitoring for new images in '{image_dir}'")
-            directories_to_monitor.append(image_dir)
-
-    try:
-        async for changes in awatch(
-                *directories_to_monitor,
-                watch_filter=image_filter,
-                raise_interrupt=True
-        ):
-            async with AsyncSession(app.state._db_engine) as db_session:
-                images_repository = ImagesRepository(db_session)
-                for change in changes:
-                    change_type, image_path = change
-                    if change_type == Change.added:
-                        try:
-                            image = await read_image_info(image_path)
-                        except InvalidImageError as e:
-                            log.warning(str(e))
-                            continue
-                        try:
-                            if await images_repository.get_image(image_path):
-                                continue
-                            await images_repository.add_image(**image)
-                            log.info(f"Discovered image '{image_path}' has been added to the database")
-                        except SQLAlchemyError as e:
-                            log.warning(f"Error while adding image '{image_path}' to the database: {e}")
-                    # if change_type == Change.deleted:
-                    #     try:
-                    #         if await images_repository.get_image(image_path):
-                    #             success = await images_repository.delete_image(image_path)
-                    #             if not success:
-                    #                 log.warning(f"Could not delete image '{image_path}' from the database")
-                    #             else:
-                    #                 log.info(f"Image '{image_path}' has been deleted from the database")
-                    #     except SQLAlchemyError as e:
-                    #         log.warning(f"Error while deleting image '{image_path}' from the database: {e}")
-    except KeyboardInterrupt:
-        # send SIGTERM to the server PID so uvicorn can shutdown the process
-        os.kill(os.getpid(), signal.SIGTERM)
-
-
-async def discover_images_on_filesystem(app: FastAPI):
+async def discover_images_on_filesystem(app: FastAPI) -> None:
 
     async with AsyncSession(app.state._db_engine) as db_session:
         images_repository = ImagesRepository(db_session)
@@ -228,3 +155,117 @@ async def discover_images_on_filesystem(app: FastAPI):
 
     # monitor if images have been manually added
     asyncio.create_task(monitor_images_on_filesystem(app))
+
+
+async def update_disk_checksums(updated_disks: List[str]) -> None:
+    """
+    Update the checksum of a list of disks in the database.
+
+    :param updated_disks: list of updated disks
+    """
+
+    from gns3server.api.server import app
+    async with AsyncSession(app.state._db_engine) as db_session:
+        images_repository = ImagesRepository(db_session)
+        for path in updated_disks:
+            image = await images_repository.get_image(path)
+            if image:
+                log.info(f"Updating image '{path}' in the database")
+                checksum = await wait_run_in_executor(md5sum, path, cache_to_md5file=False)
+                if image.checksum != checksum:
+                    await images_repository.update_image(path, checksum, "md5")
+
+class EventHandler(PatternMatchingEventHandler):
+    """
+    Watchdog event handler.
+    """
+
+    def __init__(self, queue: asyncio.Queue, loop: asyncio.BaseEventLoop, **kwargs):
+
+        self._loop = loop
+        self._queue = queue
+
+        # ignore temporary files, md5sum files, hidden files and directories
+        super().__init__(ignore_patterns=["*.tmp", "*.md5sum", ".*"], ignore_directories = True, **kwargs)
+
+    def on_closed(self, event: FileSystemEvent) -> None:
+        # monitor for closed files (e.g. when a file has finished to be copied)
+        if "/lib/" in event.src_path or "/lib64/" in event.src_path:
+            return  # ignore custom IOU libraries
+        self._loop.call_soon_threadsafe(self._queue.put_nowait, event)
+
+class EventIterator(object):
+    """
+    Watchdog Event iterator.
+    """
+
+    def __init__(self, queue: asyncio.Queue):
+        self.queue = queue
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+
+        item = await self.queue.get()
+        if item is None:
+            raise StopAsyncIteration
+        return item
+
+async def monitor_images_on_filesystem(app: FastAPI):
+
+    def watchdog(
+            path: str,
+            queue: asyncio.Queue,
+            loop: asyncio.BaseEventLoop,
+            app: FastAPI, recursive: bool = False
+    ) -> None:
+        """
+        Thread to monitor a directory for new images.
+        """
+
+        handler = EventHandler(queue, loop)
+        observer = Observer()
+        observer.schedule(handler, str(path), recursive=recursive)
+        observer.start()
+        log.info(f"Monitoring for new images in '{path}'")
+        while True:
+            time.sleep(1)
+            # stop when the app is exiting
+            if app.state.exiting:
+                observer.stop()
+                observer.join(10)
+                log.info(f"Stopping monitoring for new images in '{path}'")
+                loop.call_soon_threadsafe(queue.put_nowait, None)
+                break
+
+    queue = asyncio.Queue()
+    loop = asyncio.get_event_loop()
+    server_config = Config.instance().settings.Server
+    image_dir = os.path.expanduser(server_config.images_path)
+    asyncio.get_event_loop().run_in_executor(None, watchdog,image_dir, queue, loop, app, True)
+
+    async for filesystem_event in EventIterator(queue):
+        # read the file system event from the queue
+        image_path = filesystem_event.src_path
+        expected_image_type = None
+        if "IOU" in image_path:
+            expected_image_type = "iou"
+        elif "QEMU" in image_path:
+            expected_image_type = "qemu"
+        elif "IOS" in image_path:
+            expected_image_type = "ios"
+        async with AsyncSession(app.state._db_engine) as db_session:
+            images_repository = ImagesRepository(db_session)
+            try:
+                image = await read_image_info(image_path, expected_image_type)
+            except InvalidImageError as e:
+                log.warning(str(e))
+                continue
+            try:
+                if await images_repository.get_image(image_path):
+                    continue
+                await images_repository.add_image(**image)
+                log.info(f"Discovered image '{image_path}' has been added to the database")
+            except SQLAlchemyError as e:
+                log.warning(f"Error while adding image '{image_path}' to the database: {e}")

gns3server/main.py

@@ -30,6 +30,7 @@ import gns3server.utils.get_resource
 import os
 import sys
 import asyncio
+import argparse
 
 
 def daemonize():
@@ -59,6 +60,42 @@ def daemonize():
         print("Second fork failed: %d (%s)\n" % (e.errno, e.strerror), file=sys.stderr)
         sys.exit(1)
 
+def parse_arguments(argv):
+    """
+    Parse command line arguments
+
+    :param argv: Array of command line arguments
+    """
+    from gns3server.version import __version__
+    parser = argparse.ArgumentParser(description=f"GNS3 server version {__version__}")
+    parser.add_argument("-v", "--version", help="show the version", action="version", version=__version__)
+    parser.add_argument("--host", help="run on the given host/IP address")
+    parser.add_argument("--port", help="run on the given port", type=int)
+    parser.add_argument("--ssl", action="store_true", help="run in SSL mode")
+    parser.add_argument("--config", help="Configuration file")
+    parser.add_argument("--certfile", help="SSL cert file")
+    parser.add_argument("--certkey", help="SSL key file")
+    parser.add_argument("-L", "--local", action="store_true", help="local mode (allows some insecure operations)")
+    parser.add_argument(
+        "-A", "--allow", action="store_true", help="allow remote connections to local console ports"
+    )
+    parser.add_argument("-q", "--quiet", default=False, action="store_true", help="do not show logs on stdout")
+    parser.add_argument("-d", "--debug", default=False, action="store_true", help="show debug logs")
+    parser.add_argument("--logfile", "--log", help="send output to logfile instead of console")
+    parser.add_argument("--logmaxsize", default=10000000, help="maximum logfile size in bytes (default is 10MB)")
+    parser.add_argument(
+        "--logbackupcount", default=10, help="number of historical log files to keep (default is 10)"
+    )
+    parser.add_argument(
+        "--logcompression", default=False, action="store_true", help="compress inactive (historical) logs"
+    )
+    parser.add_argument("--daemon", action="store_true", help="start as a daemon")
+    parser.add_argument("--pid", help="store process pid")
+    parser.add_argument("--profile", help="Settings profile (blank will use default settings files)")
+
+    args = parser.parse_args(argv)
+    return parser, args
+
 
 def main():
     """
@@ -69,10 +106,11 @@ def main():
         raise SystemExit("Windows is not a supported platform to run the GNS3 server")
     if "--daemon" in sys.argv:
         daemonize()
-    from gns3server.server import Server
 
     try:
-        asyncio.run(Server().run())
+        parser, args = parse_arguments(sys.argv[1:])
+        from gns3server.server import Server
+        asyncio.run(Server().run(parser, args))
     except KeyboardInterrupt:
         pass