mercuto-client 0.3.4a1__tar.gz → 0.3.5__tar.gz

{mercuto_client-0.3.4a1 → mercuto_client-0.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mercuto-client
-Version: 0.3.4a1
+Version: 0.3.5
 Summary: Library for interfacing with Rockfield's Mercuto API
 Author-email: Daniel Whipp <daniel.whipp@rocktech.com.au>
 License-Expression: AGPL-3.0-only
@@ -22,6 +22,7 @@ Requires-Dist: pyftpdlib>=2.0.1
 Requires-Dist: python-dateutil>=2.9.0.post0
 Requires-Dist: pytz>=2025.2
 Requires-Dist: schedule>=1.2.2
+Requires-Dist: zc.lockfile>=4.0
 Requires-Dist: pydantic>=2.0
 Dynamic: license-file
 

mercuto_client-0.3.5/mercuto_client/_tests/test_ingester/test_backup.py

@@ -0,0 +1,98 @@
+import os
+import tempfile
+from pathlib import Path
+from threading import Thread
+from typing import Iterator, TypedDict
+from urllib.parse import urlparse
+
+import pyftpdlib.authorizers  # type: ignore[import-untyped]
+import pyftpdlib.handlers  # type: ignore[import-untyped]
+import pyftpdlib.servers  # type: ignore[import-untyped]
+import pytest
+
+from ...ingester.backup import FileBackup, FTPBackup
+
+
+def test_file_backup():
+    with tempfile.TemporaryDirectory() as temp_dir:
+        uri = Path(temp_dir).as_uri()
+        bak = FileBackup(urlparse(uri))
+        assert bak.process_file(__file__)
+
+
+def test_file_backup_does_not_exist_create_it():
+    with tempfile.TemporaryDirectory() as temp_dir:
+        test_dir = Path(temp_dir) / "test_dir"
+        uri = test_dir.as_uri() + "?create=true"
+        assert not test_dir.exists()
+        bak = FileBackup(urlparse(uri))
+        assert bak.process_file(__file__)
+        dest = test_dir / Path(__file__).name
+        assert test_dir.exists()
+        assert dest.exists()
+
+
+def test_file_backup_does_not_exist():
+    uri = (Path(__file__).parent / "I_DO_NOT_EXIST").as_uri()
+    with pytest.raises(ValueError, match="backup path does not exist"):
+        FileBackup(urlparse(uri))
+
+
+class TestFTPServerConfig(TypedDict):
+    host: str
+    port: int
+    user: str
+    passwd: str
+    ftp_root: str
+
+
+@pytest.fixture(scope="function")
+def mock_ftp_server() -> Iterator[TestFTPServerConfig]:
+    # Create a temporary directory to act as the FTP root
+    temp_dir = tempfile.TemporaryDirectory()
+    ftp_root = temp_dir.name
+
+    # Set up user authentication
+    authorizer = pyftpdlib.authorizers.DummyAuthorizer()
+    authorizer.add_user("user", "12345", ftp_root, perm="elradfmwMT")
+
+    # Set up FTP handler
+    handler = pyftpdlib.handlers.FTPHandler
+    handler.authorizer = authorizer
+
+    # Start the FTP server in a separate thread
+    server = pyftpdlib.servers.FTPServer(("127.0.0.1", 0), handler)
+    ip, port = server.socket.getsockname()
+
+    server_thread = Thread(target=server.serve_forever)
+    server_thread.daemon = True
+    server_thread.start()
+
+    yield {
+        "host": ip,
+        "port": port,
+        "user": "user",
+        "passwd": "12345",
+        "ftp_root": ftp_root
+    }
+
+    server.close_all()
+    temp_dir.cleanup()
+
+
+def test_ftp_backup(mock_ftp_server: TestFTPServerConfig):
+    url = f"ftp://{mock_ftp_server['user']}:{mock_ftp_server['passwd']}@{mock_ftp_server['host']}:{mock_ftp_server['port']}/my_dir?create=true"
+    os.makedirs(Path(mock_ftp_server['ftp_root']) / "my_dir", exist_ok=True)
+    bak = FTPBackup(urlparse(url))
+    bak.process_file(__file__)
+    assert bak.process_file(__file__)
+    dest = Path(mock_ftp_server['ftp_root']) / "my_dir" / Path(__file__).name
+    assert dest.exists()
+
+
+def test_ftp_backup_fails_if_dir_not_exists(mock_ftp_server: TestFTPServerConfig):
+    url = f"ftp://{mock_ftp_server['user']}:{mock_ftp_server['passwd']}@{mock_ftp_server['host']}:{mock_ftp_server['port']}/my_dir"
+    bak = FTPBackup(urlparse(url))
+    assert not bak.process_file(__file__)
+    dest = Path(mock_ftp_server['ftp_root']) / "my_dir" / Path(__file__).name
+    assert not dest.exists()

{mercuto_client-0.3.4a1 → mercuto_client-0.3.5}/mercuto_client/client.py

@@ -40,7 +40,7 @@ class MercutoClient:
         if url.endswith('/'):
             url = url[:-1]
 
-        if not url.startswith('https://'):
+        if verify_ssl and not url.startswith('https://'):
             raise ValueError(f'Url must be https, is {url}')
 
         self._url = url

mercuto_client-0.3.5/mercuto_client/ingester/__main__.py

@@ -0,0 +1,275 @@
+import argparse
+import logging
+import logging.handlers
+import os
+import signal
+import sys
+import time
+from pathlib import Path
+from typing import Any, Callable, Optional, TypeVar
+from urllib.parse import ParseResult, urlparse
+
+import schedule
+
+from ..util import get_free_space_excluding_files
+from .backup import get_backup_handler
+from .ftp import simple_ftp_server
+from .mercuto import MercutoIngester
+from .pid_file import PidFile
+from .processor import FileProcessor
+
+logger = logging.getLogger(__name__)
+
+
+T = TypeVar('T')
+
+
+def call_and_log_error(func: Callable[[], T]) -> T | None:
+    """
+    Call a function and log any exceptions that occur.
+    """
+    try:
+        return func()
+    except Exception:
+        logging.exception(f"Error in {func.__name__}")
+        return None
+
+
+class Status:
+    """
+    Status class to handle running state of the ingester.
+    """
+
+    def __init__(self):
+        self.running = True
+
+    def stop(self, code: Any, frame: Any):
+        self.running = False
+        print("Stopping")
+
+    def is_running(self):
+        return self.running
+
+
+def launch_mercuto_ingester(
+    project: str,
+    api_key: str,
+    hostname: str = 'https://api.rockfieldcloud.com.au',
+    verify_ssl: bool = True,
+    pid_file: Optional[Path] = None,
+    workdir: Optional[str] = '~/.mercuto-ingester',
+    verbose: bool = False,
+    logfile: Optional[str] = None,
+    directory: Optional[str] = None,
+    target_free_space_mb: Optional[float] = None,
+    max_files: Optional[int] = None,
+    mapping: Optional[str] = None,
+    clean: bool = False,
+    ftp_server_username: str = 'logger',
+    ftp_server_password: str = 'password',
+    ftp_server_port: int = 2121,
+    ftp_server_rename: bool = True,
+    max_attempts: int = 1000,
+    backup_location: Optional[list[ParseResult]] = None,
+    timezone: Optional[str] = None,
+):
+
+    if backup_location is None:
+        backup_location = []
+
+    with PidFile(pid_file):
+        if workdir is None:
+            workdir = os.path.join(os.path.expanduser('~'), ".mercuto-ingester")
+        elif workdir.startswith("~"):
+            workdir = os.path.expanduser(workdir)
+        else:
+            workdir = workdir
+            if not os.path.exists(workdir):
+                raise ValueError(f"Work directory {workdir} does not exist")
+
+        os.makedirs(workdir, exist_ok=True)
+
+        if verbose:
+            level = logging.DEBUG
+        else:
+            level = logging.INFO
+
+        handlers: list[logging.Handler] = []
+        handlers.append(logging.StreamHandler(sys.stderr))
+
+        if logfile is not None:
+            logfile = logfile
+        else:
+            logfile = os.path.join(workdir, 'log.txt')
+        handlers.append(logging.handlers.RotatingFileHandler(
+            logfile, maxBytes=1000000, backupCount=3))
+
+        logging.basicConfig(format='[PID %(process)d] %(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
+                            datefmt='%d/%m/%Y %H:%M:%S',
+                            level=level,
+                            handlers=handlers)
+
+        if directory is None:
+            buffer_directory = os.path.join(workdir, "buffered-files")
+        else:
+            buffer_directory = directory
+        os.makedirs(buffer_directory, exist_ok=True)
+
+        ftp_dir = os.path.join(workdir, 'temp-ftp-data')
+        os.makedirs(ftp_dir, exist_ok=True)
+
+        if target_free_space_mb is None and max_files is None:
+            target_free_space_mb = get_free_space_excluding_files(buffer_directory) * 0.25 // (1024 * 1024)  # Convert to MB
+            logging.info(f"Target remaining free space set to {target_free_space_mb} MB based on available disk space.")
+
+        logger.info(f"Using work directory: {workdir}")
+
+        database_path = os.path.join(workdir, "buffer.db")
+        if clean and os.path.exists(database_path):
+            logging.info(f"Dropping existing database at {database_path}")
+            os.remove(database_path)
+
+        ingester = MercutoIngester(
+            project_code=project,
+            api_key=api_key,
+            hostname=hostname,
+            verify_ssl=verify_ssl,
+            timezone=timezone
+        )
+
+        if mapping is not None:
+            import json
+            with open(mapping, 'r') as f:
+                mapping = json.load(f)
+            if not isinstance(mapping, dict):
+                raise ValueError(f"Mapping file {mapping} must contain a JSON object")
+            ingester.update_mapping(mapping)
+
+        pre_processing_handlers: list[Callable[[str], bool]] = [get_backup_handler(loc) for loc in backup_location]
+        processor_callbacks: list[Callable[[str], bool]] = [ingester.process_file]
+        post_processing_handlers: list[Callable[[str], bool]] = []
+
+        all_handlers = pre_processing_handlers + processor_callbacks + post_processing_handlers
+
+        processor = FileProcessor(
+            buffer_dir=buffer_directory,
+            db_path=database_path,
+            process_callback=lambda filename: all(handler(filename) for handler in all_handlers),
+            max_attempts=max_attempts,
+            target_free_space_mb=target_free_space_mb,
+            max_files=max_files)
+
+        processor.scan_existing_files()
+
+        with simple_ftp_server(directory=buffer_directory,
+                               username=ftp_server_username, password=ftp_server_password, port=ftp_server_port,
+                               callback=processor.add_file_to_db, rename=ftp_server_rename,
+                               workdir=ftp_dir):
+            call_and_log_error(ingester.ping)
+            schedule.every(60).seconds.do(call_and_log_error, ingester.ping)  # type: ignore[attr-defined]
+            schedule.every(5).seconds.do(call_and_log_error, processor.process_next_file)  # type: ignore[attr-defined]
+            schedule.every(2).minutes.do(call_and_log_error, processor.cleanup_old_files)  # type: ignore[attr-defined]
+
+            status = Status()
+            signal.signal(signal.SIGTERM, status.stop)
+
+            while status.is_running():
+                schedule.run_pending()
+                sleep_period = schedule.idle_seconds()
+                if sleep_period is None or sleep_period < 0:
+                    sleep_period = 0
+                # We need to wake up to handle ctrl-c etc
+                if sleep_period > 1:
+                    sleep_period = 1
+                time.sleep(sleep_period)
+
+            logger.warning("Shutting Down...")
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Mercuto Ingester CLI')
+    parser.add_argument('-p', '--project', type=str,
+                        required=True, help='Mercuto project code')
+    parser.add_argument('-k', '--api-key', type=str,
+                        required=True, help='API key for Mercuto')
+    parser.add_argument('-v', '--verbose', action='store_true',
+                        help='Enable verbose output')
+    parser.add_argument('-d', '--directory', type=str,
+                        help='Directory to store ingested files. Default is a directory called `buffered-files` in the workdir.')
+    parser.add_argument('-s', '--target-free-space-mb', type=int,
+                        help='Size in MB for total amount of remaining free space to keep available. \
+                            Default is 25% of the available disk space on the buffer partition excluding the directory itself', default=None)
+    parser.add_argument('--max-files', type=int,
+                        help='Maximum number of files to keep in the buffer. Default is to use the size param.', default=None)
+    parser.add_argument('--max-attempts', type=int,
+                        help='Maximum number of attempts to process a file before giving up. Default is 1000.',
+                        default=1000)
+    parser.add_argument('--workdir', type=str,
+                        help='Working directory for the ingester. Default is ~/.mercuto-ingester',)
+    parser.add_argument('--logfile', type=str,
+                        help='Log file path. No logs written if not provided. Maximum of 4 log files of 1MB each will be kept.\
+                            Default is log.txt in the workdir.')
+    parser.add_argument('--mapping', type=str,
+                        help='Path to a JSON file with channel label to channel code mapping.\
+                            If not provided, the ingester will try to detect the channels from the project.',
+                        default=None)
+    parser.add_argument('--hostname', type=str,
+                        help='Hostname to use for the Mercuto server. Default is "https://api.rockfieldcloud.com.au".',
+                        default='https://api.rockfieldcloud.com.au')
+    parser.add_argument('--clean',
+                        help='Drop the database before starting. This will not remove any buffer files and will rescan them on startup.',
+                        action='store_true')
+    parser.add_argument('--username', type=str,
+                        help='Username for the FTP server. Default is "logger".',
+                        default='logger')
+    parser.add_argument('--password', type=str,
+                        help='Password for the FTP server. Default is "password".',
+                        default='password')
+    parser.add_argument('--port', type=int,
+                        help='Port for the FTP server. Default is 2121.',
+                        default=2121)
+    parser.add_argument('--no-rename', action='store_true',
+                        help='Add the current timestamp to the end of the files received via FTP. \
+                        This is useful to avoid overwriting files with the same name.')
+    parser.add_argument('-i', '--insecure', action="store_true",
+                        help='Disable SSL verification',
+                        default=False)
+    parser.add_argument('-b', '--backup-location', action="append",
+                        help='Backup location to store ingested files. Must be a valid URL, e.g. scp://user@host/path. '
+                        'Can be specified multiple times.',
+                        type=urlparse)
+    parser.add_argument('-e', '--pid-file', help='Ths location to create the PID file', type=Path, default=None)
+    parser.add_argument('--timezone', type=str,
+                        help='Timezone to use for data uploads (e.g. "Australia/Melbourne"). \
+                        If not provided, no timezone will be sent on uploads. \
+                        Only needed if data files do not contain timezone information (E.g. Campbell Scientific loggers).',
+                        default=None)
+
+    args = parser.parse_args()
+
+    launch_mercuto_ingester(
+        project=args.project,
+        api_key=args.api_key,
+        verify_ssl=not args.insecure,
+        pid_file=args.pid_file,
+        workdir=args.workdir,
+        verbose=args.verbose,
+        logfile=args.logfile,
+        directory=args.directory,
+        target_free_space_mb=args.target_free_space_mb,
+        max_files=args.max_files,
+        mapping=args.mapping,
+        clean=args.clean,
+        ftp_server_username=args.username,
+        ftp_server_password=args.password,
+        ftp_server_port=args.port,
+        ftp_server_rename=not args.no_rename,
+        max_attempts=args.max_attempts,
+        backup_location=args.backup_location,
+        hostname=args.hostname,
+        timezone=args.timezone
+    )
+
+
+if __name__ == '__main__':
+    main()