mercuto-client 0.1.0__py3-none-any.whl

mercuto_client/exceptions.py

@@ -0,0 +1,15 @@
+import json
+
+
+class MercutoClientException(Exception):
+    pass
+
+
+class MercutoHTTPException(MercutoClientException):
+    def __init__(self, message: str, status_code: int) -> None:
+        super().__init__(message)
+        self.status_code = status_code
+        self.message = message
+
+    def json(self) -> dict:
+        return json.loads(self.message)

mercuto_client/ingester/__main__.py

@@ -0,0 +1,287 @@
+import argparse
+import fnmatch
+import itertools
+import logging
+import logging.handlers
+import os
+import sys
+import time
+from typing import Callable, TypeVar
+
+import schedule
+
+from .. import MercutoClient, MercutoHTTPException
+from ..types import DataSample
+from .ftp import simple_ftp_server
+from .parsers import detect_parser
+from .processor import FileProcessor
+from .util import batched, get_free_space_excluding_files, get_my_public_ip
+
+logger = logging.getLogger(__name__)
+
+NON_RETRYABLE_ERRORS = {400, 404, 409}  # HTTP status codes that indicate non-retryable errors
+
+
+class MercutoIngester:
+    def __init__(self, project_code: str, api_key: str, hostname: str = 'https://api.rockfieldcloud.com.au') -> None:
+        self._client = MercutoClient(url=hostname)
+        self._api_key = api_key
+        with self._client.as_credentials(api_key=api_key) as client:
+            self._project = client.projects().get_project(project_code)
+            assert self._project['code'] == project_code
+
+            self._secondary_channels = client.channels().get_channels(project_code, classification='SECONDARY')
+            self._datatables = list(itertools.chain.from_iterable([dt['datatables'] for dt in client.devices().list_dataloggers(project_code)]))
+
+        self._channel_map = {c['label']: c['code'] for c in self._secondary_channels}
+
+    def update_mapping(self, mapping: dict[str, str]) -> None:
+        """
+        Update the channel label to channel code mapping.
+        """
+        self._channel_map.update(mapping)
+        logger.info(f"Updated channel mapping: {self._channel_map}")
+
+    @property
+    def project_code(self) -> str:
+        return self._project['code']
+
+    def ping(self) -> None:
+        """
+        Ping the Mercuto serverto update the last seen IP address.
+        """
+        ip = get_my_public_ip()
+        with self._client.as_credentials(api_key=self._api_key) as client:
+            client.projects().ping_project(self.project_code, ip_address=ip)
+            logging.info(f"Pinged Mercuto server from IP: {ip} for project: {self.project_code}")
+
+    def matching_datatable(self, filename: str) -> str | None:
+        """
+        Check if any datatables on the project match this file name.
+        Returns the datatable code if a match is found, otherwise None.
+        """
+        basename = os.path.basename(filename)
+
+        def matches(test: str) -> bool:
+            """
+            test should be a pattern or a filename.
+            E.g. "my_data.csv" or "my_data*.csv", or "/path/to/my_data*.csv"
+            Do wildcard matching as well as prefix matching.
+            """
+            test_base = os.path.basename(test)
+            if fnmatch.fnmatch(basename, test_base):
+                return True
+            lhs, _ = os.path.splitext(test_base)
+            if basename.startswith(lhs):
+                return True
+            return False
+
+        for dt in self._datatables:
+            # Match using datatable pattern
+            if matches(dt['name']):
+                return dt['code']
+            if dt['src'] and matches(dt['src']):
+                return dt['code']
+        return None
+
+    def _upload_samples(self, samples: list[DataSample]) -> bool:
+        """
+        Upload samples to the Mercuto project.
+        """
+        try:
+            with self._client.as_credentials(api_key=self._api_key) as client:
+                for batch in batched(samples, 500):
+                    client.data().upload_samples(batch)
+            return True
+        except MercutoHTTPException as e:
+            if e.status_code in NON_RETRYABLE_ERRORS:
+                logger.exception(
+                    "Error indicates bad file that should not be retried. Skipping.")
+                return True
+            else:
+                return False
+
+    def _upload_file(self, file_path: str, datatable_code: str) -> bool:
+        """
+        Upload a file to the Mercuto project.
+        """
+        logging.info(f"Uploadeding file {file_path} to datatable {datatable_code} in project {self.project_code}")
+        try:
+            with self._client.as_credentials(api_key=self._api_key) as client:
+                client.data().upload_file(
+                    project=self.project_code,
+                    datatable=datatable_code,
+                    file=file_path,
+                )
+            return True
+        except MercutoHTTPException as e:
+            if e.status_code in NON_RETRYABLE_ERRORS:
+                logger.exception(
+                    "Error indicates bad file that should not be retried. Skipping.")
+                return True
+            else:
+                return False
+
+    def process_file(self, file_path: str) -> bool:
+        """
+        Process the received file.
+        """
+        logging.info(f"Processing file: {file_path}")
+        datatable_code = self.matching_datatable(file_path)
+        if datatable_code:
+            logger.info(f"Matched datatable code: {datatable_code} for file: {file_path}")
+            return self._upload_file(file_path, datatable_code)
+        else:
+            parser = detect_parser(file_path)
+            samples = parser(file_path, self._channel_map)
+            if not samples:
+                logging.warning(f"No samples found in file: {file_path}")
+                return True
+            return self._upload_samples(samples)
+
+
+T = TypeVar('T')
+
+
+def call_and_log_error(func: Callable[[], T]) -> T | None:
+    """
+    Call a function and log any exceptions that occur.
+    """
+    try:
+        return func()
+    except Exception:
+        logging.exception(f"Error in {func.__name__}")
+        return None
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Mercuto Ingester CLI')
+    parser.add_argument('-p', '--project', type=str,
+                        required=True, help='Mercuto project code')
+    parser.add_argument('-k', '--api-key', type=str,
+                        required=True, help='API key for Mercuto')
+    parser.add_argument('-v', '--verbose', action='store_true',
+                        help='Enable verbose output')
+    parser.add_argument('-d', '--directory', type=str,
+                        help='Directory to store ingested files. Default is a directory called `buffered-files` in the workdir.')
+    parser.add_argument('-s', '--size', type=int,
+                        help='Size in MB for total amount of files to store in the buffer. \
+                            Default is 75% of the available disk space on the buffer partition excluding the directory itself', default=None)
+    parser.add_argument('--max-attempts', type=int,
+                        help='Maximum number of attempts to process a file before giving up. Default is 1000.',
+                        default=1000)
+    parser.add_argument('--workdir', type=str,
+                        help='Working directory for the ingester. Default is ~/.mercuto-ingester',)
+    parser.add_argument('--logfile', type=str,
+                        help='Log file path. No logs written if not provided. Maximum of 4 log files of 1MB each will be kept.\
+                            Default is log.txt in the workdir.')
+    parser.add_argument('--mapping', type=str,
+                        help='Path to a JSON file with channel label to channel code mapping.\
+                            If not provided, the ingester will try to detect the channels from the project.',
+                        default=None)
+    parser.add_argument('--hostname', type=str,
+                        help='Hostname to use for the Mercuto server. Default is "https://api.rockfieldcloud.com.au".',
+                        default='https://api.rockfieldcloud.com.au')
+    parser.add_argument('--clean',
+                        help='Drop the database before starting. This will not remove any buffer files and will rescan them on startup.',
+                        action='store_true')
+    parser.add_argument('--username', type=str,
+                        help='Username for the FTP server. Default is "logger".',
+                        default='logger')
+    parser.add_argument('--password', type=str,
+                        help='Password for the FTP server. Default is "password".',
+                        default='password')
+    parser.add_argument('--port', type=int,
+                        help='Port for the FTP server. Default is 2121.',
+                        default=2121)
+    parser.add_argument('--no-rename', action='store_true',
+                        help='Add the current timestamp to the end of the files received via FTP. \
+                        This is useful to avoid overwriting files with the same name.')
+
+    args = parser.parse_args()
+
+    if args.workdir is None:
+        workdir = os.path.join(os.path.expanduser('~'), ".mercuto-ingester")
+    else:
+        workdir = args.workdir
+        if not os.path.exists(args.workdir):
+            raise ValueError(f"Work directory {args.workdir} does not exist")
+    os.makedirs(workdir, exist_ok=True)
+
+    if args.verbose:
+        level = logging.DEBUG
+    else:
+        level = logging.INFO
+
+    handlers: list[logging.Handler] = []
+    handlers.append(logging.StreamHandler(sys.stderr))
+
+    if args.logfile is not None:
+        logfile = args.logfile
+    else:
+        logfile = os.path.join(workdir, 'log.txt')
+    handlers.append(logging.handlers.RotatingFileHandler(
+        logfile, maxBytes=1000000, backupCount=3))
+
+    logging.basicConfig(format='[PID %(process)d] %(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
+                        datefmt='%d/%m/%Y %H:%M:%S',
+                        level=level,
+                        handlers=handlers)
+
+    if args.directory is None:
+        buffer_directory = os.path.join(workdir, "buffered-files")
+    else:
+        buffer_directory = args.directory
+    os.makedirs(buffer_directory, exist_ok=True)
+
+    ftp_dir = os.path.join(workdir, 'temp-ftp-data')
+    os.makedirs(ftp_dir, exist_ok=True)
+
+    size = args.size
+    if size is None:
+        size = get_free_space_excluding_files(buffer_directory) * 0.75 // (1024 * 1024)  # Convert to MB
+        logging.info(f"Buffer size set to {size} MB based on available disk space.")
+
+    if args.mapping is not None:
+        import json
+        with open(args.mapping, 'r') as f:
+            mapping = json.load(f)
+        if not isinstance(mapping, dict):
+            raise ValueError(f"Mapping file {args.mapping} must contain a JSON object")
+    else:
+        mapping = {}
+
+    logger.info(f"Using work directory: {workdir}")
+
+    database_path = os.path.join(workdir, "buffer.db")
+    if args.clean and os.path.exists(database_path):
+        logging.info(f"Dropping existing database at {database_path}")
+        os.remove(database_path)
+
+    ingester = MercutoIngester(
+        project_code=args.project,
+        api_key=args.api_key,
+        hostname=args.hostname)
+
+    ingester.update_mapping(mapping)
+
+    processor = FileProcessor(
+        buffer_dir=buffer_directory,
+        db_path=database_path,
+        process_callback=ingester.process_file,
+        max_attempts=args.max_attempts,
+        free_space_mb=size)
+
+    processor.scan_existing_files()
+
+    with simple_ftp_server(directory=buffer_directory,
+                           username=args.username, password=args.password, port=args.port,
+                           callback=processor.add_file_to_db, rename=not args.no_rename,
+                           workdir=workdir):
+        schedule.every(60).seconds.do(call_and_log_error, ingester.ping)
+        schedule.every(5).seconds.do(call_and_log_error, processor.process_next_file)
+        schedule.every(2).minutes.do(call_and_log_error, processor.cleanup_old_files)
+
+        while True:
+            schedule.run_pending()
+            time.sleep(0.5)

mercuto_client/ingester/ftp.py

@@ -0,0 +1,115 @@
+import contextlib
+import logging
+import os
+import shutil
+import tempfile
+import threading
+from datetime import datetime, timezone
+from typing import Callable, Iterator, Optional
+
+from pyftpdlib import authorizers  # type: ignore[import-untyped]
+from pyftpdlib.handlers import FTPHandler  # type: ignore[import-untyped]
+from pyftpdlib.servers import FTPServer  # type: ignore[import-untyped]
+
+logger = logging.getLogger(__name__)
+
+
+@contextlib.contextmanager
+def simple_ftp_server(directory: str,
+                      username: str,
+                      password: str,
+                      port: int = 2121,
+                      callback: Optional[Callable[[str], None]] = None,
+                      workdir: Optional[str] = None,
+                      rename: bool = True,
+                      clock: Optional[Callable[[], datetime]] = None) -> Iterator[None]:
+    """
+    Wrapper for a simple FTP server that allows uploading files to a specified directory.
+    Callback function can be provided which is called with the destination path of each uploaded file.
+    Files are first uploaded to a workdirectory and then moved to the specified directory.
+    If workdir is not specified, a temporary directory is used.
+
+    :param directory: Directory where files will be uploaded.
+    :param username: Username for FTP authentication.
+    :param password: Password for FTP authentication.
+    :param port: Port on which the FTP server will listen.
+    :param callback: Optional callback function that is called with the destination path of each uploaded file.
+    :param workdir: Optional working directory where files are initially uploaded before moving to the final directory.
+    :param rename: If True, appends a timestamp to the filename to avoid overwriting existing files.
+    :param clock: Function to get the current time, defaults to datetime.now with timezone UTC
+    :return: Context manager that starts the FTP server and allows file uploads.
+
+    Runs in a background thread during context manager usage.
+
+    Example usage:
+
+    ```python
+    def my_callback(dest_path: str):
+        print(f"File uploaded to: {dest_path}")
+
+    with simple_ftp_server('/path/to/upload/dir', 'user', 'pass', port=2121, callback=my_callback) as server:
+        # Your code here, e.g. processing files
+        while True:
+            time.sleep(10)
+    ```
+    """
+
+    if clock is None:
+        def clock(): return datetime.now(timezone.utc)
+
+    def rename_file(file_path: str) -> str:
+        """
+        Rename the file by appending a timestamp to avoid overwriting.
+        Adds the timestamp before the file extension.
+        """
+        base, ext = os.path.splitext(file_path)
+        timestamp = clock().strftime("%Y%m%dT%H%M%S")
+        new_name = f"{base}_{timestamp}{ext}"
+        return new_name
+
+    class CustomFTPHandler(FTPHandler):
+        def on_file_received(self, file):
+            target = os.path.join(directory, os.path.basename(file))
+
+            if rename:
+                target = rename_file(target)
+
+            dest = shutil.move(file, target)
+            if callback:
+                callback(dest)
+
+        def on_incomplete_file_received(self, file):
+            try:
+                os.remove(file)
+            except Exception:
+                logger.exception(f"Failed to remove incomplete file: {file}")
+
+    if workdir is None:
+        workdir_ctx: contextlib.AbstractContextManager[str] = tempfile.TemporaryDirectory(prefix="ftp_workdir_")
+    else:
+        workdir_ctx = contextlib.nullcontext(workdir)
+
+    with workdir_ctx as workdir:
+        authorizer = authorizers.DummyAuthorizer()
+        authorizer.add_user(username, password,
+                            workdir, perm='lwe')
+        handler = CustomFTPHandler
+        handler.authorizer = authorizer
+        handler.banner = "FTP Server Ready."
+        handler.passive_ports = range(60000, 65535)
+
+        address = ('0.0.0.0', port)
+        server = FTPServer(address, handler)
+        server.max_cons = 60
+        server.max_cons_per_ip = 20
+
+        server_thread = threading.Thread(target=server.serve_forever, daemon=True)
+        logger.debug(f"Starting FTP server on {port}...")
+        server_thread.start()
+        try:
+            yield
+        finally:
+            logger.debug("Stopping FTP server...")
+            server.close_all()
+            server_thread.join(timeout=10)
+            logger.debug("FTP server stopped.")

mercuto_client/ingester/parsers/__init__.py

@@ -0,0 +1,42 @@
+from typing import Optional, Protocol
+
+import pytz
+
+from ...types import DataSample
+from .campbell import parse_campbell_file
+from .worldsensing import (parse_worldsensing_compact_file,
+                           parse_worldsensing_standard_file)
+
+
+class Parser(Protocol):
+    def __call__(self, filename: str, label_to_channel_code: dict[str, str],
+                 timezone: Optional[pytz.BaseTzInfo] = None) -> list[DataSample]:
+        """
+        Parse the file and return a list of DataSample objects.
+        """
+        pass
+
+
+def detect_parser(filename: str) -> Parser:
+    """
+    Detect the type of the file based on its content.
+    Returns one of: "worldsensing_compact", "worldsensing_standard", "campbell", or raises ValueError if unknown.
+    """
+    with open(filename, 'r') as f:
+        first_line = f.readline().strip()
+        if first_line.startswith('"TOA5",'):
+            return parse_campbell_file
+        elif first_line.startswith('"Datalogger","compacted"'):
+            return parse_worldsensing_compact_file
+        elif first_line.startswith('"Node ID",'):
+            return parse_worldsensing_standard_file
+        else:
+            raise ValueError(f"Unknown file type for {filename}")
+
+
+__all__ = [
+    "parse_campbell_file",
+    "parse_worldsensing_standard_file",
+    "parse_worldsensing_compact_file",
+    "detect_parser",
+]

mercuto_client/ingester/parsers/campbell.py

@@ -0,0 +1,12 @@
+from typing import Optional
+
+import pytz
+
+from ...types import DataSample
+from .generic_csv import parse_generic_csv_file
+
+
+def parse_campbell_file(filename: str, label_to_channel_code: dict[str, str],
+                        timezone: Optional[pytz.BaseTzInfo] = None) -> list[DataSample]:
+    return parse_generic_csv_file(
+        filename, label_to_channel_code, header_index=1, data_index=2, timezone=timezone)

mercuto_client/ingester/parsers/generic_csv.py

@@ -0,0 +1,114 @@
+import logging
+from datetime import datetime
+from typing import Optional
+
+import pytz
+from dateutil import parser
+
+from ...types import DataSample
+
+logger = logging.getLogger(__name__)
+
+
+def _clean(s: str) -> str:
+    s = s.strip()
+    if s.startswith('"') and s.endswith('"'):
+        s = s[1:-1]
+    return s
+
+
+def _clean_number(s: str) -> float | None:
+    cleaned = _clean(s)
+    if cleaned == 'NAN' or cleaned == 'N/A' or cleaned == 'NaN' or cleaned == 'nan' or cleaned == 'Nan':
+        return float('nan')
+    try:
+        return float(cleaned)
+    except ValueError:
+        return None
+
+
+def _parse_header(header: str) -> list[str]:
+    columns = [h.strip() for h in header.strip().split(",")]
+    if columns[0] not in ('"TIMESTAMP"', 'timestamp', 'TIMESTAMP', '"Date-and-time"'):
+        raise ValueError(
+            f"Invalid header found: {columns[0]}, expecting TIMESTAMP.")
+    # Columns have quotes around them, remove them
+    return [_clean(c) for c in columns[1:]]
+
+
+def _parse_csv_line(line: str, sep: str = ',', timestamp_index: int = 0) -> tuple[datetime, list[float | None]]:
+    """
+    Returns timestamp, values
+    """
+    values = line.strip().split(sep)
+    if len(values) < 2:
+        raise ValueError(f"Invalid number of values found: {len(values)}")
+    # First value is timestamp
+    timestamp = _clean(values[timestamp_index])
+    try:
+        dt = parser.parse(timestamp)
+    except ValueError as e:
+        raise ValueError(
+            f"Failed to parse timestamp: {timestamp} - {e}") from e
+    # Rest are values
+    return dt, [_clean_number(v) for v in values[timestamp_index+1:]]
+
+
+def parse_generic_csv_file(filename: str, label_to_channel_code: dict[str, str],
+                           header_index: int, data_index: int,
+                           timezone: Optional[pytz.BaseTzInfo] = None) -> list[DataSample]:
+    """
+    header index: Number of lines to skip before header
+    data index: Number of lines to skip after the header before data
+
+    We are avoiding using pandas here to keep dependencies minimal as this is often run on edge devices.
+    """
+
+    output: list[DataSample] = []
+    with open(filename, "r") as f:
+        for _ in range(header_index):
+            next(f, None)
+        header = next(f, None)
+        if header is None:
+            logging.error(f"Failed to read header from file: {filename}")
+            return []
+        try:
+            header_columns = _parse_header(header)
+        except ValueError as e:
+            logging.error(f"Failed to parse header: {e}")
+            return []
+
+        # Next 2 lines are metadata, skip
+        for _ in range(data_index):
+            next(f, None)
+        while (line := next(f, None)):
+            try:
+                timestamp, line_values = _parse_csv_line(line)
+            except ValueError as e:
+                logging.error(f"Failed to parse line: {e}")
+                return []
+
+            if len(header_columns) != len(line_values):
+                logging.error(
+                    f"Invalid number of values found: {len(line_values)}, expected {len(header_columns)}")
+                return []
+
+            if timezone is not None and timestamp.tzinfo is None:
+                timestamp = timezone.localize(timestamp)
+
+            for header, value in zip(header_columns, line_values):
+                if header not in label_to_channel_code:
+                    logger.error(f"Label not found in table map: {header}")
+                    continue
+
+                if value is None:
+                    logger.error(
+                        f"Failed to parse value: {value} for column {header}")
+                    continue
+                channel_code = label_to_channel_code[header]
+
+                logger.debug(
+                    f"Adding entry for label: {header} with value: {value} and timestamp: {timestamp}")
+                output.append(DataSample(timestamp=timestamp.isoformat(),
+                              channel_code=channel_code, value=value))
+    return output

mercuto_client/ingester/parsers/worldsensing.py

@@ -0,0 +1,23 @@
+from typing import Optional
+
+import pytz
+
+from .generic_csv import DataSample, parse_generic_csv_file
+
+
+def parse_worldsensing_standard_file(filename: str, label_to_channel_code: dict[str, str],
+                                     timezone: Optional[pytz.BaseTzInfo] = None) -> list[DataSample]:
+    """
+    Parse a worldsensing standard CSV file provided when downloading data or using standard CSV export.
+    """
+    return parse_generic_csv_file(
+        filename, label_to_channel_code, header_index=9, data_index=0, timezone=timezone)
+
+
+def parse_worldsensing_compact_file(filename: str, label_to_channel_code: dict[str, str],
+                                    timezone: Optional[pytz.BaseTzInfo] = None) -> list[DataSample]:
+    """
+    Parse a worldsensing custom CSV file. These are generated when using compacted CSV mechanism.
+    """
+    return parse_generic_csv_file(
+        filename, label_to_channel_code, header_index=1, data_index=0, timezone=timezone)