mercuto-client 0.2.5__tar.gz → 0.2.7__tar.gz

{mercuto_client-0.2.5 → mercuto_client-0.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mercuto-client
-Version: 0.2.5
+Version: 0.2.7
 Summary: Library for interfacing with Rockfield's Mercuto API
 Author-email: Daniel Whipp <daniel.whipp@rocktech.com.au>
 Keywords: mercuto,rockfield,infratech

{mercuto_client-0.2.5 → mercuto_client-0.2.7}/mercuto_client/ingester/__main__.py

@@ -1,6 +1,4 @@
 import argparse
-import fnmatch
-import itertools
 import logging
 import logging.handlers
 import os
@@ -10,135 +8,13 @@ from typing import Callable, TypeVar
 
 import schedule
 
-from .. import MercutoClient, MercutoHTTPException
-from ..types import DataSample
 from .ftp import simple_ftp_server
-from .parsers import detect_parser
+from .mercuto import MercutoIngester
 from .processor import FileProcessor
-from .util import batched, get_free_space_excluding_files, get_my_public_ip
+from .util import get_free_space_excluding_files
 
 logger = logging.getLogger(__name__)
 
-NON_RETRYABLE_ERRORS = {400, 404, 409}  # HTTP status codes that indicate non-retryable errors
-
-
-class MercutoIngester:
-    def __init__(self, project_code: str, api_key: str, hostname: str = 'https://api.rockfieldcloud.com.au') -> None:
-        self._client = MercutoClient(url=hostname)
-        self._api_key = api_key
-        with self._client.as_credentials(api_key=api_key) as client:
-            self._project = client.projects().get_project(project_code)
-            assert self._project['code'] == project_code
-
-            self._secondary_channels = client.channels().get_channels(project_code, classification='SECONDARY')
-            self._datatables = list(itertools.chain.from_iterable([dt['datatables'] for dt in client.devices().list_dataloggers(project_code)]))
-
-        self._channel_map = {c['label']: c['code'] for c in self._secondary_channels}
-
-    def update_mapping(self, mapping: dict[str, str]) -> None:
-        """
-        Update the channel label to channel code mapping.
-        """
-        self._channel_map.update(mapping)
-        logger.info(f"Updated channel mapping: {self._channel_map}")
-
-    @property
-    def project_code(self) -> str:
-        return self._project['code']
-
-    def ping(self) -> None:
-        """
-        Ping the Mercuto serverto update the last seen IP address.
-        """
-        ip = get_my_public_ip()
-        with self._client.as_credentials(api_key=self._api_key) as client:
-            client.projects().ping_project(self.project_code, ip_address=ip)
-            logging.info(f"Pinged Mercuto server from IP: {ip} for project: {self.project_code}")
-
-    def matching_datatable(self, filename: str) -> str | None:
-        """
-        Check if any datatables on the project match this file name.
-        Returns the datatable code if a match is found, otherwise None.
-        """
-        basename = os.path.basename(filename)
-
-        def matches(test: str) -> bool:
-            """
-            test should be a pattern or a filename.
-            E.g. "my_data.csv" or "my_data*.csv", or "/path/to/my_data*.csv"
-            Do wildcard matching as well as prefix matching.
-            """
-            test_base = os.path.basename(test)
-            if fnmatch.fnmatch(basename, test_base):
-                return True
-            lhs, _ = os.path.splitext(test_base)
-            if basename.startswith(lhs):
-                return True
-            return False
-
-        for dt in self._datatables:
-            # Match using datatable pattern
-            if matches(dt['name']):
-                return dt['code']
-            if dt['src'] and matches(dt['src']):
-                return dt['code']
-        return None
-
-    def _upload_samples(self, samples: list[DataSample]) -> bool:
-        """
-        Upload samples to the Mercuto project.
-        """
-        try:
-            with self._client.as_credentials(api_key=self._api_key) as client:
-                for batch in batched(samples, 500):
-                    client.data().upload_samples(batch)
-            return True
-        except MercutoHTTPException as e:
-            if e.status_code in NON_RETRYABLE_ERRORS:
-                logger.exception(
-                    "Error indicates bad file that should not be retried. Skipping.")
-                return True
-            else:
-                return False
-
-    def _upload_file(self, file_path: str, datatable_code: str) -> bool:
-        """
-        Upload a file to the Mercuto project.
-        """
-        logging.info(f"Uploadeding file {file_path} to datatable {datatable_code} in project {self.project_code}")
-        try:
-            with self._client.as_credentials(api_key=self._api_key) as client:
-                client.data().upload_file(
-                    project=self.project_code,
-                    datatable=datatable_code,
-                    file=file_path,
-                )
-            return True
-        except MercutoHTTPException as e:
-            if e.status_code in NON_RETRYABLE_ERRORS:
-                logger.exception(
-                    "Error indicates bad file that should not be retried. Skipping.")
-                return True
-            else:
-                return False
-
-    def process_file(self, file_path: str) -> bool:
-        """
-        Process the received file.
-        """
-        logging.info(f"Processing file: {file_path}")
-        datatable_code = self.matching_datatable(file_path)
-        if datatable_code:
-            logger.info(f"Matched datatable code: {datatable_code} for file: {file_path}")
-            return self._upload_file(file_path, datatable_code)
-        else:
-            parser = detect_parser(file_path)
-            samples = parser(file_path, self._channel_map)
-            if not samples:
-                logging.warning(f"No samples found in file: {file_path}")
-                return True
-            return self._upload_samples(samples)
-
 
 T = TypeVar('T')
 
@@ -277,7 +153,7 @@ if __name__ == '__main__':
     with simple_ftp_server(directory=buffer_directory,
                            username=args.username, password=args.password, port=args.port,
                            callback=processor.add_file_to_db, rename=not args.no_rename,
-                           workdir=workdir):
+                           workdir=ftp_dir):
         schedule.every(60).seconds.do(call_and_log_error, ingester.ping)
         schedule.every(5).seconds.do(call_and_log_error, processor.process_next_file)
         schedule.every(2).minutes.do(call_and_log_error, processor.cleanup_old_files)

mercuto_client-0.2.7/mercuto_client/ingester/mercuto.py

@@ -0,0 +1,155 @@
+import fnmatch
+import itertools
+import logging
+import os
+from typing import Optional
+
+from .. import MercutoClient, MercutoHTTPException
+from ..types import Channel, DataSample, DatatableOut, Project
+from .parsers import detect_parser
+from .util import batched, get_my_public_ip
+
+logger = logging.getLogger(__name__)
+
+NON_RETRYABLE_ERRORS = {400, 404, 409}  # HTTP status codes that indicate non-retryable errors
+
+
+class MercutoIngester:
+    def __init__(self, project_code: str, api_key: str, hostname: str = 'https://api.rockfieldcloud.com.au') -> None:
+        self._client = MercutoClient(url=hostname)
+        self._api_key = api_key
+        self._project_code = project_code
+
+        self._project: Optional[Project] = None
+        self._secondary_channels: Optional[list[Channel]] = None
+        self._datatables: Optional[list[DatatableOut]] = None
+
+        self._channel_map: dict[str, str] = {}
+
+    def _refresh_mercuto_data(self) -> None:
+        with self._client.as_credentials(api_key=self._api_key) as client:
+            self._project = client.projects().get_project(self._project_code)
+            assert self._project['code'] == self._project_code
+
+            self._secondary_channels = client.channels().get_channels(self._project_code, classification='SECONDARY')
+            self._datatables = list(itertools.chain.from_iterable([dt['datatables'] for dt in client.devices().list_dataloggers(self._project_code)]))
+
+        self._channel_map.update({c['label']: c['code'] for c in self._secondary_channels})
+
+    def _can_process(self) -> bool:
+        return self._project is not None and self._secondary_channels is not None and self._datatables is not None
+
+    def update_mapping(self, mapping: dict[str, str]) -> None:
+        """
+        Update the channel label to channel code mapping.
+        """
+        self._channel_map.update(mapping)
+        logger.info(f"Updated channel mapping: {self._channel_map}")
+
+    @property
+    def project_code(self) -> str:
+        return self._project_code
+
+    def ping(self) -> None:
+        """
+        Ping the Mercuto serverto update the last seen IP address.
+        """
+        ip = get_my_public_ip()
+        with self._client.as_credentials(api_key=self._api_key) as client:
+            client.projects().ping_project(self.project_code, ip_address=ip)
+            logging.info(f"Pinged Mercuto server from IP: {ip} for project: {self.project_code}")
+
+    def matching_datatable(self, filename: str) -> str | None:
+        """
+        Check if any datatables on the project match this file name.
+        Returns the datatable code if a match is found, otherwise None.
+        """
+        if self._datatables is None:
+            raise ValueError("Datatables not loaded. Call _refresh_mercuto_data() first.")
+
+        basename = os.path.basename(filename)
+
+        def matches(test: str) -> bool:
+            """
+            test should be a pattern or a filename.
+            E.g. "my_data.csv" or "my_data*.csv", or "/path/to/my_data*.csv"
+            Do wildcard matching as well as prefix matching.
+            """
+            test_base = os.path.basename(test)
+            if fnmatch.fnmatch(basename, test_base):
+                return True
+            lhs, _ = os.path.splitext(test_base)
+            if basename.startswith(lhs):
+                return True
+            return False
+
+        for dt in self._datatables:
+            # Match using datatable pattern
+            if matches(dt['name']):
+                return dt['code']
+            if dt['src'] and matches(dt['src']):
+                return dt['code']
+        return None
+
+    def _upload_samples(self, samples: list[DataSample]) -> bool:
+        """
+        Upload samples to the Mercuto project.
+        """
+        try:
+            with self._client.as_credentials(api_key=self._api_key) as client:
+                for batch in batched(samples, 500):
+                    client.data().upload_samples(batch)
+            return True
+        except MercutoHTTPException as e:
+            if e.status_code in NON_RETRYABLE_ERRORS:
+                logger.exception(
+                    "Error indicates bad file that should not be retried. Skipping.")
+                return True
+            else:
+                return False
+
+    def _upload_file(self, file_path: str, datatable_code: str) -> bool:
+        """
+        Upload a file to the Mercuto project.
+        """
+        logging.info(f"Uploadeding file {file_path} to datatable {datatable_code} in project {self.project_code}")
+        try:
+            with self._client.as_credentials(api_key=self._api_key) as client:
+                client.data().upload_file(
+                    project=self.project_code,
+                    datatable=datatable_code,
+                    file=file_path,
+                )
+            return True
+        except MercutoHTTPException as e:
+            if e.status_code in NON_RETRYABLE_ERRORS:
+                logger.exception(
+                    "Error indicates bad file that should not be retried. Skipping.")
+                return True
+            else:
+                return False
+
+    def process_file(self, file_path: str) -> bool:
+        """
+        Process the received file.
+        """
+
+        if not self._can_process():
+            logging.info("Refreshing Mercuto data...")
+            self._refresh_mercuto_data()
+            if not self._can_process():
+                logging.error("Failed to refresh Mercuto data. Cannot process file yet.")
+                return False
+
+        logging.info(f"Processing file: {file_path}")
+        datatable_code = self.matching_datatable(file_path)
+        if datatable_code:
+            logger.info(f"Matched datatable code: {datatable_code} for file: {file_path}")
+            return self._upload_file(file_path, datatable_code)
+        else:
+            parser = detect_parser(file_path)
+            samples = parser(file_path, self._channel_map)
+            if not samples:
+                logging.warning(f"No samples found in file: {file_path}")
+                return True
+            return self._upload_samples(samples)

{mercuto_client-0.2.5 → mercuto_client-0.2.7}/mercuto_client.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mercuto-client
-Version: 0.2.5
+Version: 0.2.7
 Summary: Library for interfacing with Rockfield's Mercuto API
 Author-email: Daniel Whipp <daniel.whipp@rocktech.com.au>
 Keywords: mercuto,rockfield,infratech

{mercuto_client-0.2.5 → mercuto_client-0.2.7}/mercuto_client.egg-info/SOURCES.txt

@@ -24,6 +24,7 @@ mercuto_client/_tests/test_ingester/test_parsers.py
 mercuto_client/ingester/__init__.py
 mercuto_client/ingester/__main__.py
 mercuto_client/ingester/ftp.py
+mercuto_client/ingester/mercuto.py
 mercuto_client/ingester/processor.py
 mercuto_client/ingester/util.py
 mercuto_client/ingester/parsers/__init__.py

{mercuto_client-0.2.5 → mercuto_client-0.2.7}/pyproject.toml

@@ -10,7 +10,7 @@ dependencies = [
     "pytz>=2025.2",
     "schedule>=1.2.2",
 ]
-version = "0.2.5"
+version = "0.2.7"
 keywords = ["mercuto", "rockfield", "infratech"]
 classifiers = [
   "Development Status :: 3 - Alpha",