mercuto-client 0.2.6.dev0__py3-none-any.whl → 0.2.8__py3-none-any.whl

mercuto_client/ingester/__main__.py

@@ -1,6 +1,4 @@
 import argparse
-import fnmatch
-import itertools
 import logging
 import logging.handlers
 import os
@@ -10,135 +8,13 @@ from typing import Callable, TypeVar
 
 import schedule
 
-from .. import MercutoClient, MercutoHTTPException
-from ..types import DataSample
 from .ftp import simple_ftp_server
-from .parsers import detect_parser
+from .mercuto import MercutoIngester
 from .processor import FileProcessor
-from .util import batched, get_free_space_excluding_files, get_my_public_ip
+from .util import get_free_space_excluding_files
 
 logger = logging.getLogger(__name__)
 
-NON_RETRYABLE_ERRORS = {400, 404, 409}  # HTTP status codes that indicate non-retryable errors
-
-
-class MercutoIngester:
-    def __init__(self, project_code: str, api_key: str, hostname: str = 'https://api.rockfieldcloud.com.au') -> None:
-        self._client = MercutoClient(url=hostname)
-        self._api_key = api_key
-        with self._client.as_credentials(api_key=api_key) as client:
-            self._project = client.projects().get_project(project_code)
-            assert self._project['code'] == project_code
-
-            self._secondary_channels = client.channels().get_channels(project_code, classification='SECONDARY')
-            self._datatables = list(itertools.chain.from_iterable([dt['datatables'] for dt in client.devices().list_dataloggers(project_code)]))
-
-        self._channel_map = {c['label']: c['code'] for c in self._secondary_channels}
-
-    def update_mapping(self, mapping: dict[str, str]) -> None:
-        """
-        Update the channel label to channel code mapping.
-        """
-        self._channel_map.update(mapping)
-        logger.info(f"Updated channel mapping: {self._channel_map}")
-
-    @property
-    def project_code(self) -> str:
-        return self._project['code']
-
-    def ping(self) -> None:
-        """
-        Ping the Mercuto serverto update the last seen IP address.
-        """
-        ip = get_my_public_ip()
-        with self._client.as_credentials(api_key=self._api_key) as client:
-            client.projects().ping_project(self.project_code, ip_address=ip)
-            logging.info(f"Pinged Mercuto server from IP: {ip} for project: {self.project_code}")
-
-    def matching_datatable(self, filename: str) -> str | None:
-        """
-        Check if any datatables on the project match this file name.
-        Returns the datatable code if a match is found, otherwise None.
-        """
-        basename = os.path.basename(filename)
-
-        def matches(test: str) -> bool:
-            """
-            test should be a pattern or a filename.
-            E.g. "my_data.csv" or "my_data*.csv", or "/path/to/my_data*.csv"
-            Do wildcard matching as well as prefix matching.
-            """
-            test_base = os.path.basename(test)
-            if fnmatch.fnmatch(basename, test_base):
-                return True
-            lhs, _ = os.path.splitext(test_base)
-            if basename.startswith(lhs):
-                return True
-            return False
-
-        for dt in self._datatables:
-            # Match using datatable pattern
-            if matches(dt['name']):
-                return dt['code']
-            if dt['src'] and matches(dt['src']):
-                return dt['code']
-        return None
-
-    def _upload_samples(self, samples: list[DataSample]) -> bool:
-        """
-        Upload samples to the Mercuto project.
-        """
-        try:
-            with self._client.as_credentials(api_key=self._api_key) as client:
-                for batch in batched(samples, 500):
-                    client.data().upload_samples(batch)
-            return True
-        except MercutoHTTPException as e:
-            if e.status_code in NON_RETRYABLE_ERRORS:
-                logger.exception(
-                    "Error indicates bad file that should not be retried. Skipping.")
-                return True
-            else:
-                return False
-
-    def _upload_file(self, file_path: str, datatable_code: str) -> bool:
-        """
-        Upload a file to the Mercuto project.
-        """
-        logging.info(f"Uploadeding file {file_path} to datatable {datatable_code} in project {self.project_code}")
-        try:
-            with self._client.as_credentials(api_key=self._api_key) as client:
-                client.data().upload_file(
-                    project=self.project_code,
-                    datatable=datatable_code,
-                    file=file_path,
-                )
-            return True
-        except MercutoHTTPException as e:
-            if e.status_code in NON_RETRYABLE_ERRORS:
-                logger.exception(
-                    "Error indicates bad file that should not be retried. Skipping.")
-                return True
-            else:
-                return False
-
-    def process_file(self, file_path: str) -> bool:
-        """
-        Process the received file.
-        """
-        logging.info(f"Processing file: {file_path}")
-        datatable_code = self.matching_datatable(file_path)
-        if datatable_code:
-            logger.info(f"Matched datatable code: {datatable_code} for file: {file_path}")
-            return self._upload_file(file_path, datatable_code)
-        else:
-            parser = detect_parser(file_path)
-            samples = parser(file_path, self._channel_map)
-            if not samples:
-                logging.warning(f"No samples found in file: {file_path}")
-                return True
-            return self._upload_samples(samples)
-
 
 T = TypeVar('T')
 
@@ -277,7 +153,7 @@ if __name__ == '__main__':
     with simple_ftp_server(directory=buffer_directory,
                            username=args.username, password=args.password, port=args.port,
                            callback=processor.add_file_to_db, rename=not args.no_rename,
-                           workdir=workdir):
+                           workdir=ftp_dir):
         schedule.every(60).seconds.do(call_and_log_error, ingester.ping)
         schedule.every(5).seconds.do(call_and_log_error, processor.process_next_file)
         schedule.every(2).minutes.do(call_and_log_error, processor.cleanup_old_files)

mercuto_client/ingester/mercuto.py

@@ -0,0 +1,155 @@
+import fnmatch
+import itertools
+import logging
+import os
+from typing import Optional
+
+from .. import MercutoClient, MercutoHTTPException
+from ..types import Channel, DataSample, DatatableOut, Project
+from .parsers import detect_parser
+from .util import batched, get_my_public_ip
+
+logger = logging.getLogger(__name__)
+
+NON_RETRYABLE_ERRORS = {400, 404, 409}  # HTTP status codes that indicate non-retryable errors
+
+
+class MercutoIngester:
+    def __init__(self, project_code: str, api_key: str, hostname: str = 'https://api.rockfieldcloud.com.au') -> None:
+        self._client = MercutoClient(url=hostname)
+        self._api_key = api_key
+        self._project_code = project_code
+
+        self._project: Optional[Project] = None
+        self._secondary_channels: Optional[list[Channel]] = None
+        self._datatables: Optional[list[DatatableOut]] = None
+
+        self._channel_map: dict[str, str] = {}
+
+    def _refresh_mercuto_data(self) -> None:
+        with self._client.as_credentials(api_key=self._api_key) as client:
+            self._project = client.projects().get_project(self._project_code)
+            assert self._project['code'] == self._project_code
+
+            self._secondary_channels = client.channels().get_channels(self._project_code, classification='SECONDARY')
+            self._datatables = list(itertools.chain.from_iterable([dt['datatables'] for dt in client.devices().list_dataloggers(self._project_code)]))
+
+        self._channel_map.update({c['label']: c['code'] for c in self._secondary_channels})
+
+    def _can_process(self) -> bool:
+        return self._project is not None and self._secondary_channels is not None and self._datatables is not None
+
+    def update_mapping(self, mapping: dict[str, str]) -> None:
+        """
+        Update the channel label to channel code mapping.
+        """
+        self._channel_map.update(mapping)
+        logger.info(f"Updated channel mapping: {self._channel_map}")
+
+    @property
+    def project_code(self) -> str:
+        return self._project_code
+
+    def ping(self) -> None:
+        """
+        Ping the Mercuto serverto update the last seen IP address.
+        """
+        ip = get_my_public_ip()
+        with self._client.as_credentials(api_key=self._api_key) as client:
+            client.projects().ping_project(self.project_code, ip_address=ip)
+            logging.info(f"Pinged Mercuto server from IP: {ip} for project: {self.project_code}")
+
+    def matching_datatable(self, filename: str) -> str | None:
+        """
+        Check if any datatables on the project match this file name.
+        Returns the datatable code if a match is found, otherwise None.
+        """
+        if self._datatables is None:
+            raise ValueError("Datatables not loaded. Call _refresh_mercuto_data() first.")
+
+        basename = os.path.basename(filename)
+
+        def matches(test: str) -> bool:
+            """
+            test should be a pattern or a filename.
+            E.g. "my_data.csv" or "my_data*.csv", or "/path/to/my_data*.csv"
+            Do wildcard matching as well as prefix matching.
+            """
+            test_base = os.path.basename(test)
+            if fnmatch.fnmatch(basename, test_base):
+                return True
+            lhs, _ = os.path.splitext(test_base)
+            if basename.startswith(lhs):
+                return True
+            return False
+
+        for dt in self._datatables:
+            # Match using datatable pattern
+            if matches(dt['name']):
+                return dt['code']
+            if dt['src'] and matches(dt['src']):
+                return dt['code']
+        return None
+
+    def _upload_samples(self, samples: list[DataSample]) -> bool:
+        """
+        Upload samples to the Mercuto project.
+        """
+        try:
+            with self._client.as_credentials(api_key=self._api_key) as client:
+                for batch in batched(samples, 500):
+                    client.data().upload_samples(batch)
+            return True
+        except MercutoHTTPException as e:
+            if e.status_code in NON_RETRYABLE_ERRORS:
+                logger.exception(
+                    "Error indicates bad file that should not be retried. Skipping.")
+                return True
+            else:
+                return False
+
+    def _upload_file(self, file_path: str, datatable_code: str) -> bool:
+        """
+        Upload a file to the Mercuto project.
+        """
+        logging.info(f"Uploadeding file {file_path} to datatable {datatable_code} in project {self.project_code}")
+        try:
+            with self._client.as_credentials(api_key=self._api_key) as client:
+                client.data().upload_file(
+                    project=self.project_code,
+                    datatable=datatable_code,
+                    file=file_path,
+                )
+            return True
+        except MercutoHTTPException as e:
+            if e.status_code in NON_RETRYABLE_ERRORS:
+                logger.exception(
+                    "Error indicates bad file that should not be retried. Skipping.")
+                return True
+            else:
+                return False
+
+    def process_file(self, file_path: str) -> bool:
+        """
+        Process the received file.
+        """
+
+        if not self._can_process():
+            logging.info("Refreshing Mercuto data...")
+            self._refresh_mercuto_data()
+            if not self._can_process():
+                logging.error("Failed to refresh Mercuto data. Cannot process file yet.")
+                return False
+
+        logging.info(f"Processing file: {file_path}")
+        datatable_code = self.matching_datatable(file_path)
+        if datatable_code:
+            logger.info(f"Matched datatable code: {datatable_code} for file: {file_path}")
+            return self._upload_file(file_path, datatable_code)
+        else:
+            parser = detect_parser(file_path)
+            samples = parser(file_path, self._channel_map)
+            if not samples:
+                logging.warning(f"No samples found in file: {file_path}")
+                return True
+            return self._upload_samples(samples)

mercuto_client-0.2.8.dist-info/METADATA

@@ -0,0 +1,65 @@
+Metadata-Version: 2.4
+Name: mercuto-client
+Version: 0.2.8
+Summary: Library for interfacing with Rockfield's Mercuto API
+Author-email: Daniel Whipp <daniel.whipp@rocktech.com.au>
+License-Expression: AGPL-3.0-only
+Project-URL: Homepage, https://mercuto.rockfieldcloud.com.au
+Project-URL: Repository, https://github.com/RockfieldTechnologiesAustralia/mercuto-client
+Project-URL: Documentation, https://github.com/RockfieldTechnologiesAustralia/mercuto-client/blob/main/README.md
+Keywords: mercuto,rockfield,infratech
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Build Tools
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.32
+Requires-Dist: pyftpdlib>=2.0.1
+Requires-Dist: python-dateutil>=2.9.0.post0
+Requires-Dist: pytz>=2025.2
+Requires-Dist: schedule>=1.2.2
+Dynamic: license-file
+
+# Mercuto Client Python Library
+
+Library for interfacing with Rockfield's Mercuto public API.
+This library is in an early development state and is subject to major structural changes at any time.
+
+(Visit our Github Repository)[https://github.com/RockfieldTechnologiesAustralia/mercuto-client]
+
+## Installation
+Install from PyPi: `pip install mercuto-client` or adding the same line into your `requirements.txt`.
+
+## Basic Usage
+
+Use the `connect()` function exposed within the main package and provide your API key.
+
+```python
+from mercuto_client import connect
+
+client = connect(api_key="<YOUR API KEY>")
+print(client.projects().get_projects())
+
+# Logout after finished.
+client.logout()
+
+```
+
+You can also use the client as a context manager. It will logout automatically.
+
+```python
+from mercuto_client import MercutoClient
+
+with MercutoClient.as_credentials(api_key='<YOUR API KEY>') as client:
+    print(client.projects().get_projects())
+```
+
+## Current Status
+This library is incomplete and may not be fully compliant with the latest Mercuto version. It is only updated periodically and provided for use without any warranty or guarantees.
+
+- [x] API Based login (Completed)
+- [ ] Username/password login

{mercuto_client-0.2.6.dev0.dist-info → mercuto_client-0.2.8.dist-info}/RECORD

@@ -14,16 +14,17 @@ mercuto_client/_tests/test_ingester/test_file_processor.py,sha256=kC1DC0phmjl7jB
 mercuto_client/_tests/test_ingester/test_ftp.py,sha256=w1CHAGcZy88D2-nY61Gj16l1nHcer9LIKaMc_DXk23o,1318
 mercuto_client/_tests/test_ingester/test_parsers.py,sha256=SJIdPi_k0rZl2Ee3UFkUo4utJz2aq9Yv5PuEfpy_gog,5961
 mercuto_client/ingester/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-mercuto_client/ingester/__main__.py,sha256=aJPFVaM7pbkifPXD0RlooXu0q1l_uE00bWQ12H6hwJU,11789
+mercuto_client/ingester/__main__.py,sha256=yF47P7WmCsRRtc8NjIeK1U7P7KJ8pQbQBUCpa1BDohs,6761
 mercuto_client/ingester/ftp.py,sha256=3-gMzoRCWjLZWeynjkwOXV59B4f0F2VnWp97fuUFTX4,4441
+mercuto_client/ingester/mercuto.py,sha256=9guacyCKsRLi39tp312kSGx8KFnSu8PevP0HoQHNBPo,6125
 mercuto_client/ingester/processor.py,sha256=XlMMM0taSHZzth39qVMsUkPO0g_ahC7Xcb01rOjQp3I,11906
 mercuto_client/ingester/util.py,sha256=yq8jgVIDeH4N1TglzaT8uf7z9m0yW7d5NPdwKFVsJKU,1931
 mercuto_client/ingester/parsers/__init__.py,sha256=2RXriMSH9-ld0W6nHZH7dDZUs8HBbAIM3B7FZwlY5b4,1364
 mercuto_client/ingester/parsers/campbell.py,sha256=jnuoQug5Rv239ANGp_1BDTM9oTH8nD8k-EJV7N82E38,416
 mercuto_client/ingester/parsers/generic_csv.py,sha256=UbujpRzOYFIGHY6sbhoVjQX-2bO1arlaFMmfurmiZVI,4025
 mercuto_client/ingester/parsers/worldsensing.py,sha256=QoSm1cH9A5zkQKaUnLDR0jWzv16RFIimc1zWgjV66PM,974
-mercuto_client-0.2.6.dev0.dist-info/licenses/LICENSE,sha256=0R2QbX4pr5XSiwUc2JoGS7Ja4npcQHyZlGJsR-E73I8,32386
-mercuto_client-0.2.6.dev0.dist-info/METADATA,sha256=92RFUrBt8cpgJaIwQyjoZeX-pEBX3O88SU5JbcMNijc,752
-mercuto_client-0.2.6.dev0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mercuto_client-0.2.6.dev0.dist-info/top_level.txt,sha256=ecV4spooVaOU8AlclvojxY1LzLW1byDywh-ayLHvKCs,15
-mercuto_client-0.2.6.dev0.dist-info/RECORD,,
+mercuto_client-0.2.8.dist-info/licenses/LICENSE,sha256=0R2QbX4pr5XSiwUc2JoGS7Ja4npcQHyZlGJsR-E73I8,32386
+mercuto_client-0.2.8.dist-info/METADATA,sha256=J9EYId6CVKgJh773sirqtkY9gyEagG7atVjuJG_YscY,2296
+mercuto_client-0.2.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mercuto_client-0.2.8.dist-info/top_level.txt,sha256=ecV4spooVaOU8AlclvojxY1LzLW1byDywh-ayLHvKCs,15
+mercuto_client-0.2.8.dist-info/RECORD,,

mercuto_client-0.2.6.dev0.dist-info/METADATA

@@ -1,20 +0,0 @@
-Metadata-Version: 2.4
-Name: mercuto-client
-Version: 0.2.6.dev0
-Summary: Library for interfacing with Rockfield's Mercuto API
-Author-email: Daniel Whipp <daniel.whipp@rocktech.com.au>
-Keywords: mercuto,rockfield,infratech
-Classifier: Development Status :: 3 - Alpha
-Classifier: Intended Audience :: Developers
-Classifier: Topic :: Software Development :: Build Tools
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Requires-Python: >=3.10
-License-File: LICENSE
-Requires-Dist: requests>=2.32
-Requires-Dist: pyftpdlib>=2.0.1
-Requires-Dist: python-dateutil>=2.9.0.post0
-Requires-Dist: pytz>=2025.2
-Requires-Dist: schedule>=1.2.2
-Dynamic: license-file