mercuto-client 0.3.0a0__py3-none-any.whl → 0.3.1__py3-none-any.whl

mercuto_client/_tests/test_ingester/test_file_processor.py

@@ -37,7 +37,7 @@ def temp_env() -> Generator[Tuple[FileProcessor, str, str], None, None]:
 
 def test_init_db(temp_env: Tuple[FileProcessor, str, str]) -> None:
     """Verify database initialization"""
-    processor, _, db_path = temp_env
+    _, _, db_path = temp_env
     conn: sqlite3.Connection = sqlite3.connect(db_path)
     cursor: sqlite3.Cursor = conn.cursor()
     cursor.execute(

mercuto_client/ingester/__main__.py

@@ -43,6 +43,8 @@ if __name__ == '__main__':
     parser.add_argument('-s', '--size', type=int,
                         help='Size in MB for total amount of files to store in the buffer. \
                             Default is 75% of the available disk space on the buffer partition excluding the directory itself', default=None)
+    parser.add_argument('--max-files', type=int,
+                        help='Maximum number of files to keep in the buffer. Default is to use the size param.', default=None)
     parser.add_argument('--max-attempts', type=int,
                         help='Maximum number of attempts to process a file before giving up. Default is 1000.',
                         default=1000)
@@ -114,7 +116,7 @@ if __name__ == '__main__':
     os.makedirs(ftp_dir, exist_ok=True)
 
     size = args.size
-    if size is None:
+    if size is None and args.max_files is None:
         size = get_free_space_excluding_files(buffer_directory) * 0.75 // (1024 * 1024)  # Convert to MB
         logging.info(f"Buffer size set to {size} MB based on available disk space.")
 
@@ -146,7 +148,8 @@ if __name__ == '__main__':
         db_path=database_path,
         process_callback=ingester.process_file,
         max_attempts=args.max_attempts,
-        free_space_mb=size)
+        free_space_mb=size,
+        max_files=args.max_files)
 
     processor.scan_existing_files()
 

mercuto_client/ingester/processor.py

@@ -8,6 +8,16 @@ from typing import Callable, Optional
 logger = logging.getLogger(__name__)
 
 
+def _default_standard_clock(_: str):
+    # Default clock function to get current time in seconds since epoch
+    return datetime.now(timezone.utc).timestamp()
+
+
+def _default_file_clock(filepath: str) -> float:
+    # Default clock function to get file creation time in seconds since epoch
+    return os.path.getctime(filepath)
+
+
 class FileProcessor:
     """
     System for processing files in a strict order with retry logic.
@@ -59,11 +69,7 @@ class FileProcessor:
         self._max_attempts = max_attempts
         self._process_callback = process_callback
         self._free_space_mb = free_space_mb
-        if clock is None:
-            # Default clock function to get current time in seconds since epoch
-            def clock(_): return datetime.now(timezone.utc).timestamp()
-        self._clock = clock
-
+        self._clock = clock if clock is not None else _default_standard_clock
         os.makedirs(self._buffer_dir, exist_ok=True)
         self._init_db()
 
@@ -102,9 +108,7 @@ class FileProcessor:
 
         """
         if clock is None:
-            # Default clock function to get current time in seconds since epoch
-            def clock(filepath: str) -> float:
-                return os.path.getctime(filepath)
+            clock = _default_file_clock
 
         conn = sqlite3.connect(self._db_path)
         cursor = conn.cursor()
@@ -241,7 +245,7 @@ class FileProcessor:
 
         def free_space_mb() -> float:
             """Returns the free space in MB on the partition where the buffer directory is located."""
-            total, used, free = shutil.disk_usage(self._buffer_dir)
+            _, _, free = shutil.disk_usage(self._buffer_dir)
             return free / (1024 * 1024)
 
         while free_space_mb() < self._free_space_mb:
@@ -261,19 +265,17 @@ class FileProcessor:
             return False
 
         filepath = oldest_file[0]
-        removed = False
         if os.path.exists(filepath):
             os.remove(filepath)
             cursor.execute(
                 "DELETE FROM file_buffer WHERE filepath = ?", (filepath,))
             conn.commit()
-            removed = True
             logger.info(f"Deleted oldest file {filepath}")
         else:
             logger.warning(f"Oldest file {filepath} does not exist.")
 
         conn.close()
-        return removed
+        return True
 
     def add_file_to_db(self, filepath: str) -> None:
         """Adds a new file to database and triggers processing."""

{mercuto_client-0.3.0a0.dist-info → mercuto_client-0.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mercuto-client
-Version: 0.3.0a0
+Version: 0.3.1
 Summary: Library for interfacing with Rockfield's Mercuto API
 Author-email: Daniel Whipp <daniel.whipp@rocktech.com.au>
 License-Expression: AGPL-3.0-only

{mercuto_client-0.3.0a0.dist-info → mercuto_client-0.3.1.dist-info}/RECORD

@@ -8,17 +8,17 @@ mercuto_client/util.py,sha256=PWNp8BU9wa_sKMPxW0R1yveEF1XslZXG8sw6BZvUoO8,2603
 mercuto_client/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 mercuto_client/_tests/conftest.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 mercuto_client/_tests/test_ingester/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-mercuto_client/_tests/test_ingester/test_file_processor.py,sha256=kC1DC0phmjl7jBMMBJYrs9Tx4NL9xKJNmqVX5FNH59s,7399
+mercuto_client/_tests/test_ingester/test_file_processor.py,sha256=J-fscfEcnXdWAthmHzPbjBPB3BwsRcX-ZB7yaoX56lo,7391
 mercuto_client/_tests/test_ingester/test_ftp.py,sha256=w1CHAGcZy88D2-nY61Gj16l1nHcer9LIKaMc_DXk23o,1318
 mercuto_client/_tests/test_ingester/test_parsers.py,sha256=R9GnzAaGu_tva0s23VpwEVfEsEUcto3kN3EloIxZvVY,5917
 mercuto_client/_tests/test_mocking/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 mercuto_client/_tests/test_mocking/conftest.py,sha256=M-HikiSj_uuYAEYQnFA0uDtSBpZjO2YXjzxNbluQdMQ,266
 mercuto_client/_tests/test_mocking/test_mock_identity.py,sha256=394r6A_xK78tse4CZA345JpIlIKF_GnPz2xXEjc_Nho,313
 mercuto_client/ingester/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-mercuto_client/ingester/__main__.py,sha256=HZisjXuj5wzdUJ9vLqjhCDqLobflLLVLuI-7PclvVgw,6762
+mercuto_client/ingester/__main__.py,sha256=c7dKkl4Z3aHJuzbrbdAvxstNKzQ9eAem2za9GBvxQ_E,6999
 mercuto_client/ingester/ftp.py,sha256=3-gMzoRCWjLZWeynjkwOXV59B4f0F2VnWp97fuUFTX4,4441
 mercuto_client/ingester/mercuto.py,sha256=djlP9HVoBoB2R-pQVogyMvHpPpbWEjCX5vbMhlEhc8U,6077
-mercuto_client/ingester/processor.py,sha256=XlMMM0taSHZzth39qVMsUkPO0g_ahC7Xcb01rOjQp3I,11906
+mercuto_client/ingester/processor.py,sha256=QM5zcX6QlXYAn4_cxqWSGHfgNICVrYcPXF3TGMVnjDU,11907
 mercuto_client/ingester/parsers/__init__.py,sha256=lOt4TyP08hK56wl7XHMlqmj62mDE0Idx8UJL0wljtwM,1398
 mercuto_client/ingester/parsers/campbell.py,sha256=S5enYbajVTm3zSQYkEP6JRVUw94Z7ky100j8p5qLCls,441
 mercuto_client/ingester/parsers/generic_csv.py,sha256=v4rwO4oJb1Ue6zirO0TGtvewOSp9f6ZUT_sWvozvQHo,4051
@@ -34,8 +34,8 @@ mercuto_client/modules/core.py,sha256=KzSmKH71qK9wDM6I-LvXY9oA2nBJFLoQXawK2ei3uC
 mercuto_client/modules/data.py,sha256=hMhzkkr0f7N9kVKvUQE65uXDpcCxGILcNUhZLSD3nD4,20742
 mercuto_client/modules/fatigue.py,sha256=SPftwW5rMk6LcsIGWO8OHc9r6DA99qt0AyAVOaNRt_8,6397
 mercuto_client/modules/identity.py,sha256=VeMGsq-s5B9xorP9G2TxwIDYUSyyDGuFAJ8Ilg7_mn4,7898
-mercuto_client-0.3.0a0.dist-info/licenses/LICENSE,sha256=0R2QbX4pr5XSiwUc2JoGS7Ja4npcQHyZlGJsR-E73I8,32386
-mercuto_client-0.3.0a0.dist-info/METADATA,sha256=u2Yn0V3LiTyZaMtKoJenocAtWXKsqvHN-7BrVBa7YGc,2423
-mercuto_client-0.3.0a0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mercuto_client-0.3.0a0.dist-info/top_level.txt,sha256=ecV4spooVaOU8AlclvojxY1LzLW1byDywh-ayLHvKCs,15
-mercuto_client-0.3.0a0.dist-info/RECORD,,
+mercuto_client-0.3.1.dist-info/licenses/LICENSE,sha256=0R2QbX4pr5XSiwUc2JoGS7Ja4npcQHyZlGJsR-E73I8,32386
+mercuto_client-0.3.1.dist-info/METADATA,sha256=OXr8wjeUoZF3X5CT8QzO0w559HTPCyY2qeUueRr22Mc,2421
+mercuto_client-0.3.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mercuto_client-0.3.1.dist-info/top_level.txt,sha256=ecV4spooVaOU8AlclvojxY1LzLW1byDywh-ayLHvKCs,15
+mercuto_client-0.3.1.dist-info/RECORD,,