openmodule 13.2.0__tar.gz → 13.3.0__tar.gz

{openmodule-13.2.0 → openmodule-13.3.0}/ChangeLog

@@ -1,6 +1,11 @@
 CHANGES
 =======
 
+v13.3.0
+-------
+
+* [OM-359] cleanup helper function
+
 v13.2.0
 -------
 
@@ -153,11 +158,6 @@ v11.1.0.rc4
 
 * arivo-schedule in public pip
 * use temporary arivo-schedule package
-
-v11.1.0.rc3
------------
-
-* changed docs and requirements for schedule library until its a pip package
 * removed mock rpcs from schema
 
 v11.0.3
@@ -218,4 +218,3 @@ v10.0.1
 v10.0.0
 -------
 
-* v10

{openmodule-13.2.0/openmodule.egg-info → openmodule-13.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: openmodule
-Version: 13.2.0
+Version: 13.3.0
 Summary: Libraries for developing the arivo openmodule
 Home-page: https://gitlab.com/arivo-public/device-python/openmodule.git
 Author: ARIVO
@@ -123,6 +123,7 @@ pip install -e <path_to_openmodule_root>/openmodule_commands/
 - [Utils](docs/utils.md)
 - [Deprecated Features](docs/deprecated.md)
 - [Testing](docs/testing.md)
+- [File Cleanup](docs/cleanup.md)
 
 ### Openmodule Commands
 

{openmodule-13.2.0 → openmodule-13.3.0}/README.md

@@ -103,6 +103,7 @@ pip install -e <path_to_openmodule_root>/openmodule_commands/
 - [Utils](docs/utils.md)
 - [Deprecated Features](docs/deprecated.md)
 - [Testing](docs/testing.md)
+- [File Cleanup](docs/cleanup.md)
 
 ### Openmodule Commands
 

openmodule-13.3.0/docs/cleanup.md

@@ -0,0 +1,42 @@
+# File cleanup
+
+With the exception for some globally defined file_patterns (e.g. debug images, log rotation files) every service is responsible for the cleanup of its files.
+The helper function `schedule_file_cleanup` schedules a file cleanup at 2:xx local time, where xx is service name dependent to avoid that all services delete at the same time.
+Important notes:
+* only files will be deleted, directories are kept
+* files are only deleted once per day
+* files are deleted based on modified timestamp
+* a scheduler must be used which does `.run_pending()` regularly
+* as glob is only inclusive, an exclude regex can be specified for special cases
+
+
+## Examples
+
+```python
+import time
+import re
+import schedule
+from datetime import timedelta
+from openmodule.utils.cleanup import schedule_file_cleanup
+
+scheduler = schedule.Scheduler()
+
+# delete specific file after 30 days
+schedule_file_cleanup(scheduler, "/data/file_to_keep_30_days.txt", timedelta(days=30))
+
+# delete png files in folder after 1 day
+schedule_file_cleanup(scheduler, "/data/images/*.png", timedelta(days=1))
+
+# delete jpg files in subfolders after 30 days
+schedule_file_cleanup(scheduler, "/data/*/*.jpg", timedelta(days=30))
+
+# delete jpg files in subfolders recursively after 30 days (e.g. /data/a.jpg, /data/d1/a.jpg, /data/d1/d2/a.jpg)
+schedule_file_cleanup(scheduler, "/data/**/*.jpg", timedelta(days=30))
+
+# delete jpg files in subfolders, except if regex matches, recursively after 30 days (e.g. /data/d1/a.jpg, but not /data/vehicle_images/a.jpg)
+schedule_file_cleanup(scheduler, "/data/**/*.jpg", timedelta(days=30), re.compile('.*/vehicle_images/.*'))
+
+while True:
+    scheduler.run_pending()
+    time.sleep(1.0)
+```

openmodule-13.3.0/openmodule/utils/cleanup.py

@@ -0,0 +1,40 @@
+import glob
+import hashlib
+import os
+import re
+import time
+from datetime import timedelta
+from typing import Optional
+
+from schedule import Scheduler
+
+from openmodule.config import settings
+
+
+def _file_cleanup(file_pattern: str, retention_time: timedelta, exclude_regex: Optional[re.Pattern] = None):
+    """
+    removes files matching file_mask and older than retention_time and not matching exclude_regex if given
+    """
+    min_keep_time = time.time() - retention_time.total_seconds()
+    fns = glob.glob(file_pattern, recursive=True)
+    for fn in fns:
+        if os.path.getmtime(fn) < min_keep_time and os.path.isfile(fn) \
+                and not (exclude_regex and exclude_regex.match(fn)):
+            os.remove(fn)
+
+
+def schedule_file_cleanup(scheduler: Scheduler, file_pattern: str, retention_time: timedelta,
+                          exclude_regex: Optional[re.Pattern] = None):
+    """
+    schedules cleanup of files matching file_mask and older than retention_time at every day at 02:xx
+    where xx is a service name dependent offset
+    :param scheduler: scheduler where file cleanup is scheduled
+    :param file_pattern: pattern for files (glob syntax) that files must match to be deleted
+    :param retention_time: minimum age of file before its deleted
+    :param exclude_regex: files matching exclude_regex are excluded even if they match file_pattern
+    """
+    service_name_hash = hashlib.sha256(settings.NAME.encode("utf-8")).hexdigest()
+    random_offset = int(service_name_hash, 16) % 60
+
+    scheduler.every().day.at(f"02:{random_offset:02}", settings.TIMEZONE).do(_file_cleanup, file_pattern,
+                                                                             retention_time, exclude_regex)

{openmodule-13.2.0 → openmodule-13.3.0/openmodule.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: openmodule
-Version: 13.2.0
+Version: 13.3.0
 Summary: Libraries for developing the arivo openmodule
 Home-page: https://gitlab.com/arivo-public/device-python/openmodule.git
 Author: ARIVO
@@ -123,6 +123,7 @@ pip install -e <path_to_openmodule_root>/openmodule_commands/
 - [Utils](docs/utils.md)
 - [Deprecated Features](docs/deprecated.md)
 - [Testing](docs/testing.md)
+- [File Cleanup](docs/cleanup.md)
 
 ### Openmodule Commands
 

{openmodule-13.2.0 → openmodule-13.3.0}/openmodule.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ test-requirements.txt
 tox.ini
 docs/access_service.md
 docs/anonymization.md
+docs/cleanup.md
 docs/coding_standard.md
 docs/commands.md
 docs/connection_status_listener.md
@@ -81,6 +82,7 @@ openmodule/rpc/server.py
 openmodule/utils/__init__.py
 openmodule/utils/access_service.py
 openmodule/utils/charset.py
+openmodule/utils/cleanup.py
 openmodule/utils/csv_export.py
 openmodule/utils/databox.py
 openmodule/utils/db_helper.py
@@ -145,6 +147,7 @@ tests/test_test_gate.py
 tests/test_test_zeromq.py
 tests/test_utils_access_service.py
 tests/test_utils_charset.py
+tests/test_utils_cleanup.py
 tests/test_utils_csv_export.py
 tests/test_utils_databox.py
 tests/test_utils_eventlog.py

openmodule-13.3.0/openmodule.egg-info/pbr.json

@@ -0,0 +1 @@
+{"git_version": "4623c40", "is_release": true}

openmodule-13.3.0/tests/test_utils_cleanup.py

@@ -0,0 +1,107 @@
+import glob
+import os
+import re
+import shutil
+from datetime import datetime, timedelta
+from unittest import TestCase
+
+import freezegun
+import schedule
+from dateutil.tz import UTC, gettz, tzlocal
+
+from openmodule.config import settings
+from openmodule.utils.cleanup import _file_cleanup, schedule_file_cleanup
+
+
+class CleanupTest(TestCase):
+    test_dir = "/tmp/cleanup_test/"
+
+    def setUp(self):
+        super().setUp()
+        try:
+            shutil.rmtree(self.test_dir)
+        except OSError:
+            pass
+        os.makedirs(self.test_dir)
+
+    def test_cleanup_file_pattern_and_retention_time(self):
+        open(os.path.join(self.test_dir, "a.png"), 'w').close()
+        open(os.path.join(self.test_dir, "a2.png"), 'w').close()
+        open(os.path.join(self.test_dir, "b.png"), 'w').close()
+        current_time = datetime.utcnow()
+
+        # 1 minute less than retention time old -> no delete
+        delta = timedelta(days=3)
+        with freezegun.freeze_time(current_time + delta - timedelta(minutes=1)):
+            _file_cleanup("/tmp/cleanup_test/a*.png", delta)
+            self.assertEqual(3, len(glob.glob("/tmp/cleanup_test/*.png")))
+
+        # older than retention time old -> delete according to pattern -> only b.png left
+        with freezegun.freeze_time(current_time + delta + timedelta(minutes=1)):
+            _file_cleanup("/tmp/cleanup_test/a*.png", delta)
+            self.assertEqual(["/tmp/cleanup_test/b.png"], glob.glob("/tmp/cleanup_test/*.png"))
+
+    def test_recursive(self):
+        os.makedirs("/tmp/cleanup_test/d1/d2/d3/d4")
+        open(os.path.join(self.test_dir, "/tmp/cleanup_test/a.png"), 'w').close()
+        open(os.path.join(self.test_dir, "/tmp/cleanup_test/d1/a.png"), 'w').close()
+        open(os.path.join(self.test_dir, "/tmp/cleanup_test/d1/d2/a.png"), 'w').close()
+        open(os.path.join(self.test_dir, "/tmp/cleanup_test/d1/d2/d3/a.png"), 'w').close()
+        open(os.path.join(self.test_dir, "/tmp/cleanup_test/d1/d2/d3/d4/a.png"), 'w').close()
+
+        self.assertEqual(5, len(glob.glob("/tmp/cleanup_test/**/*.png", recursive=True)))
+        with freezegun.freeze_time(datetime.utcnow() + timedelta(days=2)):
+            _file_cleanup("/tmp/cleanup_test/*/a.png", timedelta(days=1))  # delete only .../d1/a.png
+            self.assertEqual(4, len(glob.glob("/tmp/cleanup_test/**/*.png", recursive=True)))
+
+            _file_cleanup("/tmp/cleanup_test/**/a.png", timedelta(days=1))  # delete all
+            self.assertEqual(0, len(glob.glob("/tmp/cleanup_test/**/*.png", recursive=True)))
+
+    def test_exclude_regex(self):
+        os.makedirs("/tmp/cleanup_test/vehicle_images/")
+        os.makedirs("/tmp/cleanup_test/vehicle_loop/")
+        os.makedirs("/tmp/cleanup_test/new_track/")
+        open(os.path.join(self.test_dir, "/tmp/cleanup_test/vehicle_images/a.jpg"), 'w').close()
+        open(os.path.join(self.test_dir, "/tmp/cleanup_test/vehicle_loop/b.jpg"), 'w').close()
+        open(os.path.join(self.test_dir, "/tmp/cleanup_test/new_track/c.jpg"), 'w').close()
+
+        with freezegun.freeze_time(datetime.utcnow() + timedelta(days=2)):
+            _file_cleanup("/tmp/cleanup_test/*/*.jpg", timedelta(days=1), re.compile('.*/vehicle_images/.*'))
+            self.assertEqual(["/tmp/cleanup_test/vehicle_images/a.jpg"],
+                             glob.glob("/tmp/cleanup_test/*/*.jpg", recursive=True))
+
+    def test_scheduling(self):
+        scheduler = schedule.Scheduler()
+        open(os.path.join(self.test_dir, "1.png"), 'w').close()
+        open(os.path.join(self.test_dir, "2.png"), 'w').close()
+
+        # 2 days to ensure we delete even if we hit an unlucky race condition when testcase is run
+        base_time = (datetime.utcnow() + timedelta(days=2))\
+            .replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=gettz(settings.TIMEZONE))\
+            .astimezone(UTC)
+        with freezegun.freeze_time(base_time, tz_offset=datetime.now(tz=tzlocal()).utcoffset()):
+            settings.NAME = "om_service_test_1"  # offset 17 minutes
+            schedule_file_cleanup(scheduler, "/tmp/cleanup_test/1.png", timedelta(0))
+            settings.NAME = "om_service_test_2"  # offset 1 minute
+            schedule_file_cleanup(scheduler, "/tmp/cleanup_test/2.png", timedelta(0))
+
+        # 2:00 -> no job should be run -> 2 files present
+        with freezegun.freeze_time(base_time + timedelta(hours=2),
+                                   tz_offset=datetime.now(tz=tzlocal()).utcoffset()):
+            scheduler.run_pending()
+            self.assertEqual(2, len(glob.glob("/tmp/cleanup_test/*.png")))
+        # 2:01 -> 2.png should be deleted because job ran
+        with freezegun.freeze_time(base_time + timedelta(hours=2, minutes=1),
+                                   tz_offset=datetime.now(tz=tzlocal()).utcoffset()):
+            scheduler.run_pending()
+            self.assertEqual(["/tmp/cleanup_test/1.png"], glob.glob("/tmp/cleanup_test/*.png"))
+        # 2:16 -> no more deletions
+        with freezegun.freeze_time(base_time + timedelta(hours=2, minutes=16),
+                                   tz_offset=datetime.now(tz=tzlocal()).utcoffset()):
+            scheduler.run_pending()
+            self.assertEqual(["/tmp/cleanup_test/1.png"], glob.glob("/tmp/cleanup_test/*.png"))
+        # 2:17 -> 1.png should be deleted because job ran
+        with freezegun.freeze_time(base_time + timedelta(hours=2, minutes=17),
+                                   tz_offset=datetime.now(tz=tzlocal()).utcoffset()):
+            scheduler.run_pending()
+            self.assertEqual([], glob.glob("/tmp/cleanup_test/*.png"))

openmodule-13.2.0/openmodule.egg-info/pbr.json

@@ -1 +0,0 @@
-{"git_version": "0bcec3c", "is_release": true}