berryworld 1.0.0.181358__py3-none-any.whl → 1.0.0.182055__py3-none-any.whl

berryworld/__init__.py

@@ -1,5 +1,5 @@
 from .xml_parser import XMLparser
-from .handy_mix import HandyMix
+from .handy_mix import HandyMix, URLResolver
 from .transportation_solver import TransportationAlgorithm
 from .allocation_solver import AllocationSolver
 from .pickle_management import PickleManagement
@@ -24,3 +24,4 @@ from .vivantio import Vivantio
 from .teams_logging import TeamsLogging
 from .vivantio_logging import VivantioLogging
 from .snowflake_conn import SnowflakeConn
+from .logging import PythonLogs

berryworld/handy_mix.py

@@ -234,3 +234,26 @@ class HandyMix:
                         ~data[float_col].isnull(), float_col].astype(str).str.replace('.0', '', regex=True)
 
         return data
+
+
+class URLResolver:
+    @staticmethod
+    def resolve_url_requested(url_requested, endpoint_requested):
+        """Resolve the URL requested.
+        :param url_requested: The base url that includes the project root
+        :param endpoint_requested: The endpoint that will be added to the base url
+        """
+        url_requested = str(url_requested)
+        endpoint_requested = str(endpoint_requested)
+
+        domains = ["berryworld.com", "berryworld.net"]
+
+        for domain in domains:
+            if domain in url_requested:
+                if url_requested.endswith('/'):
+                    url_requested = url_requested.rstrip('/')
+
+                url_requested = f"{url_requested}{endpoint_requested}"
+                break
+
+        return url_requested

berryworld/logging.py

@@ -0,0 +1,31 @@
+
+
+
+class PythonLogs:
+    """ Register the Python Logs """
+    def __init__(self, conn, batch_process):
+        self.conn = conn
+        self.batch_process = batch_process
+        self.batch_log_id = None
+
+    def start(self, log_type):
+        """ Start the Python Log
+        :param log_type: Type of the log
+        """
+        batch_log = self.conn.run_statement(
+            f"EXEC d365bc.spPythonBatchLogInsert @DataType = {self.batch_process}, @LogType = '{log_type}'",
+            commit_as_transaction=False)
+        self.batch_log_id = batch_log['OUTPUT'][0]
+
+    def end(self, batch_log_id=None):
+        """ End the Python Log
+        :param batch_log_id: Batch Log Id
+        """
+        if batch_log_id is None:
+            batch_log_id = self.batch_log_id
+        if batch_log_id is None:
+            raise Exception('Batch Log Id is not set')
+
+        self.conn.run_statement(
+            f"EXEC d365bc.spPythonBatchLogUpdate @BatchLogId = {batch_log_id}",
+            commit_as_transaction=False)

berryworld/persistent_storage.py

@@ -1,6 +1,7 @@
 import os
-import shutil
 import time
+import shutil
+import pandas as pd
 from pathlib import Path
 from datetime import datetime
 
@@ -9,13 +10,13 @@ class PersistentStorage:
     """ Connect to Persistent Storage """
 
     def __init__(self, base=None):
-        """ Initialize the class
-        -----------------------------
+        """ Workout the root and the base path
+        :param base: Base path to be used for the project
         """
         if os.name == 'nt':
             root = os.getcwd()
         else:
-            root = '/mnt/datascience-persistent-store-file-share/'
+            root = os.path.join(os.getcwd(), 'mnt/datascience-persistent-store-file-share')
 
         if base is not None:
             self.base_path = os.path.join(root, base)
@@ -24,47 +25,64 @@ class PersistentStorage:
 
         self.connect()
 
-    def format_path(self, path, sub_path=None):
-        if self.base_path in path:
-            format_path = path
+    def format_path(self, path_, sub_path=None):
+        """ Format the path
+        :param path_: Path to be formatted
+        :param sub_path: Sub path to be added to the path
+        """
+        if self.base_path in path_:
+            format_path = path_
         else:
-            format_path = os.path.join(self.base_path, path)
+            format_path = os.path.join(self.base_path, path_)
 
         if sub_path is not None:
             format_path = os.path.join(format_path, sub_path)
 
         return format_path
 
-    def if_exists(self, path):
-        path = self.format_path(path)
-        obj = Path(path)
+    def if_exists(self, path_='.'):
+        """ Check if the path exists
+        :param path_: Path to be checked
+        """
+        path_ = os.path.join(self.base_path, str(self.format_path(path_)))
+        obj = Path(path_)
         status = obj.exists()
         return status
 
     def connect(self):
+        """ Check if the base path exists """
         if not self.if_exists(self.base_path):
             raise ValueError("The current project doesn't have access to persistent storage, please check YAML file.")
 
     def return_base_path(self):
+        """ Return the base path """
         return self.base_path
 
-    def return_input_path(self, path):
-        path = self.format_path(path)
-        return path
+    def return_input_path(self, path_):
+        """ Return the input path
+        :param path_: Path to be returned
+        """
+        path_ = self.format_path(path_)
+        return path_
 
-    def create_folder(self, path):
+    def create_folder(self, path_='.'):
+        """ Create a folder
+        :param path_: Path to create the folder
+        """
         created = True
-        folder_path = self.format_path(path)
+        folder_path = os.path.join(self.base_path, str(self.format_path(path_)))
         try:
             Path(folder_path).mkdir(parents=True, exist_ok=True)
         except Exception as e:
             print(e)
             created = False
-
         return created
 
-    def list_folders(self, path):
-        folder_path = self.format_path(path)
+    def list_folders(self, path_='.'):
+        """ List all the folders in the path
+        :param path_: Path to list the folders
+        """
+        folder_path = os.path.join(self.base_path, str(self.format_path(path_)))
         try:
             if self.if_exists(folder_path):
                 folders = [f for f in os.listdir(folder_path) if os.path.isdir(os.path.join(folder_path, f))]
@@ -76,22 +94,27 @@ class PersistentStorage:
 
         return folders
 
-    def create_file(self, data, path, file_name=None):
+    def create_file(self, data_, path_='.', file_name=None):
+        """ Create a file
+        :param data_: Data to be written to the file
+        :param path_: Path to create the file
+        :param file_name: Name of the file to be created
+        """
         created = True
-        folder_path = self.format_path(path)
+        folder_path = os.path.join(self.base_path, str(self.format_path(path_)))
         if file_name is None:
             file_path = os.path.join(folder_path, datetime.now().strftime("%H_%M_%S_%f"))
         else:
             file_path = os.path.join(folder_path, file_name)
 
         try:
-            if type(data) == bytes:
+            if type(data_) == bytes:
                 with open(file_path, 'wb') as f:
-                    f.write(data)
+                    f.write(data_)
                     f.close()
             else:
                 with open(file_path, 'w') as f:
-                    f.write(data)
+                    f.write(data_)
                     f.close()
         except Exception as e:
             print(e)
@@ -99,8 +122,11 @@ class PersistentStorage:
 
         return created
 
-    def list_files(self, path):
-        folder_path = self.format_path(path)
+    def list_files(self, path_='.'):
+        """ List all the files in the path
+        :param path_: Path to list the files
+        """
+        folder_path = os.path.join(self.base_path, str(self.format_path(path_)))
         try:
             if self.if_exists(folder_path):
                 files = [f for f in os.listdir(folder_path) if os.path.isfile(os.path.join(folder_path, f))]
@@ -113,8 +139,14 @@ class PersistentStorage:
         return files
 
     def move_files(self, source_folder, target_folder, source_files=None, move_all=False):
-        source_path = self.format_path(source_folder)
-        target_path = self.format_path(target_folder)
+        """ Move files from source to target folder
+        :param source_folder: Source folder
+        :param target_folder: Target folder
+        :param source_files: Files to be moved
+        :param move_all: Move all files
+        """
+        source_path = str(self.format_path(source_folder))
+        target_path = str(self.format_path(target_folder))
         if source_files is None:
             source_files = self.list_files(source_folder)
             all_source_files = source_files
@@ -144,6 +176,10 @@ class PersistentStorage:
         return moved_files
 
     def move_file(self, source_file_path, target_file_path):
+        """ Move a file from source to target path
+        :param source_file_path: Source file path
+        :param target_file_path: Target file path
+        """
         try:
             if self.if_exists(source_file_path) & Path(source_file_path).is_file():
                 shutil.move(source_file_path, target_file_path)
@@ -152,16 +188,22 @@ class PersistentStorage:
             print(e)
             return False
 
-    def get_create_time(self, path):
-        path = self.format_path(path)
-        if self.if_exists(path):
-            create_time = os.path.getmtime(path)
+    def get_folder_creation(self, path_):
+        """ Get the create time of the file
+        :param path_: Path to get the create time
+        """
+        path_ = str(self.format_path(path_))
+        if self.if_exists(path_):
+            create_time = pd.to_datetime(time.ctime(os.path.getctime(path_)))
             return create_time
         else:
             return None
 
-    def delete_folder(self, path):
-        folder_path = self.format_path(path)
+    def delete_folder(self, path_):
+        """ Delete a folder
+        :param path_: Path to delete the folder
+        """
+        folder_path = os.path.join(self.base_path, str(self.format_path(path_)))
         delete = True
         try:
             if self.if_exists(folder_path):
@@ -172,14 +214,40 @@ class PersistentStorage:
 
         return delete
 
-    def delete_file(self, path):
-        file_path = self.format_path(path)
+    def delete_file(self, path_):
+        """ Delete a file
+        :param path_: Path to delete the file
+        """
+        file_path = os.path.join(self.base_path, str(self.format_path(path_)))
         delete = True
         try:
-            if self.if_exists(file_path) & Path(path).is_file():
+            if self.if_exists(file_path) & Path(path_).is_file():
                 os.remove(file_path)
         except Exception as e:
             print(e)
             delete = False
 
         return delete
+
+    def get_file_creation(self, path_):
+        """ Get when a file was created
+        :param path_: Path to the file
+        """
+        path_ = str(self.format_path(path_))
+        if self.if_exists(path_):
+            create_time = pd.to_datetime(time.ctime(os.path.getctime(path_)))
+            return create_time
+        else:
+            return None
+
+    def delete_older_files(self, path_, date):
+        """ Archive the files
+        :param path_: Path to the files
+        :param date: Date to archive the files
+        """
+        file_path = os.path.join(self.base_path, str(self.format_path(path_)))
+
+        files_to_remove = [file for file in os.listdir(file_path) if
+                           date >= self.get_file_creation(os.path.join(file_path, file))]
+        for file in files_to_remove:
+            self.delete_file(os.path.join(file_path, file))

{berryworld-1.0.0.181358.dist-info → berryworld-1.0.0.182055.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: berryworld
-Version: 1.0.0.181358
+Version: 1.0.0.182055
 Summary: Handy classes to improve ETL processes
 Home-page: https://www.berryworld.com
 Author: BerryWorld ltd

{berryworld-1.0.0.181358.dist-info → berryworld-1.0.0.182055.dist-info}/RECORD

@@ -1,4 +1,4 @@
-berryworld/__init__.py,sha256=MMGWZ2WI82CgxWapFBsC3913rb3vspOWEQwAa8yMSTw,1141
+berryworld/__init__.py,sha256=FxoH2n864zRK6cVG47s8ciwDldfMB3zVzfa3ZRwtLWk,1185
 berryworld/aks_logs.py,sha256=Gb2_cokiZbEX01Yoptd0MxpDociaug-GrXdwliyxFBo,2293
 berryworld/allocation_solver.py,sha256=asFtaCAze6-eHUGWXA0kAp67UBS-Upj1KKdrVLj_ttQ,8513
 berryworld/app_logs.py,sha256=MKzKPYd3JuPfOQNAapIgaeZeFHw1z_w2mbn9I6QCADE,4180
@@ -9,10 +9,11 @@ berryworld/devops.py,sha256=BAsVonVwCXoApUOovkt-BCzwc6KnXjxRDGff_ejSGw8,9719
 berryworld/email_con.py,sha256=uSBzs_Ijz9pUPWt9e7U3TCB7i6q7hU1bB5vhsTB_tmw,14448
 berryworld/email_logging.py,sha256=LeSrTExhQhar49gJR2wGC1dS0lqsNpFl9pS3eYWqnuo,4936
 berryworld/generate_env.py,sha256=Tk9Z_u7cA4Ve8YYTyLH2qwmLVAuYoTIWoFc0h8Va8lY,7842
-berryworld/handy_mix.py,sha256=lRUes_V4jL-3ADOzwiKqi1Dukx-HWXvmf9-4s4MLwsA,9740
+berryworld/handy_mix.py,sha256=4ou8_pwRLmV809ZVgWdRgH9Dlb6TpPOaarCjcREEwG4,10510
+berryworld/logging.py,sha256=tOxzaFALQm3aVVEc3I7t8MU6PwgUI9VSnzNCH9yRryo,1013
 berryworld/logic_apps.py,sha256=a0uU4tNO3v2w7grdBv-OOx4hUf7VBIerJpwZ9U-29dQ,14591
 berryworld/microsoft_teams.py,sha256=8uPo0yku-euBj2VdzBoZCeX3IcsCCOqISLqaVZUVxfA,16030
-berryworld/persistent_storage.py,sha256=_lGdXa7IyxfMF3xNF9y26X_z9RDb2Ah7R0oF61HR8Gc,5764
+berryworld/persistent_storage.py,sha256=uGDARPP_CxuIykvRhwTMoQE8AfC6IyGNaWZxRRzOzJc,8627
 berryworld/pickle_management.py,sha256=O49ojVtTqYCT510rVRTbZWWaur_-5q3HSVG03Azn8mQ,2393
 berryworld/postgres_connection.py,sha256=whKDnchd5Feqpmxpoh2vlyn36EKHR-dVEULYq0N_4wA,8287
 berryworld/power_automate.py,sha256=9rDuRy0v-Ttq-SThid4lOB_tD4ibkyEmobiROpa--g4,25414
@@ -29,8 +30,8 @@ tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_allocation_config.py,sha256=e12l6fE9U57eSPS35g6ekJ_hol7-RHg89JV60_m1BlE,4633
 tests/test_handy_mix_config.py,sha256=Un56mz9KJmdn4K4OwzHAHLSRzDU1Xv2nFrONNuzOG04,2594
 tests/test_xml_parser.py,sha256=3QTlhFEd6KbK6nRFKZnc35tad6wqukTbe4QrFi8mr_8,859
-berryworld-1.0.0.181358.dist-info/LICENSE,sha256=vtkVCJM6E2af2gnsi2XxKPr4WY-uIbvzVLXieFND0UU,1074
-berryworld-1.0.0.181358.dist-info/METADATA,sha256=znF4x6wLc65ByBfuts_wxOdfDJYTrxRJgcsPbdFnwnc,1174
-berryworld-1.0.0.181358.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
-berryworld-1.0.0.181358.dist-info/top_level.txt,sha256=GIZ5qy-P5oxfEH755vA1IMFeTVdX3-40JxMe6nOe5I8,17
-berryworld-1.0.0.181358.dist-info/RECORD,,
+berryworld-1.0.0.182055.dist-info/LICENSE,sha256=vtkVCJM6E2af2gnsi2XxKPr4WY-uIbvzVLXieFND0UU,1074
+berryworld-1.0.0.182055.dist-info/METADATA,sha256=8WVQFKkeQC12_VMXILN50UmUi14zsuGxkg8BEo4UZCc,1174
+berryworld-1.0.0.182055.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+berryworld-1.0.0.182055.dist-info/top_level.txt,sha256=GIZ5qy-P5oxfEH755vA1IMFeTVdX3-40JxMe6nOe5I8,17
+berryworld-1.0.0.182055.dist-info/RECORD,,

{berryworld-1.0.0.181358.dist-info → berryworld-1.0.0.182055.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.2.0)
+Generator: setuptools (75.3.0)
 Root-Is-Purelib: true
 Tag: py3-none-any