pyxecm 2.0.3__py3-none-any.whl → 3.0.0__py3-none-any.whl

pyxecm/coreshare.py

@@ -27,6 +27,7 @@ import time
 import urllib.parse
 from http import HTTPStatus
 from importlib.metadata import version
+from urllib.parse import parse_qs, urlparse
 
 import requests
 
@@ -522,7 +523,7 @@ class CoreShare:
                 Can be used to provide a more specific error message
                 in case an error occurs.
             show_error (bool, optional):
-                True: write an error to the log file
+                True: write an error to the log file (this is the default)
                 False: write a warning to the log file
 
         Returns:
@@ -933,6 +934,71 @@ class CoreShare:
 
     # end method definition
 
+    def get_groups_iterator(
+        self,
+        count: int | None = None,
+    ) -> iter:
+        """Get an iterator object that can be used to traverse all Core Share groups.
+
+        Returning a generator avoids loading a large number of items into memory at once. Instead you
+        can iterate over the potential large list of Core Share groups.
+
+        Example usage:
+            groups = core_share_object.get_groups_iterator(page_size=10)
+            for group in groups:
+                logger.info("Traversing Core Share group -> %s", group["name"])
+
+        Args:
+            count (int | None, optional):
+                The chunk size for the number of groups returned by one
+                REST API call. If None, then a default of 250 is used.
+
+        Returns:
+            iter:
+                A generator yielding one OTDS group per iteration.
+                If the REST API fails, returns no value.
+
+        """
+
+        offset = 0
+
+        while True:
+            response = self.get_groups(
+                offset=offset,
+                count=count,
+            )
+            if not response or not response.get("results", []):
+                # Don't return None! Plain return is what we need for iterators.
+                # Natural Termination: If the generator does not yield, it behaves
+                # like an empty iterable when used in a loop or converted to a list:
+                return
+
+            # Yield users one at a time:
+            yield from response["results"]
+
+            # See if we have an additional result page.
+            # If not terminate the iterator and return
+            # no value.
+
+            next_page_url = response["_links"].get("next")
+            if not next_page_url:
+                # Don't return None! Plain return is what we need for iterators.
+                # Natural Termination: If the generator does not yield, it behaves
+                # like an empty iterable when used in a loop or converted to a list:
+                return
+            next_page_url = next_page_url.get("href")
+
+            # Extract the query string from the URL
+            query = urlparse(next_page_url).query
+
+            # Parse the query parameters into a dictionary
+            params = parse_qs(query)
+
+            # Get the 'offset' value as an integer (it's a list by default)
+            offset = int(params.get("offset", [0])[0])
+
+    # end method definition
+
     def add_group(
         self,
         group_name: str,
@@ -1276,7 +1342,7 @@ class CoreShare:
             dict | None:
                 Dictionary with the Core Share group data or None if the request fails.
 
-            Example result:
+            Example Response:
             {
                 'results': [
                     {
@@ -1343,15 +1409,15 @@ class CoreShare:
 
     # end method definition
 
-    def get_users(self) -> dict | None:
+    def get_users(self) -> list | None:
         """Get Core Share users.
 
         Args:
             None
 
         Returns:
-            dict | None:
-                Dictionary with the Core Share user data or None if the request fails.
+            list | None:
+                List with the Core Share user data or None if the request fails.
 
             Example response (it is a list!):
             [
@@ -2524,7 +2590,8 @@ class CoreShare:
         is_confirmed = self.get_result_value(response=user, key="isConfirmed")
         if not is_confirmed:
             self.logger.info(
-                "User -> %s is not yet confirmed - so it cannot have files to cleanup.",
+                "User -> %s (%s) is not yet confirmed - so it cannot have files to cleanup.",
+                user_login,
                 user_id,
             )
             return True
@@ -2546,14 +2613,15 @@ class CoreShare:
         # Get all folders of the user:
         response = self.get_folders(parent_id=user_root_folder_id)
         if not response or not response["results"]:
-            self.logger.info("User -> %s has no items to cleanup!", user_id)
+            self.logger.info("User -> %s (%s) has no items to cleanup!", user_login, user_id)
         else:
             items = response["results"]
             for item in items:
                 if item["isShared"]:
                     if item["owner"]["id"] == user_id:
                         self.logger.info(
-                            "User -> %s stops sharing item -> %s (%s)...",
+                            "User -> %s (%s) stops sharing item -> %s (%s)...",
+                            user_login,
                             user_id,
                             item["name"],
                             item["id"],

pyxecm/helper/data.py

@@ -857,6 +857,10 @@ class Data:
                 columns = existing_columns
 
             # Attempt to save the data frame to Excel:
+            if self._df is None:
+                self.logger.error(
+                    "Cannot write Excel file -> '%s' from empty / non-initialized data frame!", excel_path
+                )
             self._df.to_excel(
                 excel_path,
                 sheet_name=sheet_name,
@@ -868,29 +872,17 @@ class Data:
                 excel_path,
             )
 
-        except FileNotFoundError:
-            self.logger.error(
-                "Cannot write data frame to Excel file -> '%s'",
-                excel_path,
-            )
+        except FileNotFoundError as fnf_error:
+            self.logger.error("Cannot write data frame to Excel file -> '%s'; error -> %s", excel_path, str(fnf_error))
             return False
-        except PermissionError:
-            self.logger.error(
-                "Cannot write data frame to Excel file -> '%s'",
-                excel_path,
-            )
+        except PermissionError as pe:
+            self.logger.error("Cannot write data frame to Excel file -> '%s'; error -> %s", excel_path, str(pe))
             return False
-        except ValueError:
-            self.logger.error(
-                "Cannot write data frame to Excel file -> '%s'",
-                excel_path,
-            )
+        except ValueError as ve:
+            self.logger.error("Cannot write data frame to Excel file -> '%s'; error -> %s", excel_path, str(ve))
             return False
-        except OSError:
-            self.logger.error(
-                "Cannot write data frame to Excel file -> '%s'",
-                excel_path,
-            )
+        except OSError as ose:
+            self.logger.error("Cannot write data frame to Excel file -> '%s'; error -> %s", excel_path, str(ose))
             return False
 
         return True
@@ -1418,7 +1410,7 @@ class Data:
             for value in values:
                 # Do we want a special treatment for this value (e.g. the current year)
                 if value in special_values:
-                    self.logger.info("Processing special value -> '%s'...", value)
+                    self.logger.debug("Processing special value -> '%s'...", value)
                     if value not in special_url_templates and str(value) not in special_url_templates:
                         self.logger.error(
                             "Cannot find key -> '%s' in special URL templates dictionary -> %s! Skipping...",
@@ -2135,7 +2127,7 @@ class Data:
                     )
                     continue
                 # Apply cleansing to dictionary values in the main column
-                self.logger.info(
+                self.logger.debug(
                     "Cleansing for column -> '%s' has a subfield -> '%s' configured. Do cleansing for dictionary items with key -> '%s'...",
                     column,
                     dict_key,
@@ -2173,7 +2165,7 @@ class Data:
                 # Handle string columns:
                 if self.is_string_column(self._df[column]):
                     # Apply cleansing operations on string column
-                    self.logger.info(
+                    self.logger.debug(
                         "Column -> '%s' has string values. Do cleansing for string values...",
                         column,
                     )
@@ -2203,7 +2195,7 @@ class Data:
                 elif self.is_list_column(self._df[column]):
                     # Handle list-like columns for this we iterate over each list item
                     # and apply the cleansing by calling _apply_string_cleansing() for item:
-                    self.logger.info(
+                    self.logger.debug(
                         "Column -> '%s' has list values. Do cleansing for each list item...",
                         column,
                     )

pyxecm/helper/otel_config.py

@@ -0,0 +1,26 @@
+"""Define OpenTelemtry configuration."""
+
+import os
+
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
+from opentelemetry.instrumentation.threading import ThreadingInstrumentor
+from opentelemetry.sdk.resources import SERVICE_NAME, Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+resource = Resource.create(attributes={SERVICE_NAME: "pyxecm"})
+
+trace.set_tracer_provider(TracerProvider(resource=resource))
+
+if os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"):
+    trace.get_tracer_provider().add_span_processor(
+        BatchSpanProcessor(OTLPSpanExporter()),
+    )
+
+# Auto-instrument requests
+RequestsInstrumentor().instrument()
+ThreadingInstrumentor().instrument()
+
+tracer = trace.get_tracer("pyxecm")

pyxecm/helper/web.py

@@ -326,8 +326,7 @@ class HTTP:
                 )
                 os.makedirs(directory)
             with open(filename, "wb") as download_file:
-                for chunk in response.iter_content(chunk_size=chunk_size):
-                    download_file.write(chunk)
+                download_file.writelines(response.iter_content(chunk_size=chunk_size))
             self.logger.debug(
                 "File downloaded successfully as -> '%s' (size -> %s).",
                 filename,