regscale-cli 6.16.0.0__py3-none-any.whl

regscale/utils/string.py

@@ -0,0 +1,121 @@
+"""Define commonly used methods for strings."""
+
+import re
+from typing import Optional
+
+ANSI_ESCAPE_REGEX = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
+
+
+def remove_ansi_escape_sequences(text: str) -> str:
+    """Remove all ANSI escape sequences from a text
+
+    :param str text: a string to remove ANSI escape sequences from
+    :return: a string with no ANSI escape sequences
+    :rtype: str
+    """
+    return ANSI_ESCAPE_REGEX.sub("", text)
+
+
+def replace_unknown_spaces_and_newlines(input_string: str) -> str:
+    """Replace unknown spaces and newlines with a single space
+
+    :param str input_string: a string to replace unknown spaces and newlines with a single space
+    :return: a string with unknown spaces and newlines replaced with a single space
+    :rtype: str
+    """
+    return re.sub(r"\s+", " ", input_string)
+
+
+def extract_double_curly_braces_contents(query_string: str) -> list[str]:
+    """Extract the contents of double curly braces from a string
+
+    :param str query_string: a string to extract the contents of double curly braces from
+    :return: a list of strings that were contained within double curly braces
+    :rtype: list[str]
+    """
+    pattern = r"{{(.*?)}}"
+    return re.findall(pattern, query_string)
+
+
+def extract_dag_run_conf_key(query_string: str) -> Optional[str]:
+    """
+    Extract the key from a dag_run.conf key-value pair
+
+    :param str query_string: a string to extract the key from a dag_run.conf key-value pair
+    :return: the key from a dag_run.conf key-value pair
+    :rtype: Optional[str]
+    """
+    pattern = r"dag_run\.conf\[['\"]([^'\"]+)['\"]\]"
+    match = re.search(pattern, query_string)
+    return match[1] if match else None
+
+
+def extract_param(query_string: str) -> Optional[list]:
+    """
+    Extract the parameter from a params key-value pair
+
+    :param str query_string: a string to extract the parameter from a params key-value pair
+    :return: the parameter from a params key-value pair
+    :rtype: Optional[list]
+    """
+    pattern = r"\bparams\.(\w+)\s.*"
+    matches = re.findall(pattern, query_string)
+    return matches[0] if matches else None
+
+
+def levenshtein_distance(first_string: str, second_string: str) -> int:
+    """
+    Calculate the Levenshtein distance between two strings
+
+    :param str first_string: the first string to compare
+    :param str second_string: the second string to compare
+    :return: the Levenshtein distance between two strings
+    :rtype: int
+    """
+    if len(first_string) < len(second_string):
+        return levenshtein_distance(second_string, first_string)
+
+    if len(second_string) == 0:
+        return len(first_string)
+
+    previous_row = range(len(second_string) + 1)
+    for i, c1 in enumerate(first_string):
+        current_row = [i + 1]
+        for j, c2 in enumerate(second_string):
+            insertions = previous_row[j + 1] + 1
+            deletions = current_row[j] + 1
+            substitutions = previous_row[j] + (c1 != c2)
+            current_row.append(min(insertions, deletions, substitutions))
+        previous_row = current_row
+
+    return previous_row[-1]
+
+
+def parse_last_element(string: str, split_str: Optional[str] = "/") -> str:
+    """
+    Function to parse last element of a string
+
+    :param str string: String to parse
+    :param str split_str: String to split the string
+    :return: Last element of the string
+    :rtype: str
+    """
+    try:
+        return string.split(split_str)[-1]
+    except IndexError:
+        return ""
+
+
+def generate_html_table_from_dict(data: dict) -> str:
+    """
+    Function to map all key-value pairs to a description HTML Table
+
+    :param dict data: Data from Microsoft Defender
+    :return: HTML table as a string
+    :rtype: str
+    """
+    description = "<table>"
+    for key, value in data.items():
+        description += f"<tr><td>{key}</td><td>{value}</td></tr>"
+    description += "</table>"
+    return description

regscale/utils/synqly_utils.py

@@ -0,0 +1,165 @@
+"""Utility functions for interacting with the Synqly API"""
+
+import signal
+
+import httpx
+
+# Synqly Python SDK imports
+from synqly import management as mgmt
+from synqly.engine.client import SynqlyEngine
+from synqly.management.client import SynqlyManagement
+from regscale.core.app.utils.app_utils import create_logger
+
+logger = create_logger()
+
+
+# This class is an example of a multi-tenant application, meant
+# to represent your application or product. It contains a pool of tenants, each
+# of which might represent a user or organization within your application.
+#
+# This example omits application specific logic, focusing instead on the
+# steps required to configure a Synqly Integration for each tenant.
+class App:
+    """
+    This class is an example of a multi-tenant application, meant
+    to represent your application or product. It contains a pool of tenants, each
+    of which might represent a user or organization within your application.
+
+    This example omits application specific logic, focusing instead on the
+    steps required to configure a Synqly Integration for each tenant.
+
+    Parameters:
+        connector_type - The target Connector. Example: "ticketing"
+
+    Attributes:
+        connector_type - The target Connector. Example: "ticketing"
+        tenants - A dictionary of Tenant objects, keyed by tenant name.
+    """
+
+    def __init__(self, connector_type):
+        self.tenants = {}
+        self.connector_type = connector_type
+        # Initialize signal handlers to intercept Ctrl-C and perform cleanup
+        signal.signal(signal.SIGINT, lambda signal, frame: self._cleanup_handler())
+        signal.signal(signal.SIGTERM, lambda signal, frame: self._cleanup_handler())
+        self.terminated = False
+
+    def _cleanup_handler(self):
+        """
+        Deletes Synqly resources created by the example program.
+        """
+        if len(self.tenants) != 0:
+            logger.info("\nCleaning up Synqly Resources:")
+            for tenant in self.tenants.values():
+                # Delete the Synqly Account
+                tenant.synqly_management_client.accounts.delete(tenant.synqly_account_id)
+                logger.debug("Cleaned up Account " + tenant.synqly_account_id)
+            self.tenants = {}
+            self.terminated = True
+            logger.info("Cleanup complete.")
+
+    def new_tenant(self, synqly_org_token: str, new_tenant_name: str):
+        """
+        Adds a new "tenant" to the App. A tenant represents a user or
+        organization within your application.
+
+        :param str synqly_org_token: The Synqly Organization token
+        :param str new_tenant_name: The name of the new tenant
+        :raises ValueError: If the tenant already exists
+        """
+        # Make sure the tenant doesn't already exist
+        for tenant in self.tenants.values():
+            if tenant.synqly_account_id == new_tenant_name:
+                raise ValueError("Duplicate tenant id: " + id)
+
+        """
+        Create a Synqly Management API client. The client stores a token,
+        allowing us to make calls to the Synqly Management API. The Management
+        API is used to create Synqly Accounts and Integrations.
+        """
+        # (Optional) configure a custom httpx_client so that all errors are retried
+        transport = httpx.HTTPTransport(retries=3)
+        management_client = SynqlyManagement(
+            token=synqly_org_token,
+            httpx_client=httpx.Client(transport=transport),
+        )
+
+        """
+        Each tenant needs an associated Account in Synqly, so we create that now.
+        """
+        account_request = mgmt.CreateAccountRequest(fullname=new_tenant_name)
+        account_response = management_client.accounts.create(request=account_request)
+        account_id = account_response.result.account.id
+
+        self.tenants[new_tenant_name] = Tenant(
+            tenant_name=new_tenant_name,
+            synqly_account_id=account_id,
+            synqly_management_client=management_client,
+            # The Synqly Engine client will be initialized in configure_integration,
+            # so we leave it as None for now.
+            synqly_engine_client=None,
+        )
+
+    def configure_integration(self, tenant_name: str, provider_config: mgmt.ProviderConfig):
+        """
+        Configures a Synqly Integration for a simulated tenant
+
+        :param str tenant_name: The name of the tenant
+        :param mgmt.ProviderConfig provider_config: The provider configuration
+        :raises TenantNotFoundException: If the tenant name is not found
+        """
+
+        # Locate the tenant in the App's tenant pool
+        if tenant_name not in self.tenants.keys():
+            raise TenantNotFoundException(tenant_name + " not found")
+
+        tenant = self.tenants[tenant_name]
+
+        # Use the Management API to create a Synqly Integration
+        integration_req = mgmt.CreateIntegrationRequest(
+            fullname="Python SDK Integration",
+            provider_config=provider_config,
+        )
+        integration_resp = tenant.synqly_management_client.integrations.create(
+            account_id=tenant.synqly_account_id, request=integration_req
+        )
+        logger.debug(
+            "Created {} Integration '{}' for {}".format(
+                integration_resp.result.integration.category,
+                integration_resp.result.integration.id,
+                tenant_name,
+            )
+        )
+
+        # Add Synqly Engine client to the Tenant for use in the background job
+        self.tenants[tenant_name].synqly_engine_client = SynqlyEngine(
+            token=integration_resp.result.token.access.secret,
+        )
+
+
+class Tenant:
+    """
+    Represents a tenant within the application. A tenant could be a user,
+    organization, or any other entity for which it would make sense to create
+    an Integration.
+    """
+
+    def __init__(
+        self,
+        tenant_name,
+        synqly_account_id,
+        synqly_management_client,
+        synqly_engine_client,
+    ):
+        self.tenant_name = tenant_name
+        self.synqly_account_id = synqly_account_id
+        self.synqly_management_client = synqly_management_client
+        self.synqly_engine_client = synqly_engine_client
+
+
+class TenantNotFoundException(Exception):
+    """
+    Exception raised when a tenant is not found
+    """
+
+    pass

regscale/utils/threading/__init__.py

@@ -0,0 +1,8 @@
+"""
+Use of importing the threading module from the core.utils package
+"""
+
+from .threadhandler import ThreadManager
+from .threadsafe_counter import ThreadSafeCounter
+from .threadsafe_dict import ThreadSafeDict
+from .threadsafe_list import ThreadSafeList

regscale/utils/threading/threadhandler.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""Thread handler to create and start threads"""
+
+from concurrent.futures import ThreadPoolExecutor, TimeoutError, as_completed
+from typing import Any, Callable, List, Tuple, Union
+
+
+class ThreadManager:
+    """
+    Class to manage threads and tasks
+    """
+
+    def __init__(self, max_workers: int = 20):
+        self.executor = ThreadPoolExecutor(max_workers=max_workers)
+        self.futures = []
+
+    def submit_task(self, func: Callable, *args, **kwargs) -> None:
+        """
+        Submit a task to the executor
+
+        :param Callable func: Function to execute
+        :rtype: None
+        """
+        future = self.executor.submit(func, *args, **kwargs)
+        self.futures.append(future)
+
+    def submit_tasks_from_list(self, func: Callable, items: List[Any], *args, **kwargs) -> None:
+        """
+        Submit multiple tasks to the executor, used for a list of items
+
+        :param Callable func: Function to execute
+        :param List[Any] items: List of items to process
+        :rtype: None
+        """
+        for item in items:
+            self.submit_task(func, item, *args, **kwargs)
+
+    def execute_and_verify(
+        self,
+        timeout: int = None,
+        check_for_failures: bool = True,
+        terminate_after: bool = False,
+        return_passed: bool = False,
+    ) -> Union[List[Tuple[bool, Any]], List[Any]]:
+        """
+        Execute the tasks and verify if they were successful
+
+        :param int timeout: Timeout for the tasks
+        :param bool check_for_failures: Whether to check for failures, default True
+        :param bool terminate_after: Whether to terminate the executor after the tasks are completed, default False
+        :param bool return_passed: Whether to return only the passed tasks, default False
+        :return: List of tuples with a bool indicating success & the result, or list of results if return_passed is True
+        :rtype: Union[List[Tuple[bool, Any]], List[Any]]
+        """
+        results = []
+        try:
+            for future in as_completed(self.futures, timeout=timeout):
+                try:
+                    result = future.result()
+                    results.append((True, result))
+                except TimeoutError:
+                    results.append((False, "Task timed out"))
+                except Exception as e:
+                    results.append((False, str(e)))
+        finally:
+            self.futures = []
+            if terminate_after:
+                self.shutdown()
+        if check_for_failures:
+            import logging
+
+            logger = logging.getLogger(__name__)
+            for success, result in results:
+                if not success:
+                    logger.error(f"Task failed with error: {result}")
+        if return_passed:
+            return [result for success, result in results if success]
+        # clear the futures list
+        return results
+
+    def shutdown(self, wait: bool = True) -> None:
+        """
+        Shutdown the executor
+
+        :param bool wait: Whether to wait for the tasks to complete
+        :rtype: None
+        """
+        self.executor.shutdown(wait=wait)
+
+
+def create_threads(process: Callable, args: Tuple, thread_count: int) -> None:
+    """
+    Function to create x threads using ThreadPoolExecutor
+
+    :param Callable process: function for the threads to execute
+    :param Tuple args: args for the provided process
+    :param int thread_count: # of threads needed
+    :rtype: None
+    """
+    # set max threads
+    from regscale.core.app.application import Application
+
+    app = Application()
+    max_threads = app.config["maxThreads"]
+    if threads := min(thread_count, max_threads):
+        # start the threads with the number of threads allowed
+        with ThreadPoolExecutor(max_workers=threads) as executor:
+            # iterate and start the threads that were requested
+            for thread in range(threads):
+                # assign each thread the passed process and args along with the thread number
+                executor.submit(process, args, thread)
+
+
+def thread_assignment(thread: int, total_items: int) -> list:
+    """
+    Function to iterate through items and returns a list the
+    provided thread should be assigned and use during its execution
+
+    :param int thread: current thread number
+    :param int total_items: Total # of items to process with threads
+    :return: List of items to process for the given thread
+    :rtype: list
+    """
+    from regscale.core.app.application import Application
+
+    app = Application()
+    # set max threads
+    max_threads = app.config["maxThreads"]
+
+    return [x for x in range(total_items) if x % max_threads == thread]

regscale/utils/threading/threadsafe_counter.py

@@ -0,0 +1,47 @@
+"""
+This module contains the ThreadSafeCounter class, which is a thread-safe counter.
+"""
+
+import threading
+
+
+class ThreadSafeCounter:
+    """
+    Class to create a thread-safe counter
+    """
+
+    def __init__(self):
+        self.value: int = 0
+        self._lock = threading.Lock()
+
+    def increment(self) -> int:
+        """
+        Increment the counter by 1
+
+        :return: The new value of the counter after incrementing
+        :rtype: int
+        """
+        with self._lock:
+            self.value += 1
+            return self.value
+
+    def decrement(self) -> int:
+        """
+        Decrement the counter by 1
+
+        :return: The new value of the counter after decrementing
+        :rtype: int
+        """
+        with self._lock:
+            self.value -= 1
+            return self.value
+
+    def set(self, value: int) -> None:
+        """
+        Set the counter to a value
+
+        :param int value: Value to set the counter to
+        :rtype: None
+        """
+        with self._lock:
+            self.value = value