vavacars-data-utils 1.3.6__tar.gz → 1.4.0__tar.gz

{vavacars_data_utils-1.3.6 → vavacars_data_utils-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: vavacars_data_utils
-Version: 1.3.6
+Version: 1.4.0
 Summary: Package with utils
 Home-page: https://dev.azure.com/vavacars/DataScience/_git/Data.Utils
 Author: Vavacars Data Science Team
@@ -22,10 +22,14 @@ Requires-Dist: sqlalchemy<2.0; extra == "sql"
 Requires-Dist: psycopg2-binary; extra == "sql"
 Requires-Dist: pyodbc; extra == "sql"
 Requires-Dist: msal; extra == "sql"
-Provides-Extra: emai
-Requires-Dist: azure-communication-email; extra == "emai"
+Provides-Extra: email
+Requires-Dist: azureml-core; extra == "email"
+Requires-Dist: azure-communication-email; extra == "email"
 Provides-Extra: azure
 Requires-Dist: azureml-core; extra == "azure"
+Provides-Extra: loganalytics
+Requires-Dist: azureml-core; extra == "loganalytics"
+Requires-Dist: azure-monitor-query; extra == "loganalytics"
 
 # Vavacars
 

{vavacars_data_utils-1.3.6 → vavacars_data_utils-1.4.0}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = vavacars_data_utils
-version = 1.3.6
+version = 1.4.0
 author = Vavacars Data Science Team
 author_email = acg@vava.cars
 description = Package with utils
@@ -40,10 +40,14 @@ sql =
 	psycopg2-binary
 	pyodbc
 	msal
-emai = 
+email = 
+	azureml-core
 	azure-communication-email
 azure = 
 	azureml-core
+loganalytics = 
+	azureml-core
+	azure-monitor-query
 
 [egg_info]
 tag_build = 

{vavacars_data_utils-1.3.6 → vavacars_data_utils-1.4.0}/vava_utils/azure/az_helper.py

@@ -9,6 +9,7 @@ from azureml.core.authentication import ServicePrincipalAuthentication
 from azureml.core.model import Model
 from azureml.data.datapath import DataPath
 
+from azure.storage.blob import BlobServiceClient
 
 from vava_utils.utils.singleton import singleton
 
@@ -26,7 +27,7 @@ class Az_Helper:
                 - service_principal_password
                 - subscription_id
                 - resource_group
-                - workspace_name
+                - workspace
         """
         self.auth = ServicePrincipalAuthentication(
             tenant_id=config["tenant_id"],
@@ -39,6 +40,9 @@ class Az_Helper:
             workspace_name=config["workspace"],
             auth=self.auth,
         )
+        self.blob = BlobServiceClient.from_connection_string(
+            f"DefaultEndpointsProtocol=https;AccountName={self.account_name};AccountKey={account_key};EndpointSuffix=core.windows.net"
+        )
 
     def register_model(self, path, name, include_version=True, **kwargs):
         """
@@ -258,3 +262,15 @@ class Az_Helper:
             tags=tags,
             **kwargs,
         )
+
+    def write_blob_file(self, container, local_file):
+        """
+        Write local file to Azure Blob Storage
+
+        Parameters:
+            container (string): Azure Blob Storage container name
+            local_file (string): local file path
+        """
+        blob = Datastore.get(self.ws, container)
+        blob.upload_files(files=[local_file], target_path=".", overwrite=True)
+

vavacars_data_utils-1.4.0/vava_utils/loganalytics/__init__.py

@@ -0,0 +1 @@
+from .loganalytics_helper import LogAnalyticsHelper

vavacars_data_utils-1.4.0/vava_utils/loganalytics/loganalytics_helper.py

@@ -0,0 +1,51 @@
+import logging
+import pandas as pd
+from azure.identity import ClientSecretCredential
+from azure.monitor.query import LogsQueryClient, LogsQueryStatus
+from azure.core.exceptions import HttpResponseError
+import datetime
+
+
+class UnknownQueryStatusError(Exception):
+    """Custom exception for unknown query status"""
+
+    pass
+
+
+class LogAnalyticsHelper:
+
+    def __init__(
+        self,
+        service_principal_id: str,
+        service_principal_password: str,
+        tentant_id: str,
+        workspace_id: str = "58ca4766-72b7-4610-8d1c-4b6a88a49a01",
+    ):
+        self.client = LogsQueryClient(
+            ClientSecretCredential(tentant_id, service_principal_id, service_principal_password)
+        )
+        self.workspace_id = workspace_id
+
+    def query(self, query: str, start: datetime.datetime = None, end: datetime.datetime = None, **kwargs):
+
+        timespan = None if start is None or end is None else (start, end)
+        response = self.client.query_workspace(workspace_id=self.workspace_id, query=query, timespan=timespan, **kwargs)
+
+        if response.status == LogsQueryStatus.SUCCESS:
+            data = response.tables
+        elif response.status == LogsQueryStatus.PARTIAL:
+            logging.warning(f"Partial query success. Error: {response.partial_error.message}")
+            data = response.partial_data
+        else:
+            logging.error(f"Unknown query status: {response.status}")
+            raise UnknownQueryStatusError(f"Received an unknown query status: {response.status}")
+
+        dataframes = [pd.DataFrame(t.rows, columns=t.columns) for t in data]
+
+        if len(dataframes) == 1:
+            return dataframes[0]
+        elif len(dataframes) > 1:
+            return dataframes
+        else:
+            logging.warning("No data found in the query result")
+            return None

{vavacars_data_utils-1.3.6 → vavacars_data_utils-1.4.0}/vava_utils/sql/sql_helper.py

@@ -49,18 +49,39 @@ def get_mysql_helper(host, user, port, database=None, password=None, **kwargs):
     return SQL_Helper(engine, engine_type=MYSQL)
 
 
-def get_sqlserver_helper(
+def _get_sqlserver_helper_from_user_password(
+    host,
+    database,
+    user,
+    password,
+    port,
+    db_driver,
+    fast_execute_many,
+    **kwargs,
+):
+    conn_str = urllib.parse.quote_plus(
+        f"DRIVER={{{db_driver}}};SERVER={host},{port};DATABASE={database};UID={user};PWD={password}"
+    )
+    engine = create_engine(
+        url=URL.create("mssql+pyodbc", query={"odbc_connect": conn_str}),
+        fast_executemany=fast_execute_many,
+        pool_pre_ping=True,
+        **kwargs,
+    )
+    return SQL_Helper(engine, engine_type=SQL_SERVER)
+
+
+def _get_sqlserver_helper_from_service_principal(
     host,
     database,
     service_principal_id,
     service_principal_secret,
     tenant_id,
-    port=1433,
-    db_driver="ODBC Driver 17 for SQL Server",
-    fast_execute_many=True,
+    port,
+    db_driver,
+    fast_execute_many,
     **kwargs,
 ):
-
     creds = ConfidentialClientApplication(
         client_id=service_principal_id,
         client_credential=service_principal_secret,
@@ -89,6 +110,49 @@ def get_sqlserver_helper(
     return SQL_Helper(engine, engine_type=SQL_SERVER)
 
 
+def get_sqlserver_helper(
+    host,
+    database,
+    service_principal_id=None,
+    service_principal_secret=None,
+    tenant_id=None,
+    user=None,
+    password=None,
+    port=1433,
+    db_driver="ODBC Driver 17 for SQL Server",
+    fast_execute_many=True,
+    **kwargs,
+):
+
+    if user and password:
+        return _get_sqlserver_helper_from_user_password(
+            host,
+            database,
+            user,
+            password,
+            port,
+            db_driver,
+            fast_execute_many,
+            **kwargs,
+        )
+    elif service_principal_id and service_principal_secret and tenant_id:
+        return _get_sqlserver_helper_from_service_principal(
+            host,
+            database,
+            service_principal_id,
+            service_principal_secret,
+            tenant_id,
+            port,
+            db_driver,
+            fast_execute_many,
+            **kwargs,
+        )
+    else:
+        raise ValueError(
+            "Either user and password or service_principal_id, service_principal_secret and tenant_id must be provided"
+        )
+
+
 def _value_to_str(v):
     if isinstance(v, bool):
         return str(int(v))

{vavacars_data_utils-1.3.6 → vavacars_data_utils-1.4.0}/vavacars_data_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: vavacars_data_utils
-Version: 1.3.6
+Version: 1.4.0
 Summary: Package with utils
 Home-page: https://dev.azure.com/vavacars/DataScience/_git/Data.Utils
 Author: Vavacars Data Science Team
@@ -22,10 +22,14 @@ Requires-Dist: sqlalchemy<2.0; extra == "sql"
 Requires-Dist: psycopg2-binary; extra == "sql"
 Requires-Dist: pyodbc; extra == "sql"
 Requires-Dist: msal; extra == "sql"
-Provides-Extra: emai
-Requires-Dist: azure-communication-email; extra == "emai"
+Provides-Extra: email
+Requires-Dist: azureml-core; extra == "email"
+Requires-Dist: azure-communication-email; extra == "email"
 Provides-Extra: azure
 Requires-Dist: azureml-core; extra == "azure"
+Provides-Extra: loganalytics
+Requires-Dist: azureml-core; extra == "loganalytics"
+Requires-Dist: azure-monitor-query; extra == "loganalytics"
 
 # Vavacars
 

{vavacars_data_utils-1.3.6 → vavacars_data_utils-1.4.0}/vavacars_data_utils.egg-info/SOURCES.txt

@@ -12,6 +12,8 @@ setup.cfg
 ./vava_utils/camunda/camunda_helper.py
 ./vava_utils/email/__init__.py
 ./vava_utils/email/email_helper.py
+./vava_utils/loganalytics/__init__.py
+./vava_utils/loganalytics/loganalytics_helper.py
 ./vava_utils/salesforce/__init__.py
 ./vava_utils/salesforce/salesforce_helper.py
 ./vava_utils/smartiq/__init__.py
@@ -30,6 +32,37 @@ setup.cfg
 ./vava_utils/utils/utils.py
 ./vava_utils/vavaprice/__init__.py
 ./vava_utils/vavaprice/vavaprice_helper.py
+test/utils/__init__.py
+test/utils/test_strings.py
+vava_utils/__init__.py
+vava_utils/azure/__init__.py
+vava_utils/azure/az_helper.py
+vava_utils/bigquery/__init__.py
+vava_utils/bigquery/bigquery_helper.py
+vava_utils/camunda/__init__.py
+vava_utils/camunda/camunda_helper.py
+vava_utils/email/__init__.py
+vava_utils/email/email_helper.py
+vava_utils/loganalytics/__init__.py
+vava_utils/loganalytics/loganalytics_helper.py
+vava_utils/salesforce/__init__.py
+vava_utils/salesforce/salesforce_helper.py
+vava_utils/smartiq/__init__.py
+vava_utils/smartiq/smartiq_helper.py
+vava_utils/smartiq/smartiq_helper_v2.py
+vava_utils/sql/__init__.py
+vava_utils/sql/sql_helper.py
+vava_utils/turkey/__init__.py
+vava_utils/turkey/dates.py
+vava_utils/utils/__init__.py
+vava_utils/utils/callables.py
+vava_utils/utils/comparisons.py
+vava_utils/utils/naming.py
+vava_utils/utils/singleton.py
+vava_utils/utils/strings.py
+vava_utils/utils/utils.py
+vava_utils/vavaprice/__init__.py
+vava_utils/vavaprice/vavaprice_helper.py
 vavacars_data_utils.egg-info/PKG-INFO
 vavacars_data_utils.egg-info/SOURCES.txt
 vavacars_data_utils.egg-info/dependency_links.txt

{vavacars_data_utils-1.3.6 → vavacars_data_utils-1.4.0}/vavacars_data_utils.egg-info/requires.txt

@@ -10,9 +10,14 @@ azureml-core
 google-cloud-bigquery
 google-api-core
 
-[emai]
+[email]
+azureml-core
 azure-communication-email
 
+[loganalytics]
+azureml-core
+azure-monitor-query
+
 [sql]
 sqlalchemy<2.0
 psycopg2-binary