flexmetric 0.4.2__py3-none-any.whl → 0.4.4__py3-none-any.whl

flexmetric/metric_process/database_connections.py

@@ -0,0 +1,55 @@
+import clickhouse_connect
+import sqlite3
+from flexmetric.logging_module.logger import get_logger
+import os
+
+logger = get_logger(__name__)
+
+logger.info("database logs")
+
+def create_clickhouse_client(db_conf):
+    id = db_conf.get('id')
+    host = db_conf.get('host', 'localhost')
+    port = db_conf.get('port', 9440)
+    username = db_conf.get('username', 'default')
+    password = db_conf.get('password', '')
+
+    client_cert = db_conf.get('client_cert')
+    client_cert_key = db_conf.get('client_cert_key')
+    ca_cert = db_conf.get('ca_cert')
+
+    secure = bool(client_cert and client_cert_key and ca_cert)
+
+    settings = {
+        'host': host,
+        'port': port,
+        'username': username,
+        'password': password,
+        'secure': secure,
+    }
+
+    if secure:
+        settings.update({
+            'client_cert': client_cert,
+            'client_cert_key': client_cert_key,
+            'secure_ca': ca_cert,
+        })
+
+    client = clickhouse_connect.get_client(**settings)
+    logger.info(f"Clickhouse connection '{id}' created")
+    return client
+
+
+def create_sqlite_client(db_conf: dict):
+    db_path = db_conf.get('db_connection')
+    db_name = db_conf.get('db_name', 'default_sqlite_db')
+
+    if not db_path or not os.path.isfile(db_path):
+        raise FileNotFoundError(f"SQLite database file not found at {db_path}")
+
+    try:
+        conn = sqlite3.connect(db_path)
+        logger.info(f"SQLite connection '{db_name}' created")
+        return conn
+    except Exception as e:
+        raise ConnectionError(f"Failed to create SQLite client: {e}")

flexmetric/metric_process/database_processing.py

@@ -1,6 +1,11 @@
 import yaml
-import sqlite3
 import re
+from flexmetric.metric_process.database_connections import create_clickhouse_client,create_sqlite_client
+from flexmetric.metric_process.queries_execution import execute_clickhouse_command,execute_sqlite_query
+from flexmetric.logging_module.logger import get_logger
+logger = get_logger(__name__)
+
+logger.info("query execution")
 
 
 def read_yaml_file(file_path):
@@ -17,74 +22,119 @@ def get_database_config(databases, db_name):
     )
 
 
-def execute_sqlite_query(db_path, query):
-    try:
-        conn = sqlite3.connect(db_path)
-        cursor = conn.cursor()
-        cursor.execute(query)
-        result = cursor.fetchone()
-        conn.close()
-        return float(result[0]) if result and result[0] is not None else None
-    except Exception as e:
-        print(f"[ERROR] SQLite query failed on {db_path}: {e}")
-        return None
-
 
 def is_safe_query(query):
-    # Remove leading spaces and brackets
     cleaned_query = query.strip().lower()
-    # Match only queries that start with "select"
     return re.match(r"^\(*\s*select", cleaned_query) is not None
 
 
-def process_database_queries(queries_file, databases_file):
-    # Get queries from queries file
-    queries_config = read_yaml_file(queries_file)
-    # Get database from database file
-    databases_config = read_yaml_file(databases_file)
+def read_yaml_file(filepath: str) -> dict:
+    try:
+        with open(filepath, 'r') as f:
+            return yaml.safe_load(f)
+    except Exception as e:
+        raise Exception(f"Error reading {filepath}: {e}")
 
-    commands = queries_config.get("commands", [])
-    databases = databases_config.get("databases", [])
 
-    all_results = []
+def create_clients_from_config(config_file: str):
+    db_configs = read_yaml_file(config_file).get('databases', [])
+    clients = {}
+
+    for db_conf in db_configs:
+        db_id = db_conf.get('id')
+        logger.info(db_id)
+        db_type = db_conf.get('type')
+
+        if not db_id:
+            logger.info(f"Skipping unnamed database block: {db_conf}")
+            continue
 
-    for cmd in commands:
         try:
-            db_conf = get_database_config(databases, cmd["database"])
+            if db_type == 'clickhouse':
+                client = create_clickhouse_client(db_conf)
+                clients[db_id] = client
+                logger.info(clients)
 
-            if db_conf["db_type"] != "sqlite":
-                print(
-                    f"[WARN] Unsupported database type: {db_conf['db_type']} in command {cmd['name']}"
-                )
-                continue
+            elif db_type == 'sqlite':
+                client = create_sqlite_client(db_conf)
+                clients[db_id] = client
+                logger.info(clients)
 
-            db_path = db_conf["db_connection"]
-            query = cmd["query"]
-            labels = cmd.get('labels', [])
-            label_values = cmd.get('label_values', [])
-            main_label = cmd.get('main_label', 'default_db_metric')
+            else:
+                logger.info(f"Unsupported database type: {db_type}")
 
-            # check if query is safe
-            if is_safe_query(query):
-                value = execute_sqlite_query(db_path, query)
+        except Exception as e:
+            logger.error(f"Failed to create client '{db_id}': {e}")
+    return clients
+
+def process_and_get_value(cmd,rows_data,column_names):
+    labels = cmd.get('labels', [])
+    value_column = cmd.get('value_column', [])
+    label_values = cmd.get('label_values', [])
+    main_label = cmd.get('main_label', 'default_db_metric')
+    missing_columns = [col for col in labels + [value_column] if col not in column_names]
+    if missing_columns:
+        raise ValueError(f"Missing columns in result: {missing_columns}")
+    rows = []
+    for row in rows_data:
+        label_values = [str(row[column_names.index(col)]) for col in labels]
+        value = row[column_names.index(value_column)]
+
+        rows.append({
+            'label': label_values,
+            'value': value
+        })
+
+    return {
+        'result': rows,
+        'labels': labels,
+        'main_label': main_label
+    }
+
+def execute_commands(clients: dict, commands_file: str):
+    commands = read_yaml_file(commands_file).get('commands', [])
+    results = []
+    for cmd in commands:
+        print("CMD : ",cmd)
+        if cmd == None:
+            continue
+        cmd_id = cmd.get('id')
+        db_id = cmd.get('database_id')
+        db_type = cmd.get('type')
+        query = cmd.get('query')
+        if not cmd_id or not db_id or not db_type or not query:
+            logger.info(f"Missing required fields in command '{cmd_id}'")
+            continue
+
+        client = clients.get(db_id)
+        logger.info(clients)
+        logger.info(db_id)
+        logger.info(client)
+        if not client:
+            logger.info(f"No client for database_id '{db_id}' in command '{cmd_id}'")
+            continue
+        try:
+            if db_type == 'clickhouse':
+                logger.info("In clickhouse")
+                response,column_names = execute_clickhouse_command(client,query)
+                result = process_and_get_value(cmd,response,column_names)
+                results.append(result)
+            elif db_type == 'sqlite':
+                response ,column_names = execute_sqlite_query(client,query)
+                result = process_and_get_value(cmd,response,column_names)
+                results.append(result)
             else:
-                print(f"[WARN] Unsupported query type: {query}")
-                return None
-
-            if not isinstance(label_values, list):
-                label_values = [label_values]
-
-            result = {
-                'result': [{
-                    'label': label_values,
-                    'value': value
-                }],
-                'labels': labels,
-                'main_label': main_label
-            }
-            all_results.append(result)
+                logger.info(f"Unknown type '{db_type}' in command '{cmd_id}'")
+                continue
+                
         except Exception as e:
-            print(
-                f"[ERROR] Processing command '{cmd.get('name', 'unknown')}' failed: {e}"
-            )
-    return all_results
+            logger.error(f"Command '{cmd_id}' failed: {e}")
+    return results
+
+
+def process_database_queries(queries_file, databases_file):
+    try:
+        client_configs = create_clients_from_config(databases_file)
+        return execute_commands(clients=client_configs,commands_file=queries_file)
+    except Exception as ex:
+        logger.error(f"Exception : {ex}")

flexmetric/metric_process/prometheus_agent.py

@@ -256,4 +256,4 @@ def main():
         run_flask(args.host, args.port)
 # # args = arguments()
 # # measure(args)
-# main()
+main()

flexmetric/metric_process/queries_execution.py

@@ -0,0 +1,24 @@
+from flexmetric.logging_module.logger import get_logger
+logger = get_logger(__name__)
+
+logger.info("query execution")
+
+def execute_clickhouse_command(client, command: str):
+    try:
+        result = client.query(command)
+        row_list = result.result_rows
+        column_names = result.column_names
+        return row_list,column_names
+    except Exception as e:
+        logger.error(f"Error executing command: {e}")
+        return None
+def execute_sqlite_query(conn, query):
+    try:
+        cursor = conn.cursor()
+        cursor.execute(query)
+        rows = cursor.fetchall()
+        column_names = [desc[0] for desc in cursor.description]
+        return rows, column_names
+    except Exception as ex:
+        logger.error(f"Exception during SQLite query: {ex}")
+        return [], []

{flexmetric-0.4.2.dist-info → flexmetric-0.4.4.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: flexmetric
-Version: 0.4.2
-Summary: A secure flexible Prometheus exporter for commands, databases, functions, and scripts.
+Version: 0.4.4
+Summary: A secure flexible Prometheus exporter for commands, databases, functions.
 Home-page: https://github.com/nikhillingadhal1999/flexmetric
 Author: Nikhil Lingadhal
 License: MIT
@@ -20,6 +20,7 @@ Requires-Dist: setuptools
 Requires-Dist: wheel
 Requires-Dist: twine
 Requires-Dist: flask
+Requires-Dist: clickhouse-connect
 Dynamic: author
 Dynamic: classifier
 Dynamic: description
@@ -188,21 +189,52 @@ Filesystem      Size  Used Avail Use% Mounted on
 | `timeout_seconds` | Maximum time (in seconds) to wait for the command to complete. If it exceeds this time, the command is aborted.             |
 
 ## Database mode
-
+file - database.yaml
 ```yaml
 databases:
-  - name: mydb
-    db_type: sqlite
-    db_connection: /path/to/my.db
-````
+  - id: "active_user_count"
+    type: "clickhouse"
+    host: "localhost"
+    port: 8123
+    username: "default"
+    password: ""
+    client_cert: ""
+    client_key: ""
+    ca_cert: ""
+
+  - id: "userdb"
+    type: "sqlite"
+    db_connection: "/path/to/my.db"
+```
+file - queries.yaml
 ```yaml
 commands:
-  - name: user_count
-    database: userdb
-    query: "SELECT COUNT(*) FROM users;"
-    main_label: database_user_count
-    labels: ["database_name", "table_name"]
-    label_values: ["userdb", "users"]
+  - id: "active_user_count"
+    type: "clickhouse"
+    database_id: "active_user_count"
+    query: |
+      SELECT
+        country AS country_name,
+        COUNT() AS active_user_count
+      FROM users
+      WHERE is_active = 1
+      GROUP BY country
+    main_label: "active_user_count"
+    labels: ["country_name"]
+    value_column: "active_user_count"
+
+  - id: "list_all_users_sqlite"
+    type: "sqlite"
+    database_id: "userdb"
+    query: |
+      SELECT
+        id,
+        name
+      FROM users
+    main_label: "user_list"
+    labels: ["id", "name"]
+    value_column: "id"
+
 ```
 ## Functions mode
 

{flexmetric-0.4.2.dist-info → flexmetric-0.4.4.dist-info}/RECORD

@@ -6,13 +6,15 @@ flexmetric/file_recognition/exec_file.py,sha256=9wBbnqPConxtLhqWTgigEr8VQG-fa9K_
 flexmetric/logging_module/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flexmetric/logging_module/logger.py,sha256=hXj9m2Q_KxJVI5YRHRoK6PXV5tO6NmvuVjq2Ipx_1tE,447
 flexmetric/metric_process/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-flexmetric/metric_process/database_processing.py,sha256=hbVbzIdO21NF0F3nILJ4d1x8fkks9P8s-wtn8qZ91qw,2739
+flexmetric/metric_process/database_connections.py,sha256=CqYrzXkwoB7yq4SpIu7_jDDNHSemaPsfDJj5DLwHSXM,1574
+flexmetric/metric_process/database_processing.py,sha256=_7C1DyC9jtgiTUti4_YYFpQlE0BlJwbaEx3OPpDCGSs,4633
 flexmetric/metric_process/expiring_queue.py,sha256=1oC0MjloitPiRo7yDgVarz81ETEQavKI_W-GFUvp5_Y,1920
 flexmetric/metric_process/process_commands.py,sha256=clGMQhLNcuJUO1gElpAS9Dyk0KU5w41DIguczjo7ceA,4089
-flexmetric/metric_process/prometheus_agent.py,sha256=rZ9pQrqA3J_-sJtl48qkFIHIUH01iert05u4SmGN4Yc,7714
+flexmetric/metric_process/prometheus_agent.py,sha256=pDPqiPtHLv1cfF6dPRNg2LtAhVF1UKf8MNZ_S9xuRQY,7712
+flexmetric/metric_process/queries_execution.py,sha256=NdnQ9Flbc8_6VOSY3aHxSmhcCsZf2zIxKTnc0hCp1oU,790
 flexmetric/metric_process/views.py,sha256=BY695dCpTkJRc1KLC9RNpFTieFdHeHvyqyefmHhMauE,3297
-flexmetric-0.4.2.dist-info/METADATA,sha256=qu-Y5W68ysqbjGsw-6NtQUC8D166f78-ivkUcdzvLls,10651
-flexmetric-0.4.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-flexmetric-0.4.2.dist-info/entry_points.txt,sha256=urVePn5EWr3JqNvkYP7OsB_h2_Bqvv-Wq1MJRBexm8A,79
-flexmetric-0.4.2.dist-info/top_level.txt,sha256=zBlrNwKhXUNhgu9RRZnXxYwYnmE-eocRe6wKSmQROA4,11
-flexmetric-0.4.2.dist-info/RECORD,,
+flexmetric-0.4.4.dist-info/METADATA,sha256=EvJA9-Ov8X-KyO810Qj_UQsDeoEA38SyLTcnKU-6dgc,11276
+flexmetric-0.4.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+flexmetric-0.4.4.dist-info/entry_points.txt,sha256=urVePn5EWr3JqNvkYP7OsB_h2_Bqvv-Wq1MJRBexm8A,79
+flexmetric-0.4.4.dist-info/top_level.txt,sha256=zBlrNwKhXUNhgu9RRZnXxYwYnmE-eocRe6wKSmQROA4,11
+flexmetric-0.4.4.dist-info/RECORD,,