flexmetric 0.4.2__tar.gz → 0.4.4__tar.gz

{flexmetric-0.4.2 → flexmetric-0.4.4}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: flexmetric
-Version: 0.4.2
-Summary: A secure flexible Prometheus exporter for commands, databases, functions, and scripts.
+Version: 0.4.4
+Summary: A secure flexible Prometheus exporter for commands, databases, functions.
 Home-page: https://github.com/nikhillingadhal1999/flexmetric
 Author: Nikhil Lingadhal
 License: MIT
@@ -20,6 +20,7 @@ Requires-Dist: setuptools
 Requires-Dist: wheel
 Requires-Dist: twine
 Requires-Dist: flask
+Requires-Dist: clickhouse-connect
 Dynamic: author
 Dynamic: classifier
 Dynamic: description
@@ -188,21 +189,52 @@ Filesystem      Size  Used Avail Use% Mounted on
 | `timeout_seconds` | Maximum time (in seconds) to wait for the command to complete. If it exceeds this time, the command is aborted.             |
 
 ## Database mode
-
+file - database.yaml
 ```yaml
 databases:
-  - name: mydb
-    db_type: sqlite
-    db_connection: /path/to/my.db
-````
+  - id: "active_user_count"
+    type: "clickhouse"
+    host: "localhost"
+    port: 8123
+    username: "default"
+    password: ""
+    client_cert: ""
+    client_key: ""
+    ca_cert: ""
+
+  - id: "userdb"
+    type: "sqlite"
+    db_connection: "/path/to/my.db"
+```
+file - queries.yaml
 ```yaml
 commands:
-  - name: user_count
-    database: userdb
-    query: "SELECT COUNT(*) FROM users;"
-    main_label: database_user_count
-    labels: ["database_name", "table_name"]
-    label_values: ["userdb", "users"]
+  - id: "active_user_count"
+    type: "clickhouse"
+    database_id: "active_user_count"
+    query: |
+      SELECT
+        country AS country_name,
+        COUNT() AS active_user_count
+      FROM users
+      WHERE is_active = 1
+      GROUP BY country
+    main_label: "active_user_count"
+    labels: ["country_name"]
+    value_column: "active_user_count"
+
+  - id: "list_all_users_sqlite"
+    type: "sqlite"
+    database_id: "userdb"
+    query: |
+      SELECT
+        id,
+        name
+      FROM users
+    main_label: "user_list"
+    labels: ["id", "name"]
+    value_column: "id"
+
 ```
 ## Functions mode
 

{flexmetric-0.4.2 → flexmetric-0.4.4}/README.md

@@ -155,21 +155,52 @@ Filesystem      Size  Used Avail Use% Mounted on
 | `timeout_seconds` | Maximum time (in seconds) to wait for the command to complete. If it exceeds this time, the command is aborted.             |
 
 ## Database mode
-
+file - database.yaml
 ```yaml
 databases:
-  - name: mydb
-    db_type: sqlite
-    db_connection: /path/to/my.db
-````
+  - id: "active_user_count"
+    type: "clickhouse"
+    host: "localhost"
+    port: 8123
+    username: "default"
+    password: ""
+    client_cert: ""
+    client_key: ""
+    ca_cert: ""
+
+  - id: "userdb"
+    type: "sqlite"
+    db_connection: "/path/to/my.db"
+```
+file - queries.yaml
 ```yaml
 commands:
-  - name: user_count
-    database: userdb
-    query: "SELECT COUNT(*) FROM users;"
-    main_label: database_user_count
-    labels: ["database_name", "table_name"]
-    label_values: ["userdb", "users"]
+  - id: "active_user_count"
+    type: "clickhouse"
+    database_id: "active_user_count"
+    query: |
+      SELECT
+        country AS country_name,
+        COUNT() AS active_user_count
+      FROM users
+      WHERE is_active = 1
+      GROUP BY country
+    main_label: "active_user_count"
+    labels: ["country_name"]
+    value_column: "active_user_count"
+
+  - id: "list_all_users_sqlite"
+    type: "sqlite"
+    database_id: "userdb"
+    query: |
+      SELECT
+        id,
+        name
+      FROM users
+    main_label: "user_list"
+    labels: ["id", "name"]
+    value_column: "id"
+
 ```
 ## Functions mode
 

flexmetric-0.4.4/flexmetric/metric_process/database_connections.py

@@ -0,0 +1,55 @@
+import clickhouse_connect
+import sqlite3
+from flexmetric.logging_module.logger import get_logger
+import os
+
+logger = get_logger(__name__)
+
+logger.info("database logs")
+
+def create_clickhouse_client(db_conf):
+    id = db_conf.get('id')
+    host = db_conf.get('host', 'localhost')
+    port = db_conf.get('port', 9440)
+    username = db_conf.get('username', 'default')
+    password = db_conf.get('password', '')
+
+    client_cert = db_conf.get('client_cert')
+    client_cert_key = db_conf.get('client_cert_key')
+    ca_cert = db_conf.get('ca_cert')
+
+    secure = bool(client_cert and client_cert_key and ca_cert)
+
+    settings = {
+        'host': host,
+        'port': port,
+        'username': username,
+        'password': password,
+        'secure': secure,
+    }
+
+    if secure:
+        settings.update({
+            'client_cert': client_cert,
+            'client_cert_key': client_cert_key,
+            'secure_ca': ca_cert,
+        })
+
+    client = clickhouse_connect.get_client(**settings)
+    logger.info(f"Clickhouse connection '{id}' created")
+    return client
+
+
+def create_sqlite_client(db_conf: dict):
+    db_path = db_conf.get('db_connection')
+    db_name = db_conf.get('db_name', 'default_sqlite_db')
+
+    if not db_path or not os.path.isfile(db_path):
+        raise FileNotFoundError(f"SQLite database file not found at {db_path}")
+
+    try:
+        conn = sqlite3.connect(db_path)
+        logger.info(f"SQLite connection '{db_name}' created")
+        return conn
+    except Exception as e:
+        raise ConnectionError(f"Failed to create SQLite client: {e}")

flexmetric-0.4.4/flexmetric/metric_process/database_processing.py

@@ -0,0 +1,140 @@
+import yaml
+import re
+from flexmetric.metric_process.database_connections import create_clickhouse_client,create_sqlite_client
+from flexmetric.metric_process.queries_execution import execute_clickhouse_command,execute_sqlite_query
+from flexmetric.logging_module.logger import get_logger
+logger = get_logger(__name__)
+
+logger.info("query execution")
+
+
+def read_yaml_file(file_path):
+    with open(file_path, "r") as f:
+        return yaml.safe_load(f)
+
+
+def get_database_config(databases, db_name):
+    for db in databases:
+        if db["name"] == db_name:
+            return db
+    raise ValueError(
+        f"[ERROR] Database config for '{db_name}' not found in database.yaml."
+    )
+
+
+
+def is_safe_query(query):
+    cleaned_query = query.strip().lower()
+    return re.match(r"^\(*\s*select", cleaned_query) is not None
+
+
+def read_yaml_file(filepath: str) -> dict:
+    try:
+        with open(filepath, 'r') as f:
+            return yaml.safe_load(f)
+    except Exception as e:
+        raise Exception(f"Error reading {filepath}: {e}")
+
+
+def create_clients_from_config(config_file: str):
+    db_configs = read_yaml_file(config_file).get('databases', [])
+    clients = {}
+
+    for db_conf in db_configs:
+        db_id = db_conf.get('id')
+        logger.info(db_id)
+        db_type = db_conf.get('type')
+
+        if not db_id:
+            logger.info(f"Skipping unnamed database block: {db_conf}")
+            continue
+
+        try:
+            if db_type == 'clickhouse':
+                client = create_clickhouse_client(db_conf)
+                clients[db_id] = client
+                logger.info(clients)
+
+            elif db_type == 'sqlite':
+                client = create_sqlite_client(db_conf)
+                clients[db_id] = client
+                logger.info(clients)
+
+            else:
+                logger.info(f"Unsupported database type: {db_type}")
+
+        except Exception as e:
+            logger.error(f"Failed to create client '{db_id}': {e}")
+    return clients
+
+def process_and_get_value(cmd,rows_data,column_names):
+    labels = cmd.get('labels', [])
+    value_column = cmd.get('value_column', [])
+    label_values = cmd.get('label_values', [])
+    main_label = cmd.get('main_label', 'default_db_metric')
+    missing_columns = [col for col in labels + [value_column] if col not in column_names]
+    if missing_columns:
+        raise ValueError(f"Missing columns in result: {missing_columns}")
+    rows = []
+    for row in rows_data:
+        label_values = [str(row[column_names.index(col)]) for col in labels]
+        value = row[column_names.index(value_column)]
+
+        rows.append({
+            'label': label_values,
+            'value': value
+        })
+
+    return {
+        'result': rows,
+        'labels': labels,
+        'main_label': main_label
+    }
+
+def execute_commands(clients: dict, commands_file: str):
+    commands = read_yaml_file(commands_file).get('commands', [])
+    results = []
+    for cmd in commands:
+        print("CMD : ",cmd)
+        if cmd == None:
+            continue
+        cmd_id = cmd.get('id')
+        db_id = cmd.get('database_id')
+        db_type = cmd.get('type')
+        query = cmd.get('query')
+        if not cmd_id or not db_id or not db_type or not query:
+            logger.info(f"Missing required fields in command '{cmd_id}'")
+            continue
+
+        client = clients.get(db_id)
+        logger.info(clients)
+        logger.info(db_id)
+        logger.info(client)
+        if not client:
+            logger.info(f"No client for database_id '{db_id}' in command '{cmd_id}'")
+            continue
+        try:
+            if db_type == 'clickhouse':
+                logger.info("In clickhouse")
+                response,column_names = execute_clickhouse_command(client,query)
+                result = process_and_get_value(cmd,response,column_names)
+                results.append(result)
+            elif db_type == 'sqlite':
+                response ,column_names = execute_sqlite_query(client,query)
+                result = process_and_get_value(cmd,response,column_names)
+                results.append(result)
+            else:
+                logger.info(f"Unknown type '{db_type}' in command '{cmd_id}'")
+                continue
+                
+        except Exception as e:
+            logger.error(f"Command '{cmd_id}' failed: {e}")
+    return results
+
+
+def process_database_queries(queries_file, databases_file):
+    try:
+        client_configs = create_clients_from_config(databases_file)
+        return execute_commands(clients=client_configs,commands_file=queries_file)
+    except Exception as ex:
+        logger.error(f"Exception : {ex}")

{flexmetric-0.4.2 → flexmetric-0.4.4}/flexmetric/metric_process/prometheus_agent.py

@@ -256,4 +256,4 @@ def main():
         run_flask(args.host, args.port)
 # # args = arguments()
 # # measure(args)
-# main()
+main()

flexmetric-0.4.4/flexmetric/metric_process/queries_execution.py

@@ -0,0 +1,24 @@
+from flexmetric.logging_module.logger import get_logger
+logger = get_logger(__name__)
+
+logger.info("query execution")
+
+def execute_clickhouse_command(client, command: str):
+    try:
+        result = client.query(command)
+        row_list = result.result_rows
+        column_names = result.column_names
+        return row_list,column_names
+    except Exception as e:
+        logger.error(f"Error executing command: {e}")
+        return None
+def execute_sqlite_query(conn, query):
+    try:
+        cursor = conn.cursor()
+        cursor.execute(query)
+        rows = cursor.fetchall()
+        column_names = [desc[0] for desc in cursor.description]
+        return rows, column_names
+    except Exception as ex:
+        logger.error(f"Exception during SQLite query: {ex}")
+        return [], []

{flexmetric-0.4.2 → flexmetric-0.4.4}/flexmetric.egg-info/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: flexmetric
-Version: 0.4.2
-Summary: A secure flexible Prometheus exporter for commands, databases, functions, and scripts.
+Version: 0.4.4
+Summary: A secure flexible Prometheus exporter for commands, databases, functions.
 Home-page: https://github.com/nikhillingadhal1999/flexmetric
 Author: Nikhil Lingadhal
 License: MIT
@@ -20,6 +20,7 @@ Requires-Dist: setuptools
 Requires-Dist: wheel
 Requires-Dist: twine
 Requires-Dist: flask
+Requires-Dist: clickhouse-connect
 Dynamic: author
 Dynamic: classifier
 Dynamic: description
@@ -188,21 +189,52 @@ Filesystem      Size  Used Avail Use% Mounted on
 | `timeout_seconds` | Maximum time (in seconds) to wait for the command to complete. If it exceeds this time, the command is aborted.             |
 
 ## Database mode
-
+file - database.yaml
 ```yaml
 databases:
-  - name: mydb
-    db_type: sqlite
-    db_connection: /path/to/my.db
-````
+  - id: "active_user_count"
+    type: "clickhouse"
+    host: "localhost"
+    port: 8123
+    username: "default"
+    password: ""
+    client_cert: ""
+    client_key: ""
+    ca_cert: ""
+
+  - id: "userdb"
+    type: "sqlite"
+    db_connection: "/path/to/my.db"
+```
+file - queries.yaml
 ```yaml
 commands:
-  - name: user_count
-    database: userdb
-    query: "SELECT COUNT(*) FROM users;"
-    main_label: database_user_count
-    labels: ["database_name", "table_name"]
-    label_values: ["userdb", "users"]
+  - id: "active_user_count"
+    type: "clickhouse"
+    database_id: "active_user_count"
+    query: |
+      SELECT
+        country AS country_name,
+        COUNT() AS active_user_count
+      FROM users
+      WHERE is_active = 1
+      GROUP BY country
+    main_label: "active_user_count"
+    labels: ["country_name"]
+    value_column: "active_user_count"
+
+  - id: "list_all_users_sqlite"
+    type: "sqlite"
+    database_id: "userdb"
+    query: |
+      SELECT
+        id,
+        name
+      FROM users
+    main_label: "user_list"
+    labels: ["id", "name"]
+    value_column: "id"
+
 ```
 ## Functions mode
 

{flexmetric-0.4.2 → flexmetric-0.4.4}/flexmetric.egg-info/SOURCES.txt

@@ -14,8 +14,10 @@ flexmetric/file_recognition/exec_file.py
 flexmetric/logging_module/__init__.py
 flexmetric/logging_module/logger.py
 flexmetric/metric_process/__init__.py
+flexmetric/metric_process/database_connections.py
 flexmetric/metric_process/database_processing.py
 flexmetric/metric_process/expiring_queue.py
 flexmetric/metric_process/process_commands.py
 flexmetric/metric_process/prometheus_agent.py
+flexmetric/metric_process/queries_execution.py
 flexmetric/metric_process/views.py

{flexmetric-0.4.2 → flexmetric-0.4.4}/flexmetric.egg-info/requires.txt

@@ -5,3 +5,4 @@ setuptools
 wheel
 twine
 flask
+clickhouse-connect

{flexmetric-0.4.2 → flexmetric-0.4.4}/setup.py

@@ -2,9 +2,9 @@ from setuptools import setup, find_packages
 
 setup(
     name="flexmetric",
-    version="0.4.2",
+    version="0.4.4",
     author="Nikhil Lingadhal",
-    description="A secure flexible Prometheus exporter for commands, databases, functions, and scripts.",
+    description="A secure flexible Prometheus exporter for commands, databases, functions.",
     long_description=open("README.md").read(),
     long_description_content_type="text/markdown",
     url="https://github.com/nikhillingadhal1999/flexmetric",
@@ -22,7 +22,8 @@ setup(
         "setuptools",
         "wheel",
         "twine",
-        "flask"
+        "flask",
+        "clickhouse-connect"
     ],
     entry_points={
         "console_scripts": [

flexmetric-0.4.2/flexmetric/metric_process/database_processing.py

@@ -1,90 +0,0 @@
-import yaml
-import sqlite3
-import re
-
-
-def read_yaml_file(file_path):
-    with open(file_path, "r") as f:
-        return yaml.safe_load(f)
-
-
-def get_database_config(databases, db_name):
-    for db in databases:
-        if db["name"] == db_name:
-            return db
-    raise ValueError(
-        f"[ERROR] Database config for '{db_name}' not found in database.yaml."
-    )
-
-
-def execute_sqlite_query(db_path, query):
-    try:
-        conn = sqlite3.connect(db_path)
-        cursor = conn.cursor()
-        cursor.execute(query)
-        result = cursor.fetchone()
-        conn.close()
-        return float(result[0]) if result and result[0] is not None else None
-    except Exception as e:
-        print(f"[ERROR] SQLite query failed on {db_path}: {e}")
-        return None
-
-
-def is_safe_query(query):
-    # Remove leading spaces and brackets
-    cleaned_query = query.strip().lower()
-    # Match only queries that start with "select"
-    return re.match(r"^\(*\s*select", cleaned_query) is not None
-
-
-def process_database_queries(queries_file, databases_file):
-    # Get queries from queries file
-    queries_config = read_yaml_file(queries_file)
-    # Get database from database file
-    databases_config = read_yaml_file(databases_file)
-
-    commands = queries_config.get("commands", [])
-    databases = databases_config.get("databases", [])
-
-    all_results = []
-
-    for cmd in commands:
-        try:
-            db_conf = get_database_config(databases, cmd["database"])
-
-            if db_conf["db_type"] != "sqlite":
-                print(
-                    f"[WARN] Unsupported database type: {db_conf['db_type']} in command {cmd['name']}"
-                )
-                continue
-
-            db_path = db_conf["db_connection"]
-            query = cmd["query"]
-            labels = cmd.get('labels', [])
-            label_values = cmd.get('label_values', [])
-            main_label = cmd.get('main_label', 'default_db_metric')
-
-            # check if query is safe
-            if is_safe_query(query):
-                value = execute_sqlite_query(db_path, query)
-            else:
-                print(f"[WARN] Unsupported query type: {query}")
-                return None
-
-            if not isinstance(label_values, list):
-                label_values = [label_values]
-
-            result = {
-                'result': [{
-                    'label': label_values,
-                    'value': value
-                }],
-                'labels': labels,
-                'main_label': main_label
-            }
-            all_results.append(result)
-        except Exception as e:
-            print(
-                f"[ERROR] Processing command '{cmd.get('name', 'unknown')}' failed: {e}"
-            )
-    return all_results