lesscode-flask 0.0.27__py3-none-any.whl

redash/query_runner/mysql.py

@@ -0,0 +1,309 @@
+import logging
+import os
+import threading
+import traceback
+
+from redash.query_runner import (
+    TYPE_DATE,
+    TYPE_DATETIME,
+    TYPE_FLOAT,
+    TYPE_INTEGER,
+    TYPE_STRING,
+    BaseSQLQueryRunner,
+    InterruptException,
+    JobTimeoutException,
+    register,
+)
+from redash.settings import parse_boolean
+
+try:
+    import pymysql
+    enabled = True
+except ImportError:
+    logging.error(traceback.format_exc())
+    logging.error(f"pymysql is not exist,run:pip install pymysql==1.0.1")
+    enabled = False
+
+logger = logging.getLogger(__name__)
+types_map = {
+    0: TYPE_FLOAT,
+    1: TYPE_INTEGER,
+    2: TYPE_INTEGER,
+    3: TYPE_INTEGER,
+    4: TYPE_FLOAT,
+    5: TYPE_FLOAT,
+    7: TYPE_DATETIME,
+    8: TYPE_INTEGER,
+    9: TYPE_INTEGER,
+    10: TYPE_DATE,
+    12: TYPE_DATETIME,
+    15: TYPE_STRING,
+    16: TYPE_INTEGER,
+    246: TYPE_FLOAT,
+    253: TYPE_STRING,
+    254: TYPE_STRING,
+}
+
+
+class Result:
+    def __init__(self):
+        pass
+
+
+class Mysql(BaseSQLQueryRunner):
+    noop_query = "SELECT 1"
+
+    @classmethod
+    def configuration_schema(cls):
+        show_ssl_settings = parse_boolean(os.environ.get("MYSQL_SHOW_SSL_SETTINGS", "true"))
+
+        schema = {
+            "type": "object",
+            "properties": {
+                "host": {"type": "string", "default": "127.0.0.1"},
+                "user": {"type": "string"},
+                "passwd": {"type": "string", "title": "Password"},
+                "db": {"type": "string", "title": "Database name"},
+                "port": {"type": "number", "default": 3306},
+                "connect_timeout": {"type": "number", "default": 60, "title": "Connection Timeout"},
+                "charset": {"type": "string", "default": "utf8"},
+                "use_unicode": {"type": "boolean", "default": True},
+                "autocommit": {"type": "boolean", "default": False},
+            },
+            "order": [
+                "host",
+                "port",
+                "user",
+                "passwd",
+                "db",
+                "connect_timeout",
+                "charset",
+                "use_unicode",
+                "autocommit",
+            ],
+            "required": ["db"],
+            "secret": ["passwd"],
+        }
+
+        if show_ssl_settings:
+            schema["properties"].update(
+                {
+                    "ssl_mode": {
+                        "type": "string",
+                        "title": "SSL Mode",
+                        "default": "preferred",
+                        "extendedEnum": [
+                            {"value": "disabled", "name": "Disabled"},
+                            {"value": "preferred", "name": "Preferred"},
+                            {"value": "required", "name": "Required"},
+                            {"value": "verify-ca", "name": "Verify CA"},
+                            {"value": "verify-identity", "name": "Verify Identity"},
+                        ],
+                    },
+                    "use_ssl": {"type": "boolean", "title": "Use SSL"},
+                    "ssl_cacert": {
+                        "type": "string",
+                        "title": "Path to CA certificate file to verify peer against (SSL)",
+                    },
+                    "ssl_cert": {
+                        "type": "string",
+                        "title": "Path to client certificate file (SSL)",
+                    },
+                    "ssl_key": {
+                        "type": "string",
+                        "title": "Path to private key file (SSL)",
+                    },
+                }
+            )
+
+        return schema
+
+    @classmethod
+    def name(cls):
+        return "MySQL"
+
+    @classmethod
+    def enabled(cls):
+        return enabled
+
+    def _connection(self):
+        params = dict(
+            host=self.configuration.get("host", ""),
+            user=self.configuration.get("user", ""),
+            passwd=self.configuration.get("passwd", ""),
+            db=self.configuration["db"],
+            port=self.configuration.get("port", 3306),
+            charset=self.configuration.get("charset", "utf8"),
+            use_unicode=self.configuration.get("use_unicode", True),
+            connect_timeout=self.configuration.get("connect_timeout", 60),
+            autocommit=self.configuration.get("autocommit", True),
+        )
+
+        ssl_options = self._get_ssl_parameters()
+
+        if ssl_options:
+            params["ssl"] = ssl_options
+
+        connection = pymysql.connect(**params)
+
+        return connection
+
+    def _get_tables(self, schema):
+        query = """
+        SELECT col.table_schema as table_schema,
+               col.table_name as table_name,
+               col.column_name as column_name
+        FROM `information_schema`.`columns` col
+        WHERE col.table_schema NOT IN ('information_schema', 'performance_schema', 'mysql', 'sys');
+        """
+
+        results, error = self.run_query(query, None)
+
+        if error is not None:
+            self._handle_run_query_error(error)
+
+        for row in results["rows"]:
+            if row["table_schema"] != self.configuration["db"]:
+                table_name = "{}.{}".format(row["table_schema"], row["table_name"])
+            else:
+                table_name = row["table_name"]
+
+            if table_name not in schema:
+                schema[table_name] = {"name": table_name, "columns": []}
+
+            schema[table_name]["columns"].append(row["column_name"])
+
+        return list(schema.values())
+
+    def run_query(self, query, user):
+        ev = threading.Event()
+        thread_id = ""
+        r = Result()
+        t = None
+
+        try:
+            connection = self._connection()
+            self._run_query(query, user, connection, r, ev)
+
+            # thread_id = connection.thread_id()
+            # t = threading.Thread(target=self._run_query, args=(query, user, connection, r, ev))
+            # t.start()
+            # while not ev.wait(1):
+            #     pass
+        except (KeyboardInterrupt, InterruptException, JobTimeoutException):
+            self._cancel(thread_id)
+            #  t.join()
+            raise
+
+        return r.data, r.error
+
+    def _run_query(self, query, user, connection, r, ev):
+        try:
+            cursor = connection.cursor()
+            logger.debug("MySQL running query: %s", query)
+            cursor.execute(query)
+
+            data = cursor.fetchall()
+            desc = cursor.description
+
+            while cursor.nextset():
+                if cursor.description is not None:
+                    data = cursor.fetchall()
+                    desc = cursor.description
+
+            # TODO - very similar to pg.py
+            if desc is not None:
+                columns = self.fetch_columns([(i[0], types_map.get(i[1], None)) for i in desc])
+                rows = [dict(zip((column["name"] for column in columns), row)) for row in data]
+
+                data = {"columns": columns, "rows": rows}
+                r.data = data
+                r.error = None
+            else:
+                r.data = None
+                r.error = "No data was returned."
+
+            cursor.close()
+        except pymysql.Error as e:
+            if cursor:
+                cursor.close()
+            r.data = None
+            r.error = e.args[1]
+        finally:
+            ev.set()
+            if connection:
+                connection.close()
+
+    def _get_ssl_parameters(self):
+        if not self.configuration.get("use_ssl"):
+            return None
+
+        ssl_params = {}
+
+        if self.configuration.get("use_ssl"):
+            config_map = {"ssl_mode": "preferred", "ssl_cacert": "ca", "ssl_cert": "cert", "ssl_key": "key"}
+            for key, cfg in config_map.items():
+                val = self.configuration.get(key)
+                if val:
+                    ssl_params[cfg] = val
+
+        return ssl_params
+
+    def _cancel(self, thread_id):
+        connection = None
+        cursor = None
+        error = None
+
+        try:
+            connection = self._connection()
+            cursor = connection.cursor()
+            query = "KILL %d" % (thread_id)
+            logging.debug(query)
+            cursor.execute(query)
+        except pymysql.Error as e:
+            if cursor:
+                cursor.close()
+            error = e.args[1]
+        finally:
+            if connection:
+                connection.close()
+
+        return error
+
+
+class RDSMySQL(Mysql):
+    @classmethod
+    def name(cls):
+        return "MySQL (Amazon RDS)"
+
+    @classmethod
+    def type(cls):
+        return "rds_mysql"
+
+    @classmethod
+    def configuration_schema(cls):
+        return {
+            "type": "object",
+            "properties": {
+                "host": {"type": "string"},
+                "user": {"type": "string"},
+                "passwd": {"type": "string", "title": "Password"},
+                "db": {"type": "string", "title": "Database name"},
+                "port": {"type": "number", "default": 3306},
+                "use_ssl": {"type": "boolean", "title": "Use SSL"},
+            },
+            "order": ["host", "port", "user", "passwd", "db"],
+            "required": ["db", "user", "passwd", "host"],
+            "secret": ["passwd"],
+        }
+
+    def _get_ssl_parameters(self):
+        if self.configuration.get("use_ssl"):
+            ca_path = os.path.join(os.path.dirname(__file__), "files/rds-combined-ca-bundle.pem")
+            return {"ca": ca_path}
+
+        return None
+
+
+register(Mysql)
+# register(RDSMySQL)

redash/query_runner/pg.py

@@ -0,0 +1,284 @@
+import logging
+import os
+import select
+import traceback
+from base64 import b64decode
+from tempfile import NamedTemporaryFile
+
+try:
+    import pg8000
+
+    enabled = True
+except ImportError:
+    enabled = False
+    logging.error(f"pg8000 is not exist,run:pip install pg8000==1.31.2")
+
+from redash.query_runner import (
+    TYPE_BOOLEAN,
+    TYPE_DATE,
+    TYPE_DATETIME,
+    TYPE_FLOAT,
+    TYPE_INTEGER,
+    TYPE_STRING,
+    BaseSQLQueryRunner,
+    InterruptException,
+    JobTimeoutException,
+    register,
+)
+
+logger = logging.getLogger(__name__)
+
+types_map = {
+    20: TYPE_INTEGER,
+    21: TYPE_INTEGER,
+    23: TYPE_INTEGER,
+    700: TYPE_FLOAT,
+    1700: TYPE_FLOAT,
+    701: TYPE_FLOAT,
+    16: TYPE_BOOLEAN,
+    1082: TYPE_DATE,
+    1182: TYPE_DATE,
+    1114: TYPE_DATETIME,
+    1184: TYPE_DATETIME,
+    1115: TYPE_DATETIME,
+    1185: TYPE_DATETIME,
+    1014: TYPE_STRING,
+    1015: TYPE_STRING,
+    1008: TYPE_STRING,
+    1009: TYPE_STRING,
+    2951: TYPE_STRING,
+    1043: TYPE_STRING,
+    1002: TYPE_STRING,
+    1003: TYPE_STRING,
+}
+
+
+def full_table_name(schema, name):
+    if "." in name:
+        name = '"{}"'.format(name)
+
+    return "{}.{}".format(schema, name)
+
+
+def build_schema(query_result, schema):
+    # By default we omit the public schema name from the table name. But there are
+    # edge cases, where this might cause conflicts. For example:
+    # * We have a schema named "main" with table "users".
+    # * We have a table named "main.users" in the public schema.
+    # (while this feels unlikely, this actually happened)
+    # In this case if we omit the schema name for the public table, we will have
+    # a conflict.
+    table_names = set(
+        map(
+            lambda r: full_table_name(r["table_schema"], r["table_name"]),
+            query_result["rows"],
+        )
+    )
+
+    for row in query_result["rows"]:
+        if row["table_schema"] != "public":
+            table_name = full_table_name(row["table_schema"], row["table_name"])
+        else:
+            if row["table_name"] in table_names:
+                table_name = full_table_name(row["table_schema"], row["table_name"])
+            else:
+                table_name = row["table_name"]
+
+        if table_name not in schema:
+            schema[table_name] = {"name": table_name, "columns": []}
+
+        column = row["column_name"]
+        if row.get("data_type") is not None:
+            column = {"name": row["column_name"], "type": row["data_type"]}
+
+        schema[table_name]["columns"].append(column)
+
+
+def _create_cert_file(configuration, key, ssl_config):
+    file_key = key + "File"
+    if file_key in configuration:
+        with NamedTemporaryFile(mode="w", delete=False) as cert_file:
+            cert_bytes = b64decode(configuration[file_key])
+            cert_file.write(cert_bytes.decode("utf-8"))
+
+        ssl_config[key] = cert_file.name
+
+
+def _cleanup_ssl_certs(ssl_config):
+    for k, v in ssl_config.items():
+        if k != "sslmode":
+            os.remove(v)
+
+
+def _get_ssl_config(configuration):
+    ssl_config = {"sslmode": configuration.get("sslmode", "prefer")}
+
+    _create_cert_file(configuration, "sslrootcert", ssl_config)
+    _create_cert_file(configuration, "sslcert", ssl_config)
+    _create_cert_file(configuration, "sslkey", ssl_config)
+
+    return ssl_config
+
+
+class PostgreSQL(BaseSQLQueryRunner):
+    noop_query = "SELECT 1"
+
+    @classmethod
+    def configuration_schema(cls):
+        return {
+            "type": "object",
+            "properties": {
+                "user": {"type": "string"},
+                "password": {"type": "string"},
+                "host": {"type": "string", "default": "127.0.0.1"},
+                "port": {"type": "number", "default": 5432},
+                "dbname": {"type": "string", "title": "Database Name"},
+                "sslmode": {
+                    "type": "string",
+                    "title": "SSL Mode",
+                    "default": "prefer",
+                    "extendedEnum": [
+                        {"value": "disable", "name": "Disable"},
+                        {"value": "allow", "name": "Allow"},
+                        {"value": "prefer", "name": "Prefer"},
+                        {"value": "require", "name": "Require"},
+                        {"value": "verify-ca", "name": "Verify CA"},
+                        {"value": "verify-full", "name": "Verify Full"},
+                    ],
+                },
+                "sslrootcertFile": {"type": "string", "title": "SSL Root Certificate"},
+                "sslcertFile": {"type": "string", "title": "SSL Client Certificate"},
+                "sslkeyFile": {"type": "string", "title": "SSL Client Key"},
+            },
+            "order": ["host", "port", "user", "password"],
+            "required": ["dbname"],
+            "secret": ["password", "sslrootcertFile", "sslcertFile", "sslkeyFile"],
+            "extra_options": [
+                "sslmode",
+                "sslrootcertFile",
+                "sslcertFile",
+                "sslkeyFile",
+            ],
+        }
+
+    @classmethod
+    def type(cls):
+        return "pg"
+
+    @classmethod
+    def enabled(cls):
+        return enabled
+
+    # @classmethod
+    # def custom_json_encoder(cls, dec, o):
+    #     if isinstance(o, Range):
+    #         # From: https://github.com/psycopg/pg8000/pull/779
+    #         if o._bounds is None:
+    #             return ""
+    #
+    #         items = [o._bounds[0], str(o._lower), ", ", str(o._upper), o._bounds[1]]
+    #
+    #         return "".join(items)
+    #     return None
+
+    def _get_definitions(self, schema, query):
+        results, error = self.run_query(query, None)
+
+        if error is not None:
+            self._handle_run_query_error(error)
+
+        build_schema(results, schema)
+
+    def _get_tables(self, schema):
+        """
+        relkind constants per https://www.postgresql.org/docs/10/static/catalog-pg-class.html
+        r = regular table
+        v = view
+        m = materialized view
+        f = foreign table
+        p = partitioned table (new in 10)
+        ---
+        i = index
+        S = sequence
+        t = TOAST table
+        c = composite type
+        """
+
+        query = """
+        SELECT s.nspname as table_schema,
+               c.relname as table_name,
+               a.attname as column_name,
+               null as data_type
+        FROM pg_class c
+        JOIN pg_namespace s
+        ON c.relnamespace = s.oid
+        AND s.nspname NOT IN ('pg_catalog', 'information_schema')
+        JOIN pg_attribute a
+        ON a.attrelid = c.oid
+        AND a.attnum > 0
+        AND NOT a.attisdropped
+        WHERE c.relkind IN ('m', 'f', 'p')
+        AND has_table_privilege(s.nspname || '.' || c.relname, 'select')
+        AND has_schema_privilege(s.nspname, 'usage')
+
+        UNION
+
+        SELECT table_schema,
+               table_name,
+               column_name,
+               data_type
+        FROM information_schema.columns
+        WHERE table_schema NOT IN ('pg_catalog', 'information_schema')
+        """
+
+        self._get_definitions(schema, query)
+
+        return list(schema.values())
+
+    def _get_connection(self):
+        # self.ssl_config = _get_ssl_config(self.configuration)
+        params = dict(
+            host=self.configuration.get("host", ""),
+            user=self.configuration.get("user", ""),
+            password=self.configuration.get("passwd", ""),
+            database=self.configuration["db"],
+            port=self.configuration.get("port", 54321),
+            # **self.ssl_config,
+        )
+        connection = pg8000.connect(**params)
+        return connection
+
+    def run_query(self, query, user):
+        connection = self._get_connection()
+        cursor = connection.cursor()
+
+        try:
+            cursor.execute(query)
+            #    _wait(connection)
+
+            if cursor.description is not None:
+                columns = self.fetch_columns([(i[0], types_map.get(i[1], None)) for i in cursor.description])
+                rows = [dict(zip((column["name"] for column in columns), row)) for row in cursor]
+
+                data = {"columns": columns, "rows": rows}
+                error = None
+            else:
+                error = "Query completed but it returned no data."
+                data = None
+        except (select.error, OSError):
+            error = "Query interrupted. Please retry."
+            data = None
+        except pg8000.DatabaseError as e:
+            error = str(e)
+            data = None
+        except (KeyboardInterrupt, InterruptException, JobTimeoutException):
+            connection.cancel()
+            raise
+        finally:
+            connection.close()
+        #    _cleanup_ssl_certs(self.ssl_config)
+
+        return data, error
+
+
+register(PostgreSQL)

redash/settings/__init__.py

@@ -0,0 +1,90 @@
+import os
+
+from redash.settings.helpers import parse_boolean, array_from_string
+from funcy import distinct, remove
+
+# Whether api calls using the json query runner will block private addresses
+ENFORCE_PRIVATE_ADDRESS_BLOCK = parse_boolean(os.environ.get("REDASH_ENFORCE_PRIVATE_IP_BLOCK", "true"))
+# requests
+REQUESTS_ALLOW_REDIRECTS = parse_boolean(os.environ.get("REDASH_REQUESTS_ALLOW_REDIRECTS", "false"))
+
+
+# Query Runners
+default_query_runners = [
+    # "redash.query_runner.athena",
+    # "redash.query_runner.big_query",
+    # "redash.query_runner.google_spreadsheets",
+    # "redash.query_runner.graphite",
+    # "redash.query_runner.mongodb",
+    # "redash.query_runner.couchbase",
+    "redash.query_runner.mysql",
+    "redash.query_runner.pg",
+    "redash.query_runner.kingbase",
+    # "redash.query_runner.url",
+    # "redash.query_runner.influx_db",
+    # "redash.query_runner.influx_db_v2",
+    # "redash.query_runner.elasticsearch",
+    # "redash.query_runner.elasticsearch2",
+    # "redash.query_runner.amazon_elasticsearch",
+    # "redash.query_runner.trino",
+    # "redash.query_runner.presto",
+    # "redash.query_runner.pinot",
+    # "redash.query_runner.databricks",
+    # "redash.query_runner.hive_ds",
+    # "redash.query_runner.impala_ds",
+    # "redash.query_runner.vertica",
+    "redash.query_runner.clickhouse",
+    # "redash.query_runner.tinybird",
+    # "redash.query_runner.yandex_metrica",
+    # "redash.query_runner.yandex_disk",
+    # "redash.query_runner.rockset",
+    # "redash.query_runner.treasuredata",
+    # "redash.query_runner.sqlite",
+    # "redash.query_runner.mssql",
+    # "redash.query_runner.mssql_odbc",
+    # "redash.query_runner.memsql_ds",
+    # "redash.query_runner.jql",
+    # "redash.query_runner.google_analytics",
+    # "redash.query_runner.axibase_tsd",
+    # "redash.query_runner.salesforce",
+    # "redash.query_runner.query_results",
+    # "redash.query_runner.prometheus",
+    # "redash.query_runner.db2",
+    # "redash.query_runner.druid",
+    # "redash.query_runner.kylin",
+    # "redash.query_runner.drill",
+    # "redash.query_runner.uptycs",
+    # "redash.query_runner.snowflake",
+    # "redash.query_runner.phoenix",
+    # "redash.query_runner.json_ds",
+    # "redash.query_runner.cass",
+    # "redash.query_runner.dgraph",
+    # "redash.query_runner.azure_kusto",
+    # "redash.query_runner.exasol",
+    # "redash.query_runner.cloudwatch",
+    # "redash.query_runner.cloudwatch_insights",
+    # "redash.query_runner.corporate_memory",
+    # "redash.query_runner.sparql_endpoint",
+    # "redash.query_runner.excel",
+    # "redash.query_runner.csv",
+    # "redash.query_runner.databend",
+    # "redash.query_runner.nz",
+    # "redash.query_runner.arango",
+    # "redash.query_runner.google_analytics4",
+    # "redash.query_runner.google_search_console",
+    # "redash.query_runner.ignite",
+    # "redash.query_runner.oracle",
+    # "redash.query_runner.e6data",
+    # "redash.query_runner.risingwave",
+]
+
+enabled_query_runners = array_from_string(
+    os.environ.get("REDASH_ENABLED_QUERY_RUNNERS", ",".join(default_query_runners))
+)
+additional_query_runners = array_from_string(os.environ.get("REDASH_ADDITIONAL_QUERY_RUNNERS", ""))
+disabled_query_runners = array_from_string(os.environ.get("REDASH_DISABLED_QUERY_RUNNERS", ""))
+
+QUERY_RUNNERS = remove(
+    set(disabled_query_runners),
+    distinct(enabled_query_runners + additional_query_runners),
+)