sql-testing-tools 0.1.0__py3-none-any.whl

sql_testing_tools/BaseAccess.py

@@ -0,0 +1,170 @@
+import sqlite3
+import os
+from importlib import resources
+
+
+__db=""
+
+def setDBName(name: str):
+    global __db
+    print("Setting DB to: " + name)
+    with resources.path('sql_testing_tools', name) as db_path:
+        __db = str(db_path)
+        print(__db)
+
+
+setDBName("bayern.db")
+
+def run(sql: str):
+    con = sqlite3.connect(__db)
+    cur = con.cursor()
+    cur.execute(sql)
+    con.commit()
+    return cur
+
+def runFromFile(path: str):
+    sql = getSQLFromFile(path)
+    if not sql:
+        raise Exception("\nSQL-Datei ist leer. Aufgabe wurde noch nicht bearbeitet.")
+
+
+    if sql.lower().find("drop") != -1:
+        raise Exception("Guter Versuch, aber die Datenbank wird bei jedem Upload zurückgesetzt ;)")
+
+    try:
+        res = run(sql)
+    except Exception as e:
+        raise Exception(f"\n\nSyntax-Fehler in der SQL-Abfrage:\n{str(e)}")
+
+    headers = [h[0] for h in res.description] if res.description else []
+    rows = res.fetchall() if res else []
+
+    return headers, rows
+
+
+def runAndGetStringTable_fromFile(path: str, count: int = 5, maxLineLength: int = 85):
+    try:
+        headers, rows = runFromFile(path)
+        resultCount = len(rows)
+        rows = rows[:count]
+        s = ""
+
+        matrix = [] #[[]*(len(rows)+1)]*len(headers)
+
+        for col in range(len(headers)):
+            matrix.append([])
+            matrix[-1].append(headers[col])
+
+        if rows is not None:
+            for col in range(len(matrix)):
+                for row in range(0, len(rows)):
+                    if len(rows[row]) > col:
+                        matrix[col].append(rows[row][col])
+
+            for col in range(len(matrix)):
+                maxLength = max([len(str(x)) for x in matrix[col]])
+
+                for row in range(0, len(matrix[col])):
+                    matrix[col][row] = str(matrix[col][row]).ljust(maxLength)
+
+            normalizedRows = []
+            spacing = "  "
+            spacingLength = len(spacing) * (len(matrix[0]) - 1)
+
+            lineLength = sum([len(x[0]) for x in matrix]) + spacingLength
+            
+            
+
+            if lineLength > maxLineLength:
+                valLength = lineLength - spacingLength
+                maxColLength = int(maxLineLength/len(matrix))
+
+                for col in range(len(matrix)):
+                    for row in range(0, len(matrix[col])):
+                        cutContent = len(matrix[col][row].strip()) > maxColLength
+                        if cutContent:
+                            matrix[col][row] = matrix[col][row][:maxColLength-2] + ".."
+                        else:
+                            matrix[col][row] = matrix[col][row][:maxColLength]
+
+
+            for row in range(min(count+1, len(matrix[0]))):
+                normalizedRows.append(spacing.join([str(matrix[col][row]) for col in range(len(matrix))]))
+            normalizedRows.insert(1, len(normalizedRows[0])*"-")
+
+            s = "\n".join(normalizedRows[:count+2])
+            s += "\n" + normalizedRows[1]
+
+            if (resultCount > count):
+                s += "\n... " + str(resultCount - count) + " weitere Zeilen"
+            else:
+                s += "\n... keine weiteren Zeilen"
+
+        #longestLineInS = 85 # max([len(x.replace("\t", "    ")) for x in s.split("\n")])
+
+        #s = "-" * longestLineInS + "\n" + s + "\n" + "-" * longestLineInS
+        s = "\n\nDiese Meldung sagt nichts über die Korrektheit der Abgabe aus!\nDie ersten " + str(
+            count) + " Zeilen des Ergebnisses der SQL-Abfrage:\n\n" + s
+
+        return s.replace("None", "NULL")
+    except Exception as ex:
+        raise ex
+
+def getSQLFromFile(path: str):
+    try:
+        with open(path, "r") as f:
+            s = f.read()
+            if not s:
+                raise Exception("\nSQL-Datei ist leer. Aufgabe wurde noch nicht bearbeitet.")
+            return s
+    except FileNotFoundError:
+        raise Exception(f"\nSQL-Datei nicht gefunden! Überprüfe, dass der Name korrekt ist ({path.split('/')[-1]}) und die Datei nicht gelöscht oder in einen Unterordner verschoben wurde.")
+
+
+
+def getWorkingDir():
+    return os.getcwd()
+
+
+def getWorkingDirFiles():
+    return os.listdir()
+
+def mapDatabaseTypes(t):
+    # Map SQLite types to Python types
+    type_mapping = {
+        "INTEGER": "int",
+        "TEXT": "str",
+        "VARCHAR": "str",
+        "REAL": "float",
+        "BLOB": "bytes",
+        "NUMERIC": "float",
+        "DOUBLE": "float"
+    }
+    t = t.upper()
+    t = type_mapping.get(t, t)
+
+    if "INT" in t:
+        t = "int"
+    elif "VARCHAR" in t:
+        t = "str"
+    elif "CHAR" in t:
+        t = "str"
+
+    return t
+
+
+
+
+def getTableDict():
+    res = run("SELECT name FROM sqlite_master WHERE type='table';")
+    tables = res.fetchall()
+
+    table_dict = {}
+    for table in tables:
+        res = run(f"PRAGMA table_info({table[0]})")
+        r = res.fetchall()
+        if table[0] == "sqlite_sequence":
+            continue
+        table_dict[table[0]] = [[c[1], mapDatabaseTypes(c[2])] for c in r]
+
+    return table_dict

sql_testing_tools/Helper.py

@@ -0,0 +1,410 @@
+import os
+from . import BaseAccess as Ba
+
+
+try:
+    import sqlparse
+except ImportError:
+    print("Trying to Install required module: sqlparse\n")
+    os.system('pip install sqlparse')
+    import sqlparse
+
+
+try:
+    import requests
+except ImportError:
+    print("Trying to Install required module: requests\n")
+    os.system('pip install requests')
+    import requests
+
+from sqlparse.sql import Identifier, IdentifierList, Where, Comparison, Function, Parenthesis
+from sqlparse.tokens import Keyword, DML, Name, Wildcard
+
+
+def normalizeSQLQuery(query, baseDict):
+    try:
+        query = query.replace("\"", "'")
+        parsed = sqlparse.parse(query)[0]
+        parsed.tokens = [token for token in parsed.tokens if not token.is_whitespace]
+    except Exception as e:
+        raise Exception(f"\nSyntax-Fehler in der SQL-Abfrage.")
+
+    formatted_query = []
+    alias_map = {}
+
+    def process_identifier(identifier, alias_map, baseDict: dict):
+        if isinstance(identifier, Identifier):
+            if identifier.get_real_name() and identifier.get_parent_name() and identifier.get_parent_name().lower() in alias_map.keys():
+                return f"{alias_map[identifier.get_parent_name().lower()].lower()}.{identifier.get_real_name().lower()}"
+            elif identifier.get_real_name():
+                tables = findTableForColumn(baseDict, identifier.get_real_name(), alias_map.keys())
+                if len(tables) == 1:
+                    alias_map[tables[0].lower()] = tables[0]
+                    return f"{tables[0].lower()}.{identifier.get_real_name().lower()}"
+                else:
+                    return f"{identifier.get_real_name().lower()}"
+        return str(identifier)
+
+    def process_select(select, alias_map, insideFunction=False):
+        select_tokens = []
+        for token in select.tokens:
+            if isinstance(token, IdentifierList):
+                for identifier in token.get_identifiers():
+                    select_tokens.append(f"{process_identifier(identifier, alias_map, baseDict).lower()} as {identifier.get_real_name().lower()}")
+            elif isinstance(token, Identifier):
+                if insideFunction:
+                    select_tokens.append(process_identifier(token, alias_map, baseDict).lower())
+                else:
+                    select_tokens.append(f"{process_identifier(token, alias_map,  baseDict).lower()} as {token.get_real_name().lower()}")
+            elif isinstance(token, Function):
+                #select_tokens.append(f"{token.get_name().lower()}({process_identifier(token.get_parameters(), alias_map, baseDict).lower()})")
+                for par in token.tokens:
+                    if isinstance(par, Identifier):
+                        select_tokens.append(par.get_name().lower()+"(")
+                    if isinstance(par, Parenthesis):
+                        select_tokens[-1] += process_select(par, alias_map, True)+") as funcResult" 
+            elif token.ttype is Wildcard:
+                select_tokens.append("*")
+            else:
+                continue
+        select_tokens.sort()
+        return ",".join(select_tokens)
+
+    def process_from(from_, alias_map):
+        from_tokens = []
+        if hasattr(from_, 'tokens'):
+            for token in from_.tokens:
+                if isinstance(token, IdentifierList):
+                    for identifier in token.get_identifiers():
+                        alias_map[identifier.get_real_name().lower()] = identifier.get_alias() or identifier.get_real_name()
+                        from_tokens.append(f"{identifier.get_real_name().lower()} {alias_map[identifier.get_real_name().lower()].lower()}")
+                elif isinstance(token, Identifier):
+                    alias_map[token.get_real_name().lower()] = token.get_alias() or token.get_real_name()
+                    from_tokens.append(f"{token.get_real_name().lower()} {alias_map[token.get_real_name().lower()].lower()}")
+                elif token.ttype is not None and token.ttype is Name:
+                    alias_map[token.value.lower()] = token.value.lower()
+                    from_tokens.append(f"{token.value.lower()} {alias_map[token.value.lower()].lower()}")
+                else:
+                    continue
+        from_tokens.sort()
+        return ",".join(from_tokens)
+
+    def process_groupby(groupby_, alias_map):
+        groupby_tokens = []
+        if(isinstance(groupby_, Identifier)):
+            groupby_tokens.append(process_identifier(groupby_, alias_map, baseDict).lower())
+        elif isinstance(token, IdentifierList):
+            for identifier in token.get_identifiers():
+                groupby_tokens.append(process_identifier(identifier, alias_map, baseDict).lower())
+        groupby_tokens.sort()
+        return ",".join(groupby_tokens)
+
+    def process_condition(token, alias_map, baseDict):
+        left, operator, right = [t for t in token.tokens if not t.is_whitespace]
+
+        flipAllowed = True
+        leftLiteral = False
+        rightLiteral = False
+
+        if is_value(right):
+            flipAllowed = False
+            rightLiteral = True
+
+        if is_value(left):
+            left, right = right, left
+            if operator.value == ">":
+                operator.value = "<"
+            elif operator.value == "<":
+                operator.value = ">"
+            leftLiteral = rightLiteral
+            rightLiteral = True
+            flipAllowed = False
+
+        left = process_identifier(left, alias_map, baseDict)
+        right = process_identifier(right, alias_map, baseDict)
+
+        if flipAllowed and left.lower() >= right.lower():
+            left, right = right, left
+            if operator.value == ">":
+                operator = "<"
+            elif operator.value == "<":
+                operator = ">"
+
+        return f"{left if leftLiteral else left.lower()} {operator} {right if rightLiteral else right.lower()}"
+
+    def is_value(token):
+        return token.ttype in (sqlparse.tokens.Token.Literal.Number.Integer,
+                               sqlparse.tokens.Token.Literal.Number.Float,
+                               sqlparse.tokens.Token.Literal.String.Single,
+                               sqlparse.tokens.Token.Literal.String.Symbol)
+
+    def process_paranthesis(parenthesis, alias_map, baseDict):
+        toks = []
+
+        bracketsRequired = True
+
+
+        for token in parenthesis.tokens:
+            if isinstance(token, Comparison):
+                toks.append(process_condition(token, alias_map, baseDict))
+            elif isinstance(token, Parenthesis):
+                toks.append(process_paranthesis(token, alias_map, baseDict))
+
+            elif token.value == '(':
+                toks.append(token.value)
+            else:
+                continue
+        x = parenthesis.flatten()
+        return " ".join(toks)
+
+    def process_where(where, alias_map, baseDict):
+            conditions = []
+            current_condition = []
+
+            for token in where.tokens:
+                if token.is_whitespace or (token.ttype is Keyword and token.value.upper() == "WHERE"):
+                    continue
+
+                if token.ttype is Keyword and token.value.upper() in ('AND', 'OR'):
+                    if current_condition:
+                        conditions.append(''.join(str(t) for t in current_condition).strip())
+                        current_condition = []
+                    conditions.append(token.value.upper())
+                elif isinstance(token, Comparison):
+                    current_condition.append(process_condition(token, alias_map, baseDict))
+                elif isinstance(token, Parenthesis):
+                    current_condition.append(process_paranthesis(token, alias_map, baseDict))
+                else:
+                    current_condition.append(token)
+
+            if current_condition:
+                conditions.append(''.join(str(t) for t in current_condition).strip())
+
+            sorted_conditions = []
+            current_group = []
+            last_connector = ""
+
+            if "OR" not in conditions:
+                for condition in conditions:
+                    if condition == 'AND':
+                        pass
+                    else:
+                        current_group.append(condition)
+
+                if current_group:
+                    sorted_conditions.extend(sorted(current_group))
+
+                return " AND ".join(sorted_conditions)
+            else:
+                return " ".join(conditions)
+
+                where_index = query.upper().find('WHERE')
+                normalized_query = query[:where_index + 5] + ' ' + ' '.join(sorted_conditions)
+
+    def process_where_xx(where, alias_map, baseDict):
+        where_tokens = []
+        for token in where.tokens:
+
+            if isinstance(token, Comparison):
+                left, operator, right = [t for t in token.tokens if not t.is_whitespace]
+                left = process_identifier(left, alias_map, baseDict)
+                right = process_identifier(right, alias_map, baseDict)
+                if left.lower() >= right.lower():
+                    left, right = right, left
+                    if operator.value == ">":
+                        operator = "<"
+                    elif operator.value == "<":
+                        operator = ">"
+
+                where_tokens.append(f"{left.lower()} {operator} {right.lower()}")
+            elif token.ttype is Keyword:
+                where_tokens.append(token.value.upper())
+            elif token.is_whitespace or (token.ttype is Keyword and token.value.upper() == "WHERE"):
+                continue
+            else:
+                where_tokens.append(str(token))
+        return " ".join(where_tokens)
+
+    # First pass to process FROM clause and populate alias_map
+    for token in parsed.tokens:
+        if token.is_whitespace:
+            continue
+        elif token.ttype is Keyword and token.value.upper() == 'FROM':
+            formatted_query.append('FROM')
+            pass
+        elif isinstance(token, Identifier) and formatted_query and formatted_query[-1] == 'FROM':
+            il = IdentifierList([token])
+            formatted_query.append(process_from(il, alias_map))
+            process_from(il, alias_map)
+        elif isinstance(token, IdentifierList) and formatted_query and formatted_query[-1] == 'FROM':
+            formatted_query.append(process_from(token, alias_map))
+            process_from(token, alias_map)
+
+    formatted_query = []
+
+    # Second pass to process SELECT and WHERE clauses
+    for token in parsed.tokens:
+        if token.is_whitespace:
+            continue
+        elif token.ttype is DML and token.value.upper() == 'SELECT':
+            formatted_query.append('SELECT')
+        elif token.ttype is Keyword and token.value.upper() == 'FROM':
+            formatted_query.append('FROM')  
+        elif token.ttype is Keyword and token.value.upper() == 'GROUP BY':
+            formatted_query.append('GROUP BY')
+        elif (isinstance(token, IdentifierList) or isinstance(token, Identifier)) and formatted_query and formatted_query[-1] == 'FROM':
+            formatted_query.append(process_from(token, alias_map))
+        elif (isinstance(token, IdentifierList) or isinstance(token, Identifier)) and formatted_query and formatted_query[-1] == 'GROUP BY':
+            formatted_query.append(process_groupby(token, alias_map))
+        elif isinstance(token, Where):
+            formatted_query.append('WHERE')
+            formatted_query.append(process_where(token, alias_map, baseDict))
+        elif formatted_query and formatted_query[-1] == 'SELECT' and (isinstance(token, IdentifierList) or isinstance(token, Function) or isinstance(token, Identifier)):
+            if isinstance(token, Function):
+                token = IdentifierList([token])
+            formatted_query.append(process_select(token, alias_map))
+        else:
+            formatted_query.append(str(token))
+
+    return " ".join(formatted_query)
+
+
+def findTableForColumn(data_dict, target_value, relevantTables):
+    l = []
+    for key, value_list in data_dict.items():
+        if key.lower() in relevantTables:
+            for sublist in value_list:
+                if sublist and sublist[0].lower() == target_value.lower():
+                    l.append(key)
+    if len(l) == 0:
+        for key, value_list in data_dict.items():
+            for sublist in value_list:
+                if sublist and sublist[0].lower() == target_value.lower():
+                    l.append(key)
+    return l
+
+
+def getTableScheme(table_name: str, tableDict: dict):
+    tab = tableDict[table_name]
+
+    # Format the schema
+    schema = "(" + ",".join([f"{col[0]}:{col[1]}" for col in tab]) + ")"
+    return schema
+
+def getCosetteKeyFromFile():
+    try:
+        with open("cosette_apikey.txt", "r") as f:
+            return f.read()
+    except FileNotFoundError:
+        return "NOKEY"
+
+def buildAndSendCosetteRequest(baseDict, sql, sol):
+
+    err = ""
+    for i in range(2):
+        try:
+            apiKey=getCosetteKeyFromFile()
+
+
+            schema = ""
+            for tab in baseDict.keys():
+                schema += f"schema sch{tab}{getTableScheme(tab, baseDict)};\n"
+            for tab in baseDict.keys():
+                schema += f"table {tab}(sch{tab});\n"
+
+            q1 = "query q1\n`"+sql+"`;\n"
+            q2 = "query q2\n`"+sol+"`;\n"
+
+            cosette = "-- random Kommentar\n" + schema + q1 + q2 + "verify q1 q2;\n"
+            print(cosette)
+
+            r = requests.post("https://demo.cosette.cs.washington.edu/solve",data={"api_key": apiKey, "query": cosette}, verify=False)
+
+            print(r.text)
+            return (r.json()['result'],r.text)
+            #return r.json()['result']
+
+        except Exception as e:
+            err = str(e)
+    return ("ERR", err)
+
+
+def checkColumns(sqlPath, solPath):
+    bd = Ba.getTableDict()
+    sql = normalizeSQLQuery(Ba.getSQLFromFile(sqlPath), bd)
+    sol = normalizeSQLQuery(Ba.getSQLFromFile(solPath), bd)
+
+    if("SELECT" in sql and "FROM" in sql):
+        start = str.find(sql, "SELECT")
+        end = str.find(sql, "FROM")
+        submission = str.strip(sql[start:end])
+        print("'"+submission+"'")
+
+        start = str.find(sol, "SELECT")
+        end = str.find(sol, "FROM")
+        solution = str.strip(sol[start:end])
+        print("'"+solution+"'")
+
+        if submission == solution:
+            return ""
+    return "Ausgegebene Spalten sind nicht korrekt (oder nicht automatisch überprüfbar)."
+
+
+def checkTables(sqlPath, solPath):
+    bd = Ba.getTableDict()
+    sql = normalizeSQLQuery(Ba.getSQLFromFile(sqlPath), bd)
+    sol = normalizeSQLQuery(Ba.getSQLFromFile(solPath), bd)
+
+    if("SELECT" in sql and "FROM" in sql):
+        endFromKeywords = ["WHERE", "GROUP", "ORDER", "LIMIT", ";"]
+
+        start = str.find(sql, "FROM")
+        end = -1
+
+
+        for keyword in endFromKeywords:
+            if(str.find(sql, keyword) != -1):
+                end = str.find(sql, keyword)
+                break
+        if(end == -1):
+            end = len(sql)
+
+        submission = str.strip(sql[start:end])
+        print("'"+submission+"'")
+
+        start = str.find(sol, "FROM")
+        end = -1
+        
+        for keyword in endFromKeywords:
+            if(str.find(sol, keyword) != -1):
+                end = str.find(sol, keyword)
+                break
+        if(end == -1):
+            end = len(sol)
+
+        solution = str.strip(sol[start:end])
+        print("'"+solution+"'")
+
+        if submission == solution:
+            return ""
+    return "Verwendete Tabellen sind nicht korrekt (oder nicht automatisch überprüfbar)."
+
+
+def checkEquality(sqlPath, solPath):
+    bd = Ba.getTableDict()
+    sql = normalizeSQLQuery(Ba.getSQLFromFile(sqlPath), bd)
+    sol = normalizeSQLQuery(Ba.getSQLFromFile(solPath), bd)
+    if(sql=='' or sol==''):
+        return "\n\nSQL-Datei ist leer. Aufgabe wurde noch nicht bearbeitet."
+
+    if(sql==sol):
+        return ""
+
+    result = buildAndSendCosetteRequest(bd, sql, sol)
+
+    if(result[0] == "ERR"):
+        return "\n\nFehler bei der automatischen Überprüfung der Abgabe. Es kann keine Aussage über die Korrektheit der Abgabe getroffen werden."
+    elif(result[0] != "EQ"):
+        return "\n\nDie Abgabe stimmt nicht mit der Musterlösung überein."
+    return ""
+

sql_testing_tools/__init__.py

@@ -0,0 +1,3 @@
+from .BaseAccess import *
+from .Helper import *
+

sql_testing_tools-0.1.0.dist-info/METADATA

@@ -0,0 +1,6 @@
+Metadata-Version: 2.1
+Name: sql_testing_tools
+Version: 0.1.0
+Requires-Dist: sqlparse>=0.5.1
+Requires-Dist: requests>=2.32.3
+

sql_testing_tools-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+sql_testing_tools/BaseAccess.py,sha256=tC6dpS6vbb7a7OibGEKlAJiHGnn_OAbmDehweRR0ebA,5188
+sql_testing_tools/Helper.py,sha256=wEun9GAuEP6ibTAHoNnqWqEshWpiT1fJnicVIpwmtJM,16714
+sql_testing_tools/__init__.py,sha256=NYoXmLD8NmuU9TnMS46-gXmMutFHAXxuqaDlITZhRfY,52
+sql_testing_tools/databases/bayern.db,sha256=gREMuoSNYyCeTn2v7e5nPafPKr5IKw5XfYgbn2yxun0,4874240
+sql_testing_tools/databases/ladepunkte.db,sha256=ZR-ibbtdCtovPg6i5eAdKY-E4ahkWkZXha71c1RB6pA,7430144
+sql_testing_tools/databases/straftaten.db,sha256=Z1T5OUI1CTvxNFlFGzyjfesdhK9fivgzHxVW4lyRC2w,50950144
+sql_testing_tools-0.1.0.dist-info/METADATA,sha256=EDuvwT2Gv-AtpZYZWXaOl6H0Cfu3pj7M2KdTph703U0,131
+sql_testing_tools-0.1.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+sql_testing_tools-0.1.0.dist-info/top_level.txt,sha256=qwEkmfJ_lYhIwcLXgfU75m0_JavVbOg96fN2zSatN3E,18
+sql_testing_tools-0.1.0.dist-info/RECORD,,

sql_testing_tools-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.45.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

sql_testing_tools-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+sql_testing_tools