sql-testing-tools 0.1.4__tar.gz → 0.1.6__tar.gz

sql_testing_tools-0.1.6/PKG-INFO

@@ -0,0 +1,25 @@
+Metadata-Version: 2.1
+Name: sql_testing_tools
+Version: 0.1.6
+Description-Content-Type: text/markdown
+Requires-Dist: sqlparse>=0.5.1
+Requires-Dist: requests>=2.32.3
+
+A library that normalizes simple SQL queries and compares them first by equality of the normalized string and then using the cosette API. 
+
+
+### [Beta in Development!]
+ [![Build and Test](https://github.com/ValentinHerrmann/sql_testing_tools/actions/workflows/python-build.yml/badge.svg)](https://github.com/ValentinHerrmann/sql_testing_tools/actions/workflows/python-build.yml)
+ [![Build and Test](https://github.com/ValentinHerrmann/sql_testing_tools/actions/workflows/python-unittests.yml/badge.svg)](https://github.com/ValentinHerrmann/sql_testing_tools/actions/workflows/python-unittests.yml)
+
+
+
+### V0.1.4 + V0.1.5
+- V0.1.5 is just a reupload of V0.1.4
+- Chained conditions (with AND,OR and Paranthesises) in WHERE statement
+
+### V0.1.3
+- SELECT: columns with our without table prefix
+- FROM: one or more table from DB; no queries as tables!
+- WHERE: single conditions; no Paranthesises!
+- GROUP BY one or more columns

{sql_testing_tools-0.1.4 → sql_testing_tools-0.1.6}/README.md

@@ -1,3 +1,3 @@
-[Beta in Development]
+**[Beta in Development!]**
 
 This project is a wrapper for packaging [SQL_Testing_Tools](https://github.com/ValentinHerrmann/sql_testing_tools) into a Wheel and upload it to PyPi.

{sql_testing_tools-0.1.4 → sql_testing_tools-0.1.6}/setup.py

@@ -6,7 +6,7 @@ with open("sql_testing_tools/README.md", "r") as fh:
 
 setup(
     name='sql_testing_tools',
-    version='0.1.4',
+    version='0.1.6',
     packages=find_packages(),
     install_requires=[
         'sqlparse>=0.5.1',

sql_testing_tools-0.1.6/sql_testing_tools/Helper.py

@@ -0,0 +1,210 @@
+try:
+    import BaseAccess as Ba
+    import TokenProcessing as Tp
+except ImportError:
+    from . import BaseAccess as Ba
+    from . import TokenProcessing as Tp
+
+import requests
+import sqlparse
+
+from sqlparse.sql import Identifier, IdentifierList, Where, Comparison, Function, Parenthesis
+from sqlparse.tokens import Keyword, DML, Name, Wildcard
+
+
+
+def normalizeSQLQuery(query, baseDict):
+    try:
+        query = query.replace("\"", "'")
+        parsed = sqlparse.parse(query)[0]
+        parsed.tokens = [token for token in parsed.tokens if not token.is_whitespace]
+    except Exception as e:
+        raise Exception(f"\nSyntax-Fehler in der SQL-Abfrage.")
+
+    formatted_query = []
+    alias_map = {}
+
+    # First pass to process FROM clause and populate alias_map
+    for token in parsed.tokens:
+        if token.is_whitespace:
+            continue
+        elif token.ttype is Keyword and token.value.upper() == 'FROM':
+            formatted_query.append('FROM')
+            pass
+        elif isinstance(token, Identifier) and formatted_query and formatted_query[-1] == 'FROM':
+            il = IdentifierList([token])
+            formatted_query.append(Tp._from(il, alias_map, baseDict))
+            Tp._from(il, alias_map, baseDict)
+        elif isinstance(token, IdentifierList) and formatted_query and formatted_query[-1] == 'FROM':
+            formatted_query.append(Tp._from(token, alias_map, baseDict))
+            Tp._from(token, alias_map, baseDict)
+
+    formatted_query = []
+
+    # Second pass to process SELECT and WHERE clauses
+    for token in parsed.tokens:
+        if token.is_whitespace:
+            continue
+        elif token.ttype is DML and token.value.upper() == 'SELECT':
+            formatted_query.append('SELECT')
+        elif token.ttype is Keyword and token.value.upper() == 'FROM':
+            formatted_query.append('FROM')  
+        elif token.ttype is Keyword and token.value.upper() == 'GROUP BY':
+            formatted_query.append('GROUP BY')
+        elif (isinstance(token, IdentifierList) or isinstance(token, Identifier)) and formatted_query and formatted_query[-1] == 'FROM':
+            formatted_query.append(Tp._from(token, alias_map, baseDict))
+        elif (isinstance(token, IdentifierList) or isinstance(token, Identifier)) and formatted_query and formatted_query[-1] == 'GROUP BY':
+            formatted_query.append(Tp._groupby(token, alias_map, baseDict))
+        elif isinstance(token, Where):
+            formatted_query.append('WHERE')
+            formatted_query.append(Tp._where(token, alias_map, baseDict))
+        elif formatted_query and formatted_query[-1] == 'SELECT' and (isinstance(token, IdentifierList) or isinstance(token, Function) or isinstance(token, Identifier)):
+            if isinstance(token, Function):
+                token = IdentifierList([token])
+            formatted_query.append(Tp._select(token, alias_map, baseDict))
+        else:
+            formatted_query.append(str(token))
+
+    return " ".join(formatted_query)
+
+
+def findTableForColumn(data_dict, target_value, relevantTables):
+    l = []
+    for key, value_list in data_dict.items():
+        if key.lower() in relevantTables:
+            for sublist in value_list:
+                if sublist and sublist[0].lower() == target_value.lower():
+                    l.append(key)
+    if len(l) == 0:
+        for key, value_list in data_dict.items():
+            for sublist in value_list:
+                if sublist and sublist[0].lower() == target_value.lower():
+                    l.append(key)
+    return l
+
+
+def getTableScheme(table_name: str, tableDict: dict):
+    tab = tableDict[table_name]
+
+    # Format the schema
+    schema = "(" + ",".join([f"{col[0]}:{col[1]}" for col in tab]) + ")"
+    return schema
+
+def getCosetteKeyFromFile():
+    try:
+        with open("cosette_apikey.txt", "r") as f:
+            return f.read()
+    except FileNotFoundError:
+        return "NOKEY"
+
+def buildAndSendCosetteRequest(baseDict, sql, sol):
+
+    err = ""
+    for i in range(2):
+        try:
+            apiKey=getCosetteKeyFromFile()
+
+
+            schema = ""
+            for tab in baseDict.keys():
+                schema += f"schema sch{tab}{getTableScheme(tab, baseDict)};\n"
+            for tab in baseDict.keys():
+                schema += f"table {tab}(sch{tab});\n"
+
+            q1 = "query q1\n`"+sql+"`;\n"
+            q2 = "query q2\n`"+sol+"`;\n"
+
+            cosette = "-- random Kommentar\n" + schema + q1 + q2 + "verify q1 q2;\n"
+            print(cosette)
+
+            r = requests.post("https://demo.cosette.cs.washington.edu/solve",data={"api_key": apiKey, "query": cosette}, verify=False)
+
+            print(r.text)
+            return (r.json()['result'],r.text)
+            #return r.json()['result']
+
+        except Exception as e:
+            err = str(e)
+    return ("ERR", err)
+
+
+def checkColumns(sqlPath, solPath):
+    bd = Ba.getTableDict()
+    sql = normalizeSQLQuery(Ba.getSQLFromFile(sqlPath), bd)
+    sol = normalizeSQLQuery(Ba.getSQLFromFile(solPath), bd)
+
+    if("SELECT" in sql and "FROM" in sql):
+        start = str.find(sql, "SELECT")
+        end = str.find(sql, "FROM")
+        submission = str.strip(sql[start:end])
+        print("'"+submission+"'")
+
+        start = str.find(sol, "SELECT")
+        end = str.find(sol, "FROM")
+        solution = str.strip(sol[start:end])
+        print("'"+solution+"'")
+
+        if submission == solution:
+            return ""
+    return "Ausgegebene Spalten sind nicht korrekt (oder nicht automatisch überprüfbar)."
+
+
+def checkTables(sqlPath, solPath):
+    bd = Ba.getTableDict()
+    sql = normalizeSQLQuery(Ba.getSQLFromFile(sqlPath), bd)
+    sol = normalizeSQLQuery(Ba.getSQLFromFile(solPath), bd)
+
+    if("SELECT" in sql and "FROM" in sql):
+        endFromKeywords = ["WHERE", "GROUP", "ORDER", "LIMIT", ";"]
+
+        start = str.find(sql, "FROM")
+        end = -1
+
+
+        for keyword in endFromKeywords:
+            if(str.find(sql, keyword) != -1):
+                end = str.find(sql, keyword)
+                break
+        if(end == -1):
+            end = len(sql)
+
+        submission = str.strip(sql[start:end])
+        print("'"+submission+"'")
+
+        start = str.find(sol, "FROM")
+        end = -1
+        
+        for keyword in endFromKeywords:
+            if(str.find(sol, keyword) != -1):
+                end = str.find(sol, keyword)
+                break
+        if(end == -1):
+            end = len(sol)
+
+        solution = str.strip(sol[start:end])
+        print("'"+solution+"'")
+
+        if submission == solution:
+            return ""
+    return "Verwendete Tabellen sind nicht korrekt (oder nicht automatisch überprüfbar)."
+
+
+def checkEquality(sqlPath, solPath):
+    bd = Ba.getTableDict()
+    sql = normalizeSQLQuery(Ba.getSQLFromFile(sqlPath), bd)
+    sol = normalizeSQLQuery(Ba.getSQLFromFile(solPath), bd)
+    if(sql=='' or sol==''):
+        return "\n\nSQL-Datei ist leer. Aufgabe wurde noch nicht bearbeitet."
+
+    if(sql==sol):
+        return ""
+
+    result = buildAndSendCosetteRequest(bd, sql, sol)
+
+    if(result[0] == "ERR"):
+        return "\n\nFehler bei der automatischen Überprüfung der Abgabe. Es kann keine Aussage über die Korrektheit der Abgabe getroffen werden."
+    elif(result[0] != "EQ"):
+        return "\n\nDie Abgabe stimmt nicht mit der Musterlösung überein."
+    return ""
+
+

sql_testing_tools-0.1.6/sql_testing_tools/TokenProcessing.py

@@ -0,0 +1,361 @@
+import sqlparse
+from sqlparse.sql import Identifier, IdentifierList, Where, Comparison, Function, Parenthesis
+from sqlparse.tokens import Keyword, DML, Name, Wildcard, Whitespace
+
+try:
+    import Helper as He
+except ImportError:
+    from . import Helper as He
+
+
+
+        
+#####################################
+#####################################
+
+def _identifier(identifier, alias_map, baseDict: dict):
+        if isinstance(identifier, Identifier):
+            if identifier.get_real_name() and identifier.get_parent_name() and identifier.get_parent_name().lower() in alias_map.keys():
+                return f"{alias_map[identifier.get_parent_name().lower()].lower()}.{identifier.get_real_name().lower()}"
+            elif identifier.get_real_name():
+                tables = He.findTableForColumn(baseDict, identifier.get_real_name(), alias_map.keys())
+                if len(tables) == 1:
+                    alias_map[tables[0].lower()] = tables[0]
+                    return f"{tables[0].lower()}.{identifier.get_real_name().lower()}"
+                else:
+                    return f"{identifier.get_real_name().lower()}"
+        return str(identifier)
+
+        
+#####################################
+#####################################
+
+
+def _select(select, alias_map, baseDict: dict, insideFunction=False):
+    select_tokens = []
+    for token in select.tokens:
+        if isinstance(token, IdentifierList):
+            for identifier in token.get_identifiers():
+                select_tokens.append(f"{_identifier(identifier, alias_map, baseDict).lower()} as {identifier.get_real_name().lower()}")
+        elif isinstance(token, Identifier):
+            if insideFunction:
+                select_tokens.append(_identifier(token, alias_map, baseDict).lower())
+            else:
+                select_tokens.append(f"{_identifier(token, alias_map,  baseDict).lower()} as {token.get_real_name().lower()}")
+        elif isinstance(token, Function):
+            #select_tokens.append(f"{token.get_name().lower()}({_identifier(token.get_parameters(), alias_map, baseDict).lower()})")
+            for par in token.tokens:
+                if isinstance(par, Identifier):
+                    select_tokens.append(par.get_name().lower()+"(")
+                if isinstance(par, Parenthesis):
+                    select_tokens[-1] += _select(par, alias_map, baseDict, True)+") as funcResult" 
+        elif token.ttype is Wildcard:
+            select_tokens.append("*")
+        else:
+            continue
+    select_tokens.sort()
+    return ",".join(select_tokens)
+
+
+#####################################
+#####################################
+
+
+def _from(from_, alias_map, baseDict: dict):
+    from_tokens = []
+    if hasattr(from_, 'tokens'):
+        for token in from_.tokens:
+            if isinstance(token, IdentifierList):
+                for identifier in token.get_identifiers():
+                    alias_map[identifier.get_real_name().lower()] = identifier.get_alias() or identifier.get_real_name()
+                    from_tokens.append(f"{identifier.get_real_name().lower()} {alias_map[identifier.get_real_name().lower()].lower()}")
+            elif isinstance(token, Identifier):
+                alias_map[token.get_real_name().lower()] = token.get_alias() or token.get_real_name()
+                from_tokens.append(f"{token.get_real_name().lower()} {alias_map[token.get_real_name().lower()].lower()}")
+            elif token.ttype is not None and token.ttype is Name:
+                alias_map[token.value.lower()] = token.value.lower()
+                from_tokens.append(f"{token.value.lower()} {alias_map[token.value.lower()].lower()}")
+            else:
+                continue
+    from_tokens.sort()
+    return ",".join(from_tokens)
+
+
+#####################################
+#####################################
+
+
+def _groupby(groupby_, alias_map, baseDict: dict):
+    groupby_tokens = []
+    if(isinstance(groupby_, Identifier)):
+        groupby_tokens.append(_identifier(groupby_, alias_map, baseDict).lower())
+    elif isinstance(groupby_, IdentifierList):
+        for identifier in groupby_.get_identifiers():
+            groupby_tokens.append(_identifier(identifier, alias_map, baseDict).lower())
+    groupby_tokens.sort()
+    return ",".join(groupby_tokens)
+
+def _condition(token, alias_map, baseDict: dict):
+    left, operator, right = [t for t in token.tokens if not t.is_whitespace]
+
+    flipAllowed = True
+    leftLiteral = False
+    rightLiteral = False
+
+    if is_value(right):
+        flipAllowed = False
+        rightLiteral = True
+
+    if is_value(left):
+        left, right = right, left
+        if operator.value == ">":
+            operator.value = "<"
+        elif operator.value == "<":
+            operator.value = ">"
+        leftLiteral = rightLiteral
+        rightLiteral = True
+        flipAllowed = False
+
+    left = _identifier(left, alias_map, baseDict)
+    right = _identifier(right, alias_map, baseDict)
+
+    if flipAllowed and left.lower() >= right.lower():
+        left, right = right, left
+        if operator.value == ">":
+            operator = "<"
+        elif operator.value == "<":
+            operator = ">"
+
+    return f"{left if leftLiteral else left.lower()} {operator} {right if rightLiteral else right.lower()}"
+
+def is_value(token):
+    return token.ttype in (sqlparse.tokens.Token.Literal.Number.Integer,
+                            sqlparse.tokens.Token.Literal.Number.Float,
+                            sqlparse.tokens.Token.Literal.String.Single,
+                            sqlparse.tokens.Token.Literal.String.Symbol)
+
+
+
+#####################################
+#####################################
+
+
+def _paranthesis(parenthesis, alias_map, baseDict: dict):
+    toks = []
+
+    bracketsRequired = True
+    closeBracketsAfterNext = False
+
+    and_count = sum(1 for token in parenthesis.tokens if token.ttype is Keyword and token.value == "AND")  # Zähle "AND"-Tokens
+    or_count = sum(1 for token in parenthesis.tokens if token.ttype is Keyword and token.value == "OR")  # Zähle "AND"-Tokens
+
+
+    if(and_count==1 and or_count == 0):
+        val = _2element_par(parenthesis, alias_map, baseDict, "AND")
+        return val
+    elif(and_count==0 and or_count == 1):
+        val = _2element_par(parenthesis, alias_map, baseDict, "OR")
+        return val
+
+
+
+
+
+
+    for token in parenthesis.tokens:
+        if isinstance(token, Comparison):
+            toks.append(_condition(token, alias_map, baseDict))
+        elif isinstance(token, Parenthesis):
+            toks.append(_paranthesis(token, alias_map, baseDict))
+
+        elif token.ttype is Keyword: #AND,OR
+            if token.value == "AND":
+                pass
+            elif token.value == "OR":
+                pass
+            elif token.value == "NOT":
+                pass
+
+        elif token.value == '(':
+            toks.append(token.value)
+        else:
+            continue
+    x = parenthesis.flatten()
+    return " ".join(toks)
+
+
+def _2element_par(parenthesis, alias_map, baseDict: dict, keyword):
+    toks = []
+
+    for token in parenthesis.tokens:
+        if isinstance(token, Comparison):
+            toks.append(_condition(token, alias_map, baseDict))
+        elif isinstance(token, Parenthesis):
+            toks.append(_paranthesis(token, alias_map, baseDict))
+    toks = toks.sort()
+    fill = (" "+keyword+" ")
+    return fill.join(toks)
+
+
+def count_keywordValues(tokens, values):
+    return len([token for token in tokens if token.ttype is Keyword and token.value.upper() in values])
+
+
+def find_index_of_keyword(tokens, keyword):
+    for index, token in enumerate(tokens):
+        if token.ttype is Keyword and token.value.upper() == keyword:
+            return index
+    return -1  # Wenn kein AND-Token gefunden wird
+
+
+def _where(where, alias_map, baseDict: dict):
+    conditions = []
+    current_condition = []
+
+
+
+    isParanthesis = isinstance(where, Parenthesis)
+    
+
+    and_count = where.normalized.upper().count(" AND ")
+    or_count = where.normalized.upper().count(" OR ")
+
+
+    # No AND / OR
+    if and_count + or_count == 0:
+        return _where_simpleCondition(where, alias_map, baseDict)
+
+    # only ANDs / only ORs
+    elif and_count == 0 or or_count == 0:
+        return _where_sameStrengthKeywords(where, alias_map, baseDict, " AND " if and_count > 0 else " OR ")
+    
+    # ANDs and ORs
+
+    where_tokens = [token for token in where.tokens if token.ttype is not Whitespace and token.value != "WHERE"]    
+
+    # count keywords on top level
+    top_and_count = count_keywordValues(where_tokens, ["AND"])
+    top_or_count = count_keywordValues(where_tokens, ["OR"])
+
+    # two conditions on top level
+    if top_and_count + top_or_count == 1:
+        keyword = " AND " if top_and_count == 1 else " OR "
+        return _where_twoConditionsOnTopLevel(where, alias_map, baseDict, keyword, isParanthesis)
+    else:
+        where_tokens = _where_addBracketsAroundAND(where, alias_map, baseDict)
+        return _where(Where(where_tokens), alias_map, baseDict)
+
+def _where_addBracketsAroundAND(where, alias_map, baseDict: dict):
+    index = find_index_of_keyword(where.tokens, "AND")
+    par = Parenthesis(where.tokens[index-2:index+3])
+    where_tokens = where.tokens[:index-2] + [par] + where.tokens[index+3:]
+    return where_tokens
+
+
+
+    # for token in where.tokens:
+    #     if token.is_whitespace or (token.ttype is Keyword and token.value.upper() == "WHERE"):
+    #         continue
+
+    #     if token.ttype is Keyword and token.value.upper() in ('AND', 'OR'):
+    #         if current_condition:
+    #             conditions.append(''.join(str(t) for t in current_condition).strip())
+    #             current_condition = []
+    #         conditions.append(token.value.upper())
+    #     else:
+    #         current_condition.append(token)
+
+    # if current_condition:
+    #     conditions.append(''.join(str(t) for t in current_condition).strip())
+
+    # sorted_conditions = []
+    # current_group = []
+    # last_connector = ""
+
+    # if "OR" not in conditions:
+    #     for condition in conditions:
+    #         if condition == 'AND':
+    #             pass
+    #         else:
+    #             current_group.append(condition)
+
+    #     if current_group:
+    #         sorted_conditions.extend(sorted(current_group))
+
+    #     return " AND ".join(sorted_conditions)
+    # else:
+    #     return " ".join(conditions)
+
+    #     where_index = query.upper().find('WHERE')
+        
+    #     normalized_query = query[:where_index + 5] + ' ' + ' '.join(sorted_conditions)
+
+
+
+def _where_simpleCondition(where, alias_map, baseDict: dict):
+    cond = []
+    for tok in where.tokens:
+        if isinstance(tok, Comparison):
+            cond.append(_condition(tok, alias_map, baseDict))
+        elif isinstance(tok, Parenthesis):
+            for _tok in tok.tokens:
+                if isinstance(_tok, Comparison):
+                    cond.append(_condition(_tok, alias_map, baseDict))
+                elif isinstance(_tok, Parenthesis):
+                    tok = _tok
+    return "".join(cond)
+
+def _where_sameStrengthKeywords(where, alias_map, baseDict: dict, keyword: str):
+    cond = []
+    for tok in where.tokens:
+        if isinstance(tok, Comparison):
+            cond.append(_condition(tok, alias_map, baseDict))
+        elif isinstance(tok, Parenthesis):
+            for _tok in tok.tokens:
+                if isinstance(_tok, Comparison):
+                    cond.append(_condition(_tok, alias_map, baseDict))
+                elif isinstance(_tok, Parenthesis):
+                    tok = _tok
+    cond.sort()
+    return keyword.join(cond)
+
+def _where_twoConditionsOnTopLevel(where, alias_map, baseDict: dict, keyword, isParanthesis):
+    current_condition = []
+    for tok in where.tokens:
+        if isinstance(tok, Comparison):
+            current_condition.append(_condition(tok, alias_map, baseDict))
+        elif isinstance(tok, Parenthesis):
+            current_condition.append(_where(tok, alias_map, baseDict))
+        current_condition.sort()
+        if keyword == " AND " or isParanthesis:
+            return "(" + keyword.join(current_condition) + ")"
+        else:
+            return keyword.join(current_condition)
+
+
+
+
+# def _where_xx(where, alias_map, baseDict: dict):
+#     where_tokens = []
+#     for token in where.tokens:
+
+#         if isinstance(token, Comparison):
+#             left, operator, right = [t for t in token.tokens if not t.is_whitespace]
+#             left = _identifier(left, alias_map, baseDict)
+#             right = _identifier(right, alias_map, baseDict)
+#             if left.lower() >= right.lower():
+#                 left, right = right, left
+#                 if operator.value == ">":
+#                     operator = "<"
+#                 elif operator.value == "<":
+#                     operator = ">"
+
+#             where_tokens.append(f"{left.lower()} {operator} {right.lower()}")
+#         elif token.ttype is Keyword:
+#             where_tokens.append(token.value.upper())
+#         elif token.is_whitespace or (token.ttype is Keyword and token.value.upper() == "WHERE"):
+#             continue
+#         else:
+#             where_tokens.append(str(token))
+#     return " ".join(where_tokens)
+

sql_testing_tools-0.1.6/sql_testing_tools.egg-info/PKG-INFO

@@ -0,0 +1,25 @@
+Metadata-Version: 2.1
+Name: sql_testing_tools
+Version: 0.1.6
+Description-Content-Type: text/markdown
+Requires-Dist: sqlparse>=0.5.1
+Requires-Dist: requests>=2.32.3
+
+A library that normalizes simple SQL queries and compares them first by equality of the normalized string and then using the cosette API. 
+
+
+### [Beta in Development!]
+ [![Build and Test](https://github.com/ValentinHerrmann/sql_testing_tools/actions/workflows/python-build.yml/badge.svg)](https://github.com/ValentinHerrmann/sql_testing_tools/actions/workflows/python-build.yml)
+ [![Build and Test](https://github.com/ValentinHerrmann/sql_testing_tools/actions/workflows/python-unittests.yml/badge.svg)](https://github.com/ValentinHerrmann/sql_testing_tools/actions/workflows/python-unittests.yml)
+
+
+
+### V0.1.4 + V0.1.5
+- V0.1.5 is just a reupload of V0.1.4
+- Chained conditions (with AND,OR and Paranthesises) in WHERE statement
+
+### V0.1.3
+- SELECT: columns with our without table prefix
+- FROM: one or more table from DB; no queries as tables!
+- WHERE: single conditions; no Paranthesises!
+- GROUP BY one or more columns

{sql_testing_tools-0.1.4 → sql_testing_tools-0.1.6}/sql_testing_tools.egg-info/SOURCES.txt

@@ -2,6 +2,7 @@ README.md
 setup.py
 sql_testing_tools/BaseAccess.py
 sql_testing_tools/Helper.py
+sql_testing_tools/TokenProcessing.py
 sql_testing_tools/__init__.py
 sql_testing_tools.egg-info/PKG-INFO
 sql_testing_tools.egg-info/SOURCES.txt

sql_testing_tools-0.1.4/PKG-INFO

@@ -1,10 +0,0 @@
-Metadata-Version: 2.1
-Name: sql_testing_tools
-Version: 0.1.4
-Description-Content-Type: text/markdown
-Requires-Dist: sqlparse>=0.5.1
-Requires-Dist: requests>=2.32.3
-
-[Beta in Development]
-
-A library that normalizes simple SQL queries and compares them first by equality of the normalized string and then using the cosette API. 

sql_testing_tools-0.1.4/sql_testing_tools/Helper.py

@@ -1,410 +0,0 @@
-import os
-from . import BaseAccess as Ba
-
-
-try:
-    import sqlparse
-except ImportError:
-    print("Trying to Install required module: sqlparse\n")
-    os.system('pip install sqlparse')
-    import sqlparse
-
-
-try:
-    import requests
-except ImportError:
-    print("Trying to Install required module: requests\n")
-    os.system('pip install requests')
-    import requests
-
-from sqlparse.sql import Identifier, IdentifierList, Where, Comparison, Function, Parenthesis
-from sqlparse.tokens import Keyword, DML, Name, Wildcard
-
-
-def normalizeSQLQuery(query, baseDict):
-    try:
-        query = query.replace("\"", "'")
-        parsed = sqlparse.parse(query)[0]
-        parsed.tokens = [token for token in parsed.tokens if not token.is_whitespace]
-    except Exception as e:
-        raise Exception(f"\nSyntax-Fehler in der SQL-Abfrage.")
-
-    formatted_query = []
-    alias_map = {}
-
-    def process_identifier(identifier, alias_map, baseDict: dict):
-        if isinstance(identifier, Identifier):
-            if identifier.get_real_name() and identifier.get_parent_name() and identifier.get_parent_name().lower() in alias_map.keys():
-                return f"{alias_map[identifier.get_parent_name().lower()].lower()}.{identifier.get_real_name().lower()}"
-            elif identifier.get_real_name():
-                tables = findTableForColumn(baseDict, identifier.get_real_name(), alias_map.keys())
-                if len(tables) == 1:
-                    alias_map[tables[0].lower()] = tables[0]
-                    return f"{tables[0].lower()}.{identifier.get_real_name().lower()}"
-                else:
-                    return f"{identifier.get_real_name().lower()}"
-        return str(identifier)
-
-    def process_select(select, alias_map, insideFunction=False):
-        select_tokens = []
-        for token in select.tokens:
-            if isinstance(token, IdentifierList):
-                for identifier in token.get_identifiers():
-                    select_tokens.append(f"{process_identifier(identifier, alias_map, baseDict).lower()} as {identifier.get_real_name().lower()}")
-            elif isinstance(token, Identifier):
-                if insideFunction:
-                    select_tokens.append(process_identifier(token, alias_map, baseDict).lower())
-                else:
-                    select_tokens.append(f"{process_identifier(token, alias_map,  baseDict).lower()} as {token.get_real_name().lower()}")
-            elif isinstance(token, Function):
-                #select_tokens.append(f"{token.get_name().lower()}({process_identifier(token.get_parameters(), alias_map, baseDict).lower()})")
-                for par in token.tokens:
-                    if isinstance(par, Identifier):
-                        select_tokens.append(par.get_name().lower()+"(")
-                    if isinstance(par, Parenthesis):
-                        select_tokens[-1] += process_select(par, alias_map, True)+") as funcResult" 
-            elif token.ttype is Wildcard:
-                select_tokens.append("*")
-            else:
-                continue
-        select_tokens.sort()
-        return ",".join(select_tokens)
-
-    def process_from(from_, alias_map):
-        from_tokens = []
-        if hasattr(from_, 'tokens'):
-            for token in from_.tokens:
-                if isinstance(token, IdentifierList):
-                    for identifier in token.get_identifiers():
-                        alias_map[identifier.get_real_name().lower()] = identifier.get_alias() or identifier.get_real_name()
-                        from_tokens.append(f"{identifier.get_real_name().lower()} {alias_map[identifier.get_real_name().lower()].lower()}")
-                elif isinstance(token, Identifier):
-                    alias_map[token.get_real_name().lower()] = token.get_alias() or token.get_real_name()
-                    from_tokens.append(f"{token.get_real_name().lower()} {alias_map[token.get_real_name().lower()].lower()}")
-                elif token.ttype is not None and token.ttype is Name:
-                    alias_map[token.value.lower()] = token.value.lower()
-                    from_tokens.append(f"{token.value.lower()} {alias_map[token.value.lower()].lower()}")
-                else:
-                    continue
-        from_tokens.sort()
-        return ",".join(from_tokens)
-
-    def process_groupby(groupby_, alias_map):
-        groupby_tokens = []
-        if(isinstance(groupby_, Identifier)):
-            groupby_tokens.append(process_identifier(groupby_, alias_map, baseDict).lower())
-        elif isinstance(token, IdentifierList):
-            for identifier in token.get_identifiers():
-                groupby_tokens.append(process_identifier(identifier, alias_map, baseDict).lower())
-        groupby_tokens.sort()
-        return ",".join(groupby_tokens)
-
-    def process_condition(token, alias_map, baseDict):
-        left, operator, right = [t for t in token.tokens if not t.is_whitespace]
-
-        flipAllowed = True
-        leftLiteral = False
-        rightLiteral = False
-
-        if is_value(right):
-            flipAllowed = False
-            rightLiteral = True
-
-        if is_value(left):
-            left, right = right, left
-            if operator.value == ">":
-                operator.value = "<"
-            elif operator.value == "<":
-                operator.value = ">"
-            leftLiteral = rightLiteral
-            rightLiteral = True
-            flipAllowed = False
-
-        left = process_identifier(left, alias_map, baseDict)
-        right = process_identifier(right, alias_map, baseDict)
-
-        if flipAllowed and left.lower() >= right.lower():
-            left, right = right, left
-            if operator.value == ">":
-                operator = "<"
-            elif operator.value == "<":
-                operator = ">"
-
-        return f"{left if leftLiteral else left.lower()} {operator} {right if rightLiteral else right.lower()}"
-
-    def is_value(token):
-        return token.ttype in (sqlparse.tokens.Token.Literal.Number.Integer,
-                               sqlparse.tokens.Token.Literal.Number.Float,
-                               sqlparse.tokens.Token.Literal.String.Single,
-                               sqlparse.tokens.Token.Literal.String.Symbol)
-
-    def process_paranthesis(parenthesis, alias_map, baseDict):
-        toks = []
-
-        bracketsRequired = True
-
-
-        for token in parenthesis.tokens:
-            if isinstance(token, Comparison):
-                toks.append(process_condition(token, alias_map, baseDict))
-            elif isinstance(token, Parenthesis):
-                toks.append(process_paranthesis(token, alias_map, baseDict))
-
-            elif token.value == '(':
-                toks.append(token.value)
-            else:
-                continue
-        x = parenthesis.flatten()
-        return " ".join(toks)
-
-    def process_where(where, alias_map, baseDict):
-            conditions = []
-            current_condition = []
-
-            for token in where.tokens:
-                if token.is_whitespace or (token.ttype is Keyword and token.value.upper() == "WHERE"):
-                    continue
-
-                if token.ttype is Keyword and token.value.upper() in ('AND', 'OR'):
-                    if current_condition:
-                        conditions.append(''.join(str(t) for t in current_condition).strip())
-                        current_condition = []
-                    conditions.append(token.value.upper())
-                elif isinstance(token, Comparison):
-                    current_condition.append(process_condition(token, alias_map, baseDict))
-                elif isinstance(token, Parenthesis):
-                    current_condition.append(process_paranthesis(token, alias_map, baseDict))
-                else:
-                    current_condition.append(token)
-
-            if current_condition:
-                conditions.append(''.join(str(t) for t in current_condition).strip())
-
-            sorted_conditions = []
-            current_group = []
-            last_connector = ""
-
-            if "OR" not in conditions:
-                for condition in conditions:
-                    if condition == 'AND':
-                        pass
-                    else:
-                        current_group.append(condition)
-
-                if current_group:
-                    sorted_conditions.extend(sorted(current_group))
-
-                return " AND ".join(sorted_conditions)
-            else:
-                return " ".join(conditions)
-
-                where_index = query.upper().find('WHERE')
-                normalized_query = query[:where_index + 5] + ' ' + ' '.join(sorted_conditions)
-
-    def process_where_xx(where, alias_map, baseDict):
-        where_tokens = []
-        for token in where.tokens:
-
-            if isinstance(token, Comparison):
-                left, operator, right = [t for t in token.tokens if not t.is_whitespace]
-                left = process_identifier(left, alias_map, baseDict)
-                right = process_identifier(right, alias_map, baseDict)
-                if left.lower() >= right.lower():
-                    left, right = right, left
-                    if operator.value == ">":
-                        operator = "<"
-                    elif operator.value == "<":
-                        operator = ">"
-
-                where_tokens.append(f"{left.lower()} {operator} {right.lower()}")
-            elif token.ttype is Keyword:
-                where_tokens.append(token.value.upper())
-            elif token.is_whitespace or (token.ttype is Keyword and token.value.upper() == "WHERE"):
-                continue
-            else:
-                where_tokens.append(str(token))
-        return " ".join(where_tokens)
-
-    # First pass to process FROM clause and populate alias_map
-    for token in parsed.tokens:
-        if token.is_whitespace:
-            continue
-        elif token.ttype is Keyword and token.value.upper() == 'FROM':
-            formatted_query.append('FROM')
-            pass
-        elif isinstance(token, Identifier) and formatted_query and formatted_query[-1] == 'FROM':
-            il = IdentifierList([token])
-            formatted_query.append(process_from(il, alias_map))
-            process_from(il, alias_map)
-        elif isinstance(token, IdentifierList) and formatted_query and formatted_query[-1] == 'FROM':
-            formatted_query.append(process_from(token, alias_map))
-            process_from(token, alias_map)
-
-    formatted_query = []
-
-    # Second pass to process SELECT and WHERE clauses
-    for token in parsed.tokens:
-        if token.is_whitespace:
-            continue
-        elif token.ttype is DML and token.value.upper() == 'SELECT':
-            formatted_query.append('SELECT')
-        elif token.ttype is Keyword and token.value.upper() == 'FROM':
-            formatted_query.append('FROM')  
-        elif token.ttype is Keyword and token.value.upper() == 'GROUP BY':
-            formatted_query.append('GROUP BY')
-        elif (isinstance(token, IdentifierList) or isinstance(token, Identifier)) and formatted_query and formatted_query[-1] == 'FROM':
-            formatted_query.append(process_from(token, alias_map))
-        elif (isinstance(token, IdentifierList) or isinstance(token, Identifier)) and formatted_query and formatted_query[-1] == 'GROUP BY':
-            formatted_query.append(process_groupby(token, alias_map))
-        elif isinstance(token, Where):
-            formatted_query.append('WHERE')
-            formatted_query.append(process_where(token, alias_map, baseDict))
-        elif formatted_query and formatted_query[-1] == 'SELECT' and (isinstance(token, IdentifierList) or isinstance(token, Function) or isinstance(token, Identifier)):
-            if isinstance(token, Function):
-                token = IdentifierList([token])
-            formatted_query.append(process_select(token, alias_map))
-        else:
-            formatted_query.append(str(token))
-
-    return " ".join(formatted_query)
-
-
-def findTableForColumn(data_dict, target_value, relevantTables):
-    l = []
-    for key, value_list in data_dict.items():
-        if key.lower() in relevantTables:
-            for sublist in value_list:
-                if sublist and sublist[0].lower() == target_value.lower():
-                    l.append(key)
-    if len(l) == 0:
-        for key, value_list in data_dict.items():
-            for sublist in value_list:
-                if sublist and sublist[0].lower() == target_value.lower():
-                    l.append(key)
-    return l
-
-
-def getTableScheme(table_name: str, tableDict: dict):
-    tab = tableDict[table_name]
-
-    # Format the schema
-    schema = "(" + ",".join([f"{col[0]}:{col[1]}" for col in tab]) + ")"
-    return schema
-
-def getCosetteKeyFromFile():
-    try:
-        with open("cosette_apikey.txt", "r") as f:
-            return f.read()
-    except FileNotFoundError:
-        return "NOKEY"
-
-def buildAndSendCosetteRequest(baseDict, sql, sol):
-
-    err = ""
-    for i in range(2):
-        try:
-            apiKey=getCosetteKeyFromFile()
-
-
-            schema = ""
-            for tab in baseDict.keys():
-                schema += f"schema sch{tab}{getTableScheme(tab, baseDict)};\n"
-            for tab in baseDict.keys():
-                schema += f"table {tab}(sch{tab});\n"
-
-            q1 = "query q1\n`"+sql+"`;\n"
-            q2 = "query q2\n`"+sol+"`;\n"
-
-            cosette = "-- random Kommentar\n" + schema + q1 + q2 + "verify q1 q2;\n"
-            print(cosette)
-
-            r = requests.post("https://demo.cosette.cs.washington.edu/solve",data={"api_key": apiKey, "query": cosette}, verify=False)
-
-            print(r.text)
-            return (r.json()['result'],r.text)
-            #return r.json()['result']
-
-        except Exception as e:
-            err = str(e)
-    return ("ERR", err)
-
-
-def checkColumns(sqlPath, solPath):
-    bd = Ba.getTableDict()
-    sql = normalizeSQLQuery(Ba.getSQLFromFile(sqlPath), bd)
-    sol = normalizeSQLQuery(Ba.getSQLFromFile(solPath), bd)
-
-    if("SELECT" in sql and "FROM" in sql):
-        start = str.find(sql, "SELECT")
-        end = str.find(sql, "FROM")
-        submission = str.strip(sql[start:end])
-        print("'"+submission+"'")
-
-        start = str.find(sol, "SELECT")
-        end = str.find(sol, "FROM")
-        solution = str.strip(sol[start:end])
-        print("'"+solution+"'")
-
-        if submission == solution:
-            return ""
-    return "Ausgegebene Spalten sind nicht korrekt (oder nicht automatisch überprüfbar)."
-
-
-def checkTables(sqlPath, solPath):
-    bd = Ba.getTableDict()
-    sql = normalizeSQLQuery(Ba.getSQLFromFile(sqlPath), bd)
-    sol = normalizeSQLQuery(Ba.getSQLFromFile(solPath), bd)
-
-    if("SELECT" in sql and "FROM" in sql):
-        endFromKeywords = ["WHERE", "GROUP", "ORDER", "LIMIT", ";"]
-
-        start = str.find(sql, "FROM")
-        end = -1
-
-
-        for keyword in endFromKeywords:
-            if(str.find(sql, keyword) != -1):
-                end = str.find(sql, keyword)
-                break
-        if(end == -1):
-            end = len(sql)
-
-        submission = str.strip(sql[start:end])
-        print("'"+submission+"'")
-
-        start = str.find(sol, "FROM")
-        end = -1
-        
-        for keyword in endFromKeywords:
-            if(str.find(sol, keyword) != -1):
-                end = str.find(sol, keyword)
-                break
-        if(end == -1):
-            end = len(sol)
-
-        solution = str.strip(sol[start:end])
-        print("'"+solution+"'")
-
-        if submission == solution:
-            return ""
-    return "Verwendete Tabellen sind nicht korrekt (oder nicht automatisch überprüfbar)."
-
-
-def checkEquality(sqlPath, solPath):
-    bd = Ba.getTableDict()
-    sql = normalizeSQLQuery(Ba.getSQLFromFile(sqlPath), bd)
-    sol = normalizeSQLQuery(Ba.getSQLFromFile(solPath), bd)
-    if(sql=='' or sol==''):
-        return "\n\nSQL-Datei ist leer. Aufgabe wurde noch nicht bearbeitet."
-
-    if(sql==sol):
-        return ""
-
-    result = buildAndSendCosetteRequest(bd, sql, sol)
-
-    if(result[0] == "ERR"):
-        return "\n\nFehler bei der automatischen Überprüfung der Abgabe. Es kann keine Aussage über die Korrektheit der Abgabe getroffen werden."
-    elif(result[0] != "EQ"):
-        return "\n\nDie Abgabe stimmt nicht mit der Musterlösung überein."
-    return ""
-

sql_testing_tools-0.1.4/sql_testing_tools.egg-info/PKG-INFO

@@ -1,10 +0,0 @@
-Metadata-Version: 2.1
-Name: sql_testing_tools
-Version: 0.1.4
-Description-Content-Type: text/markdown
-Requires-Dist: sqlparse>=0.5.1
-Requires-Dist: requests>=2.32.3
-
-[Beta in Development]
-
-A library that normalizes simple SQL queries and compares them first by equality of the normalized string and then using the cosette API.