jaaql-middleware-python 4.29.0__tar.gz → 4.30.0__tar.gz

{jaaql-middleware-python-4.29.0/jaaql_middleware_python.egg-info → jaaql-middleware-python-4.30.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: jaaql-middleware-python
-Version: 4.29.0
+Version: 4.30.0
 Summary: The jaaql package, allowing for rapid development and deployment of RESTful HTTP applications
 Home-page: https://github.com/JAAQL/JAAQL-middleware-python
 Author: Software Quality Measurement and Improvement bv

{jaaql-middleware-python-4.29.0 → jaaql-middleware-python-4.30.0}/jaaql/constants.py

@@ -183,5 +183,5 @@ ROLE__postgres = "postgres"
 
 PROTOCOL__postgres = "postgresql://"
 
-VERSION = "4.29.0"
+VERSION = "4.30.0"
 

{jaaql-middleware-python-4.29.0 → jaaql-middleware-python-4.30.0}/jaaql/db/db_utils_no_circ.py

@@ -1,5 +1,3 @@
-import os
-
 from jaaql.mvc.exception_queries import QUERY__fetch_application_schemas, KG__application_schema__application, KG__application__is_live, \
     KG__application_schema__name, KEY__is_default
 from queue import Queue
@@ -17,22 +15,24 @@ def get_jaaql_connection_to_db(vault, config, database: str, jaaql_connection: D
     return create_interface_for_db(vault, config, jaaql_connection.role, database)
 
 
-def get_required_db(vault, config, jaaql_connection: DBInterface, inputs: dict, account_id: str, conn=None, interface: DBInterface = None):
+def get_required_db(vault, config, jaaql_connection: DBInterface, inputs: dict, account_id: str, conn=None, interface: DBInterface = None, db_cache=None):
     if not isinstance(inputs, dict):
         raise HttpStatusException("Expected object or string input")
 
     if conn is None:
         if KEY__application in inputs:
-            schemas = execute_supplied_statement(jaaql_connection, QUERY__fetch_application_schemas, {
-                KG__application_schema__application: inputs[KEY__application]
-            }, as_objects=True)
-            if len(schemas) == 0:
-                application__select(jaaql_connection, inputs[KEY__application],
-                                    singleton_message=f"Application '{inputs[KEY__application]}' does not exist. Are you sure you have installed it?")
-                raise HttpStatusException("Application has no schemas!")
-            if not schemas[0][KG__application__is_live]:
-                raise HttpStatusException("Application is currently being deployed. Please wait a few minutes until deployment is complete")
-            schemas = {itm[KG__application_schema__name]: itm for itm in schemas}
+            schemas = db_cache
+            if schemas is None:
+                schemas = execute_supplied_statement(jaaql_connection, QUERY__fetch_application_schemas, {
+                    KG__application_schema__application: inputs[KEY__application]
+                }, as_objects=True)
+                if len(schemas) == 0:
+                    application__select(jaaql_connection, inputs[KEY__application],
+                                        singleton_message=f"Application '{inputs[KEY__application]}' does not exist. Are you sure you have installed it?")
+                    raise HttpStatusException("Application has no schemas!")
+                if not schemas[0][KG__application__is_live]:
+                    raise HttpStatusException("Application is currently being deployed. Please wait a few minutes until deployment is complete")
+                schemas = {itm[KG__application_schema__name]: itm for itm in schemas}
 
             found_db = None
             if KEY__schema in inputs and inputs[KEY__schema] is not None:
@@ -68,11 +68,11 @@ def get_required_db(vault, config, jaaql_connection: DBInterface, inputs: dict,
 
 def submit(vault, config, db_crypt_key, jaaql_connection: DBInterface, inputs: dict, account_id: str, verification_hook: Queue = None,
            cached_canned_query_service=None, as_objects: bool = False, singleton: bool = False, keep_alive_conn: bool = False,
-           conn=None, interface: DBInterface = None):
+           conn=None, interface: DBInterface = None, db_cache=None):
     if not isinstance(inputs, dict):
         raise HttpStatusException("Expected object or string input")
 
-    required_db = get_required_db(vault, config, jaaql_connection, inputs, account_id, conn, interface)
+    required_db = get_required_db(vault, config, jaaql_connection, inputs, account_id, conn, interface, db_cache=db_cache)
 
     prevent_unused = inputs.pop(KEY__prevent_unused_parameters) if KEY__prevent_unused_parameters in inputs else True
 

{jaaql-middleware-python-4.29.0 → jaaql-middleware-python-4.30.0}/jaaql/documentation/documentation_internal.py

@@ -260,6 +260,16 @@ DOCUMENTATION__last_successful_build_timestamp = SwaggerDocumentation(
     ]
 )
 
+DOCUMENTATION__flush_cache = SwaggerDocumentation(
+    tags="Cache",
+    security=False,
+    methods=SwaggerMethod(
+        name="Flush JAAQL Cache",
+        description="Triggers a cache flush",
+        method=REST__GET,
+    )
+)
+
 DOCUMENTATION__cron = SwaggerDocumentation(
     tags="Cron",
     methods=SwaggerMethod(

{jaaql-middleware-python-4.29.0 → jaaql-middleware-python-4.30.0}/jaaql/documentation/documentation_shared.py

@@ -169,3 +169,27 @@ DOCUMENTATION__submit = SwaggerDocumentation(
         parallel_verification=True
     )
 )
+
+DOCUMENTATION__execute = SwaggerDocumentation(
+    tags="Queries",
+    methods=SwaggerMethod(
+        name="Execute JAAQL pre-prepared query",
+        description="Executes a query from a list of pre-prepared queries. Returns results",
+        method=REST__POST,
+        arguments=ARG_RESP__allow_all,
+        response=RES__allow_all,
+        parallel_verification=True
+    )
+)
+
+DOCUMENTATION__call_proc = SwaggerDocumentation(
+    tags="Queries",
+    methods=SwaggerMethod(
+        name="Execute JAAQL database procedure",
+        description="Executes a procedure, returns results",
+        method=REST__POST,
+        arguments=ARG_RESP__allow_all,
+        response=RES__allow_all,
+        parallel_verification=True
+    )
+)

{jaaql-middleware-python-4.29.0 → jaaql-middleware-python-4.30.0}/jaaql/mvc/base_model.py

@@ -1,10 +1,13 @@
 import sys
 
 import json
+import base64
 import requests
+import traceback
 
 from jwcrypto import jwk
 from cryptography import x509
+
 from jaaql.utilities.vault import Vault, DIR__vault
 from jaaql.db.db_interface import DBInterface
 import jaaql.utilities.crypt_utils as crypt_utils
@@ -181,6 +184,21 @@ class BaseJAAQLModel:
 
         return ret
 
+    def reload_cache(self):
+        print("Received cache flush instruction")
+        if os.path.exists("/queries/queries.json"):
+            try:
+                self.query_caches = json.loads(open("/queries/queries.json", "r").read())
+                for key, encoded_list in self.query_caches["queries"].items():
+                    # Replace each encoded string in the list with its decoded version.
+                    self.query_caches["queries"][key] = [
+                        base64.b64decode(encoded).decode('utf-8') for encoded in encoded_list
+                    ]
+                self.db_cache = 1
+                print("Loaded query cache")
+            except:
+                traceback.print_exc()
+
     def set_jaaql_lookup_connection(self):
         if self.vault.has_obj(VAULT_KEY__jaaql_lookup_connection):
             jaaql_uri = self.vault.get_obj(VAULT_KEY__jaaql_lookup_connection)
@@ -196,6 +214,10 @@ class BaseJAAQLModel:
         self.migration_folder = migration_folder
         self.is_container = is_container
 
+        self.query_caches = {}
+        self.db_cache = None
+
+        self.prevent_arbitrary_queries = os.environ.get("PREVENT_ARBITRARY_QUERIES", "false") == "true"
         self.is_https = os.environ.get("IS_HTTPS", "false").lower().strip() == "true"
         self.vigilant_sessions = os.environ.get("VIGILANT_SESSIONS", "false").lower().strip() == "true"
 

{jaaql-middleware-python-4.29.0 → jaaql-middleware-python-4.30.0}/jaaql/mvc/controller.py

@@ -94,8 +94,16 @@ class JAAQLController(BaseJAAQLController):
             return self.model.add_my_account_password(account_id, username, ip_address, is_the_anonymous_user, **http_inputs)
 
         @self.publish_route('/submit', DOCUMENTATION__submit)
-        def submit(http_inputs: dict, account_id: str, verification_hook: queue.Queue):
-            return self.model.submit(http_inputs, account_id, verification_hook=verification_hook)
+        def submit(http_inputs: dict, account_id: str, verification_hook: queue.Queue, ip_address: str):
+            return self.model.submit(http_inputs, account_id, verification_hook=verification_hook, ip_address=ip_address)
+
+        @self.publish_route('/execute', DOCUMENTATION__execute)
+        def execute(http_inputs: dict, account_id: str, verification_hook: queue.Queue):
+            return self.model.execute(http_inputs, account_id, verification_hook=verification_hook)
+
+        @self.publish_route('/call-proc', DOCUMENTATION__execute)
+        def call_proc(http_inputs: dict, account_id: str, verification_hook: queue.Queue):
+            return self.model.call_proc(http_inputs, account_id, verification_hook=verification_hook)
 
         @self.publish_route('/internal/clean', DOCUMENTATION__clean)
         def clean(connection: DBInterface):
@@ -137,12 +145,16 @@ class JAAQLController(BaseJAAQLController):
             self.model.add_cron_job_to_application(connection, http_inputs)
 
         @self.publish_route('/build-time', DOCUMENTATION__last_successful_build_timestamp)
-        def cron(http_inputs: dict):
+        def build_time(http_inputs: dict, ip_address: str):
             if self.is_post():
-                self.model.set_last_successful_build_time(http_inputs)
+                self.model.set_last_successful_build_time(http_inputs, ip_address)
             else:
                 return self.model.get_last_successful_build_time()
 
+        @self.publish_route('/flush-cache', DOCUMENTATION__flush_cache)
+        def flush_cache(ip_address: str):
+            self.model.flush_cache(ip_address)
+
         @self.publish_route('/webhook/<application>/<name>', DOCUMENTATION__webhook)
         def handle_webhook(application: str, name: str, body: bytes, headers: dict, args: dict, response: JAAQLResponse):
             self.model.handle_webhook(application, name, body, headers, args, response)

{jaaql-middleware-python-4.29.0 → jaaql-middleware-python-4.30.0}/jaaql/mvc/model.py

@@ -5,6 +5,7 @@ import tempfile
 import traceback
 import urllib.parse
 import uuid
+import signal
 import secrets
 from cryptography.hazmat.primitives import hashes
 
@@ -1846,10 +1847,79 @@ WHERE
         else:
             return 0
 
-    def set_last_successful_build_time(self, http_inputs: dict):
+    def set_last_successful_build_time(self, http_inputs: dict, ip_address: str):
+        if ip_address not in IPS__local:
+            raise UserUnauthorized()
+
         if os.environ.get("JAAQL_DEBUGGING") == "TRUE":
             jaaql__update(self.jaaql_lookup_connection, last_successful_build_time=http_inputs[KG__jaaql__last_successful_build_time])
 
-    def submit(self, inputs: dict, account_id: str, verification_hook: Queue = None, as_objects: bool = False, singleton: bool = False):
+    def flush_cache(self, ip_address: str):
+        if ip_address not in IPS__local:
+            raise UserUnauthorized()
+
+        os.kill(int(open("app.pid", "r").read()), signal.SIGUSR1)
+
+    def execute(self, inputs: dict, account_id: str, verification_hook: Queue = None, as_objects: bool = False, singleton: bool = False):
+        if self.db_cache == 1:
+            schemas = execute_supplied_statement(self.jaaql_lookup_connection, QUERY__fetch_application_schemas, {
+                KG__application_schema__application: self.query_caches["application"]
+            }, as_objects=True)
+            if len(schemas) == 0:
+                print("Couldn't find application " + self.query_caches["application"])
+                raise Exception()
+            self.db_cache = {itm[KG__application_schema__name]: itm for itm in schemas}
+            print("Loaded DB Cache")
+
+        for key, val in inputs["query"].items():
+            if isinstance(val, dict):
+                trimmed = val["query"].strip()
+                val["query"] = self.query_caches["queries"][trimmed.split(":")[0]][int(trimmed.split(":")[1])]
+            else:
+                trimmed = val.strip()
+                inputs["query"][key] = self.query_caches["queries"][trimmed.split(":")[0]][int(trimmed.split(":")[1])]
+
+        return submit(self.vault, self.config, self.get_db_crypt_key(), self.jaaql_lookup_connection, inputs, account_id, verification_hook,
+                      self.cached_canned_query_service, as_objects=as_objects, singleton=singleton, db_cache=self.db_cache)
+
+    def call_proc(self, inputs: dict, account_id: str, verification_hook: Queue = None, as_objects: bool = False, singleton: bool = False):
+        parameters = inputs["parameters"]
+        query = "SELECT * FROM \"" + inputs["query"] + "\"("
+
+        parameter_keys = list(parameters.keys())
+        introduced = False
+        explicit_types = inputs.get("explicit_types", {})
+
+        absent = ""
+
+        for parameter_key in parameter_keys:
+            if re.match(REGEX__dmbs_object_name, parameter_key) is None:
+                raise HttpStatusException("Unsafe parameter key " + parameter_key)
+            explicit_type = explicit_types.get(parameter_key)
+            parameter_value = parameters[parameter_key]
+
+            if introduced:
+                query += ","
+
+            if explicit_type and parameter_value is not None and explicit_type[0] != "_":
+                query += f"\n\t{parameter_key} => :{parameter_key}::{explicit_type}"
+            else:
+                query += f"\n\t{parameter_key} => :{parameter_key}{absent}"
+
+            introduced = True
+
+        query += " )"
+
+        inputs["query"] = {
+            "_jaaql_procedure": query
+        }
+
+        return submit(self.vault, self.config, self.get_db_crypt_key(), self.jaaql_lookup_connection, inputs, account_id, verification_hook,
+                      self.cached_canned_query_service, as_objects=as_objects, singleton=singleton, db_cache=self.db_cache)
+
+    def submit(self, inputs: dict, account_id: str, verification_hook: Queue = None, as_objects: bool = False, singleton: bool = False, ip_address: str = None):
+        if ip_address not in IPS__local and self.prevent_arbitrary_queries:
+            raise UnhandledJaaqlServerError("Not allowed to send queries to server!")
+
         return submit(self.vault, self.config, self.get_db_crypt_key(), self.jaaql_lookup_connection, inputs, account_id, verification_hook,
                       self.cached_canned_query_service, as_objects=as_objects, singleton=singleton)

{jaaql-middleware-python-4.29.0 → jaaql-middleware-python-4.30.0/jaaql_middleware_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: jaaql-middleware-python
-Version: 4.29.0
+Version: 4.30.0
 Summary: The jaaql package, allowing for rapid development and deployment of RESTful HTTP applications
 Home-page: https://github.com/JAAQL/JAAQL-middleware-python
 Author: Software Quality Measurement and Improvement bv