streamlit-octostar-utils 0.1.7a8__py3-none-any.whl → 0.1.7a10__py3-none-any.whl

streamlit_octostar_utils/api_crafter/celery.py

@@ -493,18 +493,9 @@ class CeleryExecutor(object):
                 task_id,
             )
 
-            try:
-                await asyncio.get_running_loop().run_in_executor(
-                    self.set_thread_pool, _send_task, task_fn, task_id, options
-                )
-
-            except CeleryExecutor.QueueFullException as e:
-                if os.path.isfile(os.path.join(self.in_folder, task_id)):
-                    with RedisFileLock(self.redis_client, os.path.join(self.in_folder, task_id)):
-                        os.remove(os.path.join(self.in_folder, task_id))
-
-                logger.warning(f"Queue full exception: {str(e)}")
-                return None
+            await asyncio.get_running_loop().run_in_executor(
+                self.set_thread_pool, _send_task, task_fn, task_id, options
+            )
 
         except asyncio.CancelledError:
             logger.info(f"Cancelling task {task_id} due to disconnect!")
@@ -694,20 +685,26 @@ class CeleryErrorRoute(DefaultErrorRoute):
     DEFAULT_SILENCED_EXCEPTIONS = {CeleryExecutor.QueueFullException: lambda exc: True}
 
     def add_default_exceptions_handler(
-        fs_app,
-        debug=False,
-        excs_to_status_codes=None,
-        silenced_excs=None,
-        log_filter=None,
+            fs_app,
+            debug=False,
+            excs_to_status_codes=None,
+            silenced_excs=None,
     ):
-        if excs_to_status_codes is None:
-            excs_to_status_codes = {
-                **DefaultErrorRoute.DEFAULT_STATUS_CODE_MAPPINGS,
-                **CeleryErrorRoute.DEFAULT_STATUS_CODE_MAPPINGS,
-            }
-        if silenced_excs is None:
-            silenced_excs = {
-                **DefaultErrorRoute.DEFAULT_SILENCED_EXCEPTIONS,
-                **CeleryErrorRoute.DEFAULT_SILENCED_EXCEPTIONS,
-            }
-        DefaultErrorRoute.add_default_exceptions_handler(fs_app, debug, excs_to_status_codes)
+        extra_status = {CeleryExecutor.QueueFullException: lambda exc: 429}
+        extra_silence = {CeleryExecutor.QueueFullException: lambda exc: True}
+
+        status_codes = {
+            **DefaultErrorRoute.DEFAULT_STATUS_CODE_MAPPINGS,
+            **(excs_to_status_codes or {}),
+            **extra_status,
+        }
+
+        silenced = {
+            **DefaultErrorRoute.DEFAULT_SILENCED_EXCEPTIONS,
+            **(silenced_excs or {}),
+            **extra_silence,
+        }
+
+        super(CeleryErrorRoute, CeleryErrorRoute).add_default_exceptions_handler(
+            fs_app, debug, status_codes, silenced
+        )

streamlit_octostar_utils/api_crafter/fastapi.py

@@ -241,20 +241,30 @@ class CommonModels(object):
             headers["Content-Disposition"] += f' filename="{filename}"'
             return StreamingResponse(zip_buffer, media_type="application/zip", headers=headers)
 
+
 class ErrorLogFilter(logging.Filter):
-    def __init__(self, silenced_excs: dict):
+    def __init__(self, silenced_excs: dict[type, callable]):
         super().__init__()
         self.silenced_excs = silenced_excs
-        
-    def filter(self, record):
-        exc_info = getattr(record, 'exc_info', None)
-        if exc_info:
-            if record.exc_info:
-                exc_type, exc_value, exc_tb = record.exc_info
-                if exc_value:
-                    for etype, cond in self.silenced_excs.items():
-                        if isinstance(exc_value, etype) and cond(exc_value):
-                            return False
+
+    def _silenced(self, exc_value) -> bool:
+        for etype, cond in self.silenced_excs.items():
+            if isinstance(exc_value, etype) and cond(exc_value):
+                return True
+        return False
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        if record.exc_info:
+            exc_type, exc_value, _ = record.exc_info
+            if self._silenced(exc_value):
+                return False
+
+        msg = record.getMessage()
+        for etype in self.silenced_excs:
+            if etype.__name__ in msg:
+                if self.silenced_excs[etype](None):
+                    return False
+
         return True
 
 
@@ -304,7 +314,7 @@ class DefaultErrorRoute:
         },
     }
 
-    def format_error(exc, body=bytes(), debug=False, excs_to_status_codes=DEFAULT_STATUS_CODE_MAPPINGS):
+    def format_error(exc, body=b"", debug=False, excs_to_status_codes=DEFAULT_STATUS_CODE_MAPPINGS):
         """Generic Error Handler"""
         status_code = 500
         for exc_type, handler in excs_to_status_codes.items():
@@ -334,10 +344,10 @@ class DefaultErrorRoute:
         return DefaultErrorRoute.format_error(exc, body, debug, excs_to_status_codes)
 
     def add_default_exceptions_handler(
-        fs_app,
-        debug=False,
-        excs_to_status_codes=None,
-        silenced_excs=None,
+            fs_app,
+            debug=False,
+            excs_to_status_codes=None,
+            silenced_excs=None,
     ):
         if excs_to_status_codes is None:
             excs_to_status_codes = DefaultErrorRoute.DEFAULT_STATUS_CODE_MAPPINGS
@@ -345,17 +355,18 @@ class DefaultErrorRoute:
             silenced_excs = DefaultErrorRoute.DEFAULT_SILENCED_EXCEPTIONS
 
         async def _async_handle_error(request: Request, exc: Exception):
-            return await DefaultErrorRoute.handle_error(request, exc, debug, excs_to_status_codes)
+            return await DefaultErrorRoute.handle_error(b"", exc, debug, excs_to_status_codes)
 
         # Added all three since FastAPI seems to intercept some exceptions before Exception
         fs_app.add_exception_handler(RequestValidationError, _async_handle_error)
         fs_app.add_exception_handler(StarletteHTTPException, _async_handle_error)
         fs_app.add_exception_handler(Exception, _async_handle_error)
-        
+
         log_filter = ErrorLogFilter(silenced_excs)
-        logging.getLogger("uvicorn.error").addFilter(log_filter)
-        logging.getLogger("uvicorn").addFilter(log_filter)
-        logging.getLogger("fastapi").addFilter(log_filter)
+        for name in ("uvicorn.error", "uvicorn.access", "fastapi", "uvicorn"):
+            logging.getLogger(name).addFilter(log_filter)
+
+        logging.getLogger().addFilter(log_filter)
 
 class RequestCancelledMiddleware:
     def __init__(self, app):

streamlit_octostar_utils/api_crafter/parser/entities_parser.py

@@ -60,6 +60,10 @@ class EntitiesInvalidReason(Enum):
     INVALID_REL_TYPE = 20
     UNKNOWN_UUID = 21
     WRONG_SOURCE_OR_TARGET_TYPE = 22
+    
+    
+EntityDict = dict
+InvalidErrorDict = dict
 
 
 class EntitiesParser(object):
@@ -547,29 +551,42 @@ class EntitiesParser(object):
         self._get_ontology()
         if not self.ontology:
             raise RuntimeError("Ontology could not be fetched for validation!")
+        ontology = self.ontology
         from streamlit_octostar_utils.ontology.validation import (
             validate_and_format_timbr_type,
         )
         from streamlit_octostar_utils.ontology.inheritance import is_child_concept
 
         processed_entities = []
-        processsed_relationships = []
+        processed_relationships = []
         for entity in entities:
-            if entity.concept_name not in self.ontology["concepts"]:
-                processed_entities.append((EntitiesInvalidReason.INVALID_TYPE, entity))
-                continue
+            if entity.concept_name not in ontology["concepts"]:
+                processed_entities.append(
+                    (
+                        EntitiesInvalidReason.INVALID_TYPE,
+                        entity,
+                        {"error_message": f"{entity} has invalid concept name"},
+                    )
+                )
             entity_properties = [
-                k for k in entity.fields.keys() if not k.startswith("#")
+                k
+                for k in entity.fields.keys()
+                if not k.startswith("#") and k != "extra_fields"
             ]
             ontology_properties = {
                 e["property_name"]: e
-                for e in self.ontology["concepts"][entity.concept_name]["allProperties"]
+                for e in ontology["concepts"][entity.concept_name]["allProperties"]
             }
             if any(prop not in ontology_properties for prop in entity_properties):
                 processed_entities.append(
-                    (EntitiesInvalidReason.INVALID_PROPERTIES, entity)
+                    (
+                        EntitiesInvalidReason.INVALID_PROPERTIES,
+                        entity,
+                        {
+                            "error_message": f"{entity} has invalid entity properties: {str(set(entity_properties) - set(ontology_properties))}"
+                        },
+                    )
                 )
-                continue
             for prop in entity_properties:
                 try:
                     entity.fields[prop] = validate_and_format_timbr_type(
@@ -577,46 +594,66 @@ class EntitiesParser(object):
                     )
                 except:
                     processed_entities.append(
-                        (EntitiesInvalidReason.INVALID_PROPERTIES, entity)
+                        (
+                            EntitiesInvalidReason.INVALID_PROPERTIES,
+                            entity,
+                            {
+                                "error_message": f"{entity} has an invalid property type for property {prop}"
+                            },
+                        )
                     )
-                continue
-            processed_entities.append((EntitiesInvalidReason.VALID, entity))
+            processed_entities.append((EntitiesInvalidReason.VALID, entity, {}))
         ontology_relationships = {
-            r["relationship_name"]: r for r in self.ontology["relationships"]
+            r["relationship_name"]: r for r in ontology["relationships"]
         }
         for rel in relationships:
             if rel.name not in ontology_relationships.keys():
-                processsed_relationships.append(
-                    (EntitiesInvalidReason.INVALID_REL_TYPE, rel)
+                processed_relationships.append(
+                    (
+                        EntitiesInvalidReason.INVALID_REL_TYPE,
+                        rel,
+                        {"error_message": f"{rel} relationship name is invalid"},
+                    )
                 )
-                continue
-            entities_by_id = {e.unique_id: e for e in entities}
+            entities_by_id = {e[1].unique_id: e[1] for e in processed_entities}
             if (
                 rel.concept_from not in entities_by_id.keys()
                 or rel.concept_to not in entities_by_id.keys()
             ):
-                processsed_relationships.append(
-                    (EntitiesInvalidReason.UNKNOWN_UUID, rel)
+                processed_relationships.append(
+                    (
+                        EntitiesInvalidReason.UNKNOWN_UUID,
+                        rel,
+                        {
+                            "error_message": f"for {rel}, source concept or target concept are invalid"
+                        },
+                    )
                 )
-                continue
             ontology_relationship = ontology_relationships[rel.name]
             rel_concept_types = (
                 entities_by_id[rel.concept_from].concept_name,
                 entities_by_id[rel.concept_to].concept_name,
             )
             if not is_child_concept(
-                rel_concept_types[0], ontology_relationship["concept"], self.ontology
+                rel_concept_types[0],
+                ontology_relationship["concept"],
+                ontology,
             ) or not is_child_concept(
                 rel_concept_types[1],
                 ontology_relationship["target_concept"],
-                self.ontology,
+                ontology,
             ):
-                processsed_relationships.append(
-                    (EntitiesInvalidReason.WRONG_SOURCE_OR_TARGET_TYPE, rel)
+                processed_relationships.append(
+                    (
+                        EntitiesInvalidReason.WRONG_SOURCE_OR_TARGET_TYPE,
+                        rel,
+                        {
+                            "error_message": f"for {rel}, type for source concept or for target concept is invalid"
+                        },
+                    )
                 )
-                continue
-            processsed_relationships.append((EntitiesInvalidReason.VALID, rel))
-        return processed_entities, processsed_relationships
+            processed_relationships.append((EntitiesInvalidReason.VALID, rel, {}))
+        return processed_entities, processed_relationships
 
     def replace_unique_id(self, id_from, id_to, entities, relationships):
         for entity in entities:

{streamlit_octostar_utils-0.1.7a8.dist-info → streamlit_octostar_utils-0.1.7a10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: streamlit-octostar-utils
-Version: 0.1.7a8
+Version: 0.1.7a10
 Summary: 
 License: MIT
 Author: Octostar

{streamlit_octostar_utils-0.1.7a8.dist-info → streamlit_octostar_utils-0.1.7a10.dist-info}/RECORD

@@ -1,11 +1,11 @@
 streamlit_octostar_utils/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 streamlit_octostar_utils/api_crafter/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-streamlit_octostar_utils/api_crafter/celery.py,sha256=3vJ7gDf_WyGrXquuxHNA9ybw1ljRAB_l3wJDNVUZs_0,30213
-streamlit_octostar_utils/api_crafter/fastapi.py,sha256=s4WayFitfS2fX_FGtv1s56RmLB9rx7cfwuFTZjzrKoY,14133
+streamlit_octostar_utils/api_crafter/celery.py,sha256=bQq8P95j0XDWqx6Hdnm96PJL8kcp9ZcSKjveZUCQTVk,29874
+streamlit_octostar_utils/api_crafter/fastapi.py,sha256=2bktT5Mwjs9XixWcOqUKMoLM_cgKl-cqZDUa2Imf4xA,14357
 streamlit_octostar_utils/api_crafter/nifi.py,sha256=fQ5k9eZl2oSQZ2BexZYwKUiO05-FryUi7wnCd_56P9Y,44375
 streamlit_octostar_utils/api_crafter/parser/__init__.py,sha256=YeYWF6sdQiCFV_RKNW2t9Vs6KJExE2pbXxWTe_DOayY,107
 streamlit_octostar_utils/api_crafter/parser/combine_fields.py,sha256=ddc44xkajw8MU0peAX_263DL7rPXbTKbHUjpOhRgvyU,8790
-streamlit_octostar_utils/api_crafter/parser/entities_parser.py,sha256=7mUxwcOcaIhDx3GnJ42p5L0PUfAiEvE_fkopNIujlL8,30067
+streamlit_octostar_utils/api_crafter/parser/entities_parser.py,sha256=Z740IViQ8WX3xBzgWzsO6JHUps6GyNLJE8nlpt3KTzc,31418
 streamlit_octostar_utils/api_crafter/parser/generics.py,sha256=GefvTUWkqsICS_TdXA2p73_Uzh6041g88mWql3Fr9cY,1861
 streamlit_octostar_utils/api_crafter/parser/info.py,sha256=L2cdtsmfIdFSkAnIXhGhQZG6YMZRtY1GgLXn-847avA,649
 streamlit_octostar_utils/api_crafter/parser/linkchart_functions.py,sha256=WfHl-EfH4SWjpk8civS7HL11eIl4SAV5Dtwe8mTidrc,7163
@@ -37,7 +37,7 @@ streamlit_octostar_utils/threading/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzp
 streamlit_octostar_utils/threading/async_task_manager.py,sha256=q7N6YZwUvIYMzkSHmsJNheNVCv93c03H6Hyg9uH8pvk,4747
 streamlit_octostar_utils/threading/session_callback_manager.py,sha256=LvZVP4g6tvKtYmI13f2j1sX_7hm61Groqp5xJine9_k,3973
 streamlit_octostar_utils/threading/session_state_hot_swapper.py,sha256=6eeCQI6A42hp4DmW2NQw2rbeR-k9N8DhfBKQdN_fbLU,811
-streamlit_octostar_utils-0.1.7a8.dist-info/LICENSE,sha256=dkwVPyV03fPHHtERnF6RnvRXcll__tud9gWca1RcgnQ,1073
-streamlit_octostar_utils-0.1.7a8.dist-info/METADATA,sha256=Qu54yUOzUMNjTN-JEZLeQa59Ibb2UNLZ1j3bj5XPT84,2267
-streamlit_octostar_utils-0.1.7a8.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-streamlit_octostar_utils-0.1.7a8.dist-info/RECORD,,
+streamlit_octostar_utils-0.1.7a10.dist-info/LICENSE,sha256=dkwVPyV03fPHHtERnF6RnvRXcll__tud9gWca1RcgnQ,1073
+streamlit_octostar_utils-0.1.7a10.dist-info/METADATA,sha256=s86wH6OAhUfZBkiM2dIFUcw1x7g3COOUjBkWSYnmJGc,2268
+streamlit_octostar_utils-0.1.7a10.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+streamlit_octostar_utils-0.1.7a10.dist-info/RECORD,,