ul-api-utils 7.10.3__tar.gz → 7.10.5__tar.gz

{ul-api-utils-7.10.3 → ul-api-utils-7.10.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ul-api-utils
-Version: 7.10.3
+Version: 7.10.5
 Summary: Python api utils
 Author: Unic-lab
 Author-email: 

{ul-api-utils-7.10.3 → ul-api-utils-7.10.5}/example/conf.py

@@ -4,20 +4,18 @@ from ul_db_utils.modules.postgres_modules.db import DbConfig
 
 from ul_api_utils.modules.api_sdk import ApiSdk
 from ul_api_utils.modules.api_sdk_config import ApiSdkConfig, ApiSdkIdentifyTypeEnum, \
-    ApiSdkFlaskDebuggingPluginsEnabled, SocketIOConfig
+    ApiSdkFlaskDebuggingPluginsEnabled
 from example.permissions import permissions
+from ul_api_utils.resources.socketio import SocketIOConfigType, SocketIOConfig
+
 
 sdk = ApiSdk(ApiSdkConfig(
-    server_socketio_config=SocketIOConfig(
+    socket_config=SocketIOConfig(
+        app_type=SocketIOConfigType.EXTERNAL_PROCESS,
         message_queue='redis://localhost:16379',
         logs_enabled=True,
         engineio_logs_enabled=False,
     ),
-    # client_socketio_config=SocketIOConfig(
-    #     message_queue='redis://localhost:16379',
-    #     logs_enabled=True,
-    #     engineio_logs_enabled=False,
-    # ),
     service_name='example_service',
     permissions=permissions,
     cache_storage_uri='redis://localhost:16379',

{ul-api-utils-7.10.3 → ul-api-utils-7.10.5}/example/main.py

@@ -1,6 +1,8 @@
 # from gevent import monkey
 #
 # monkey.patch_all()
+
+
 # UNCOMMENT THIS IN CASE YOU'RE TESTING GEVENT WORKERS WITH SOCKET.IO
 
 import os

{ul-api-utils-7.10.3 → ul-api-utils-7.10.5}/example/sockets/on_message.py

@@ -1,9 +1,13 @@
+import logging
 from typing import Any
 
 from example.conf import sdk
 
 
+logger = logging.getLogger(__name__)
+
+
 @sdk.socket.on('message')
 def handle_message(data: Any) -> None:
-    print('received message: ')
-    print(data)
+    logger.info('received message: ')
+    logger.info(data)

{ul-api-utils-7.10.3 → ul-api-utils-7.10.5}/example/workers/worker.py

@@ -5,8 +5,16 @@ from unipipeline.worker.uni_worker_consumer_message import UniWorkerConsumerMess
 from ul_api_utils.modules.worker_context import WorkerContext
 from ul_api_utils.modules.worker_sdk import WorkerSdk
 from ul_api_utils.modules.worker_sdk_config import WorkerSdkConfig
+from ul_api_utils.resources.socketio import SocketIOConfig, SocketIOConfigType
 
-initialized_worker = WorkerSdk(WorkerSdkConfig())
+initialized_worker = WorkerSdk(WorkerSdkConfig(
+    socket_config=SocketIOConfig(
+        app_type=SocketIOConfigType.EXTERNAL_PROCESS,
+        message_queue='redis://localhost:16379',
+        logs_enabled=True,
+        engineio_logs_enabled=False,
+    ),
+))
 
 
 class Msg(UniMessage):

{ul-api-utils-7.10.3 → ul-api-utils-7.10.5}/setup.py

@@ -9,7 +9,7 @@ with open(path.join(HERE, 'README.md'), encoding='utf-8') as f:
 
 setup(
     name='ul-api-utils',
-    version='7.10.3',
+    version='7.10.5',
     description='Python api utils',
     author='Unic-lab',
     long_description=long_description,
@@ -67,12 +67,12 @@ setup(
         "requests==2.28.1",
         "cryptography==38.0.1",
         "colored==1.4.3",
-        "flask_socketio==5.3.6",
+        "flask-socketio==5.3.6",
         "ormsgpack==1.4.1",
         "msgpack==1.0.4",
         "msgpack-types==0.2.0",
         "fastavro==1.7.0",
-        "factory_boy==3.3.0",
+        "factory-boy==3.3.0",
         "sentry-sdk[flask]==1.9.2",
         "faker==24.8.0",
         "types-requests==2.28.8",
@@ -89,7 +89,7 @@ setup(
         # "opentelemetry-instrumentation-requests==0.27b0",
         # "opentelemetry-exporter-jaeger==1.8.0",
         # "opentelemetry-instrumentation-sqlalchemy==0.27b0",
-        # "ul-db-utils==3.2.1",        # ACTUALIZE, BUT DO NOT UNCOMMENT PLEASE
+        # "ul-db-utils==3.2.2",        # ACTUALIZE, BUT DO NOT UNCOMMENT PLEASE
         # "ul-py-tool==1.15.42",        # ACTUALIZE, BUT DO NOT UNCOMMENT PLEASE
     ],
 )

{ul-api-utils-7.10.3 → ul-api-utils-7.10.5}/ul_api_utils/commands/cmd_start.py

@@ -4,6 +4,7 @@ import re
 import subprocess
 import sys
 from datetime import datetime
+from typing import Optional
 
 from ul_py_tool.commands.cmd import Cmd
 from ul_py_tool.utils.arg_str2bool import arg_str2bool
@@ -26,6 +27,8 @@ class CmdStart(Cmd):
     max_requests: int
     max_requests_jitter: int
     worker_class: str
+    statsd_endpoint: Optional[str]
+    statsd_prefix: Optional[str]
 
     @property
     def app_rel_dir(self) -> str:
@@ -51,6 +54,8 @@ class CmdStart(Cmd):
         parser.add_argument('--max-requests', dest='max_requests', type=int, default=1000, required=False)
         parser.add_argument('--max-requests-jitter', dest='max_requests_jitter', type=int, default=50, required=False)
         parser.add_argument('--worker-class', dest='worker_class', type=str, default='sync', required=False)
+        parser.add_argument('--statsd_endpoint', dest='statsd_endpoint', type=str, default=None, required=False)
+        parser.add_argument('--statsd_prefix', dest='statsd_prefix', type=str, default=None, required=False)
 
     def run(self) -> None:
         env = os.environ.copy()
@@ -84,6 +89,13 @@ class CmdStart(Cmd):
             *(['--reload'] if debug else ['--preload']),
             f'{self.app_module}:{self.app_name}',
         ]
+
+        if self.statsd_endpoint and self.env != ENV_LOCAL:
+            if self.statsd_prefix:
+                args.extend([f'--statsd-host={self.statsd_endpoint}', f"--statsd-prefix={self.statsd_prefix}"])
+            else:
+                args.extend([f'--statsd-host={self.statsd_endpoint}', f"--statsd-prefix={self.app_module.split('.')[1]}_{self.env}"])
+
         write_stdout(f'name={name}')
         write_stdout(f'args={args}')
         subprocess.run(['gunicorn', '--check-config', '--print-config', *args], cwd=os.getcwd(), stdout=sys.stdout, stderr=sys.stderr, text=True, env=env)

{ul-api-utils-7.10.3 → ul-api-utils-7.10.5}/ul_api_utils/modules/api_sdk.py

@@ -26,7 +26,7 @@ from ul_api_utils.debug import stat
 from ul_api_utils.debug.debugger import Debugger
 from ul_api_utils.errors import UserAbstractApiError, ResourceRuntimeApiError, ResponseTypeRuntimeApiError
 from ul_api_utils.internal_api.internal_api_check_context import internal_api_check_context
-from ul_api_utils.modules.api_sdk_config import ApiSdkConfig, SocketAsyncModesEnum
+from ul_api_utils.modules.api_sdk_config import ApiSdkConfig
 from ul_api_utils.modules.intermediate_state import try_init, try_configure
 from ul_api_utils.resources.caching import ULCache, TCacheMode, ULCacheMode, ULCacheConfig
 from ul_api_utils.resources.debugger_scripts import load_debugger_static_scripts
@@ -35,7 +35,7 @@ from ul_api_utils.resources.health_check.resource import init_health_check_resou
 from ul_api_utils.resources.not_implemented import not_implemented_handler
 from ul_api_utils.resources.permissions import load_permissions
 from ul_api_utils.resources.rate_limitter import init_rate_limiter
-from ul_api_utils.resources.socketio import init_socketio
+from ul_api_utils.resources.socketio import init_socket_io, SocketAsyncModesEnum, SocketIOConfigType
 from ul_api_utils.resources.swagger import load_swagger, ApiSdkResource
 from ul_api_utils.sentry import sentry
 from ul_api_utils.utils.api_encoding import ApiEncoding
@@ -139,7 +139,7 @@ class ApiSdk:
 
     def init_with_flask(self, app_name: str, *, db_config: Optional['MongoDbConfig'] | Optional['DbConfig'] = None) -> Flask:
         self._initialized_flask_name = try_init(self, app_name)
-        self._sio = init_socketio(self._config.server_socketio_config, self._config.client_socketio_config)
+        self._sio = init_socket_io(config=self._config.socket_config)
 
         if db_config is not None and type(db_config).__name__ == 'MongoDbConfig':
             from ul_db_utils.utils.waiting_for_mongo import waiting_for_mongo
@@ -254,16 +254,16 @@ class ApiSdk:
             debugger_enabled=self._debugger_enabled_with_pin,
         ))
 
-        if self._config.server_socketio_config:
+        if self._config.socket_config and self._config.socket_config.app_type is SocketIOConfigType.SERVER:
             assert self._sio  # mypy
             self._sio.init_app(
                 flask_app,
-                message_queue=self._config.server_socketio_config.message_queue,
+                message_queue=self._config.socket_config.message_queue,
                 async_mode=SocketAsyncModesEnum.GEVENT.value,
-                channel=self._config.server_socketio_config.channel,
-                cors_allowed_origins=self._config.server_socketio_config.cors_allowed_origins,
-                logger=self._config.server_socketio_config.logs_enabled,
-                engineio_logger=self._config.server_socketio_config.engineio_logs_enabled,
+                channel=self._config.socket_config.channel,
+                cors_allowed_origins=self._config.socket_config.cors_allowed_origins,
+                logger=self._config.socket_config.logs_enabled,
+                engineio_logger=self._config.socket_config.engineio_logs_enabled,
             )
             load_modules_by_template([os.path.join(APPLICATION_DIR, 'sockets', '*')])
 

{ul-api-utils-7.10.3 → ul-api-utils-7.10.5}/ul_api_utils/modules/api_sdk_config.py

@@ -1,23 +1,17 @@
 from enum import Enum
-from typing import List, Optional, Callable, Union, Any
+from typing import List, Optional, Callable, Union
 
-from pydantic import BaseModel, Extra, root_validator
+from pydantic import BaseModel, Extra
 
 from ul_api_utils.access import PermissionRegistry, PermissionDefinition
 from ul_api_utils.modules.api_sdk_jwt import ApiSdkJwt
+from ul_api_utils.resources.socketio import SocketIOConfig
 
 
 def join_route_paths(prev_sect: str, next_sect: str) -> str:
     return prev_sect.rstrip('/') + '/' + next_sect.lstrip('/')
 
 
-class SocketAsyncModesEnum(Enum):
-    THREADING = 'threading'
-    GEVENT = 'gevent'
-    EVENTLET = 'eventlet'
-    GEVENT_UWSGI = 'gevent_uwsgi'
-
-
 class ApiSdkIdentifyTypeEnum(Enum):
     DISABLED = 'DISABLED'
     CLIENT_IP = 'IP'
@@ -36,14 +30,6 @@ class ApiSdkFlaskDebuggingPluginsEnabled(BaseModel):
     flask_monitoring_dashboard: bool = False
 
 
-class SocketIOConfig(BaseModel):
-    message_queue: str
-    channel: Optional[str] = "flask-socketio"
-    cors_allowed_origins: Optional[str] = "*"
-    logs_enabled: Optional[bool] = False
-    engineio_logs_enabled: Optional[bool] = False
-
-
 class ApiSdkConfig(BaseModel):
     service_name: str
     permissions: Optional[Union[Callable[[], PermissionRegistry], PermissionRegistry]] = None
@@ -56,8 +42,7 @@ class ApiSdkConfig(BaseModel):
     http_auth: Optional[ApiSdkHttpAuth] = None
 
     static_url_path: Optional[str] = None
-    server_socketio_config: Optional[SocketIOConfig] = None
-    client_socketio_config: Optional[SocketIOConfig] = None
+    socket_config: Optional[SocketIOConfig] = None
     web_error_template: Optional[str] = None
 
     rate_limit: Union[str, List[str]] = '100/minute'  # [count (int)] [per|/] [second|minute|hour|day|month|year][s]
@@ -71,26 +56,6 @@ class ApiSdkConfig(BaseModel):
 
     api_route_path_prefix: str = '/api'
 
-    @staticmethod
-    def validate_socketio(values: dict[str, Any]) -> None:
-        """
-        ApiSdk instance could be either a Socket.IO server or Socket.IO client.
-        Client do not need the flask_app process attached to it, but server does.
-
-        Client should use the same config as server (message_queue).
-        Validate that ApiSdk instance couldn't be a server and a client simultaneously.
-        """
-        server_conf, client_conf = values.get("server_socketio_config"), values.get("client_socketio_config")
-        service = values["service_name"]
-        error_msg = f"Can't create ApiSdkConfig for {service}. ApiSdk cannot be a socketio server and client simultaneously"
-        if all([server_conf, client_conf]):
-            raise AssertionError(error_msg)
-
-    @root_validator
-    def validator(cls, values: dict[str, Any]) -> dict[str, Any]:
-        cls.validate_socketio(values)
-        return values
-
     class Config:
         extra = Extra.forbid
         allow_mutation = False

{ul-api-utils-7.10.3 → ul-api-utils-7.10.5}/ul_api_utils/modules/worker_sdk.py

@@ -12,9 +12,11 @@ from ul_api_utils.conf import APPLICATION_START_DT
 from ul_api_utils.modules.intermediate_state import try_init, try_configure
 from ul_api_utils.modules.worker_context import WorkerContext
 from ul_api_utils.modules.worker_sdk_config import WorkerSdkConfig
+from ul_api_utils.resources.socketio import init_socket_io
 from ul_api_utils.sentry import sentry
 
 if TYPE_CHECKING:
+    import flask_socketio  # type: ignore # lib without mypy stubs
     from ul_db_utils.modules.postgres_modules.db import DbConfig
 
 TI = TypeVar('TI', bound=UniMessage)
@@ -27,6 +29,7 @@ class WorkerSdk:
         '_config',
         '_db_initialized',
         '_flask_app_cache',
+        '_sio',
     )
 
     def __init__(self, config: WorkerSdkConfig) -> None:
@@ -35,6 +38,12 @@ class WorkerSdk:
         self._config = config
         self._db_initialized = False
         self._flask_app_cache: Optional[Flask] = None
+        self._sio: Optional['flask_socketio.SocketIO'] = None
+
+    @property
+    def socket(self) -> 'flask_socketio.SocketIO':
+        assert self._sio is not None, "SocketIO client is not configured, try adding SocketIOConfig to your WorkerSdk first."
+        return self._sio
 
     @property
     def _flask_app(self) -> Flask:
@@ -50,7 +59,7 @@ class WorkerSdk:
 
     def init(self, app_name: str, *, db_config: Optional['DbConfig'] = None) -> 'WorkerSdk':
         self._initialized_flask_name = try_init(self, app_name)
-
+        self._sio = init_socket_io(config=self._config.socket_config)
         if db_config is not None:
             from ul_db_utils.utils.waiting_for_postgres import waiting_for_postgres
             self._db_initialized = True

{ul-api-utils-7.10.3 → ul-api-utils-7.10.5}/ul_api_utils/modules/worker_sdk_config.py

@@ -1,7 +1,10 @@
 from pydantic import BaseModel, Extra
 
+from ul_api_utils.resources.socketio import SocketIOConfig
+
 
 class WorkerSdkConfig(BaseModel):
+    socket_config: SocketIOConfig | None = None
 
     class Config:
         extra = Extra.forbid

ul-api-utils-7.10.5/ul_api_utils/resources/socketio.py

@@ -0,0 +1,57 @@
+from enum import Enum
+
+from flask_socketio import SocketIO  # type: ignore
+from pydantic import BaseModel
+
+
+class SocketIOConfigType(Enum):
+    """
+    Defines two basic approaches of creating Socket.IO instance:
+        1. SERVER - Socket.IO instance is being attached to the Flask App,
+           async gunicorn workers (gevent, eventlet), requires monkey-patching of
+           psycopg and default library. Socket application.
+        2. EXTERNAL_PROCESS - Socket.IO instance isn't attached to Flask App,
+           sync gunicorn workers, doesn't require monkey-patching. Could be any
+           API, worker, etc. Used only to submit events from an external process
+           to the server clients.
+
+    Further reading: https://flask-socketio.readthedocs.io/en/latest/deployment.html#emitting-from-an-external-process
+    """
+    SERVER = "SERVER"
+    EXTERNAL_PROCESS = "EXTERNAL_PROCESS"
+
+
+class SocketIOConfig(BaseModel):
+    app_type: SocketIOConfigType = SocketIOConfigType.SERVER
+    message_queue: str
+    channel: str | None = "flask-socketio"
+    cors_allowed_origins: str | None = "*"
+    logs_enabled: bool | None = False
+    engineio_logs_enabled: bool | None = False
+
+
+class SocketAsyncModesEnum(Enum):
+    THREADING = 'threading'
+    GEVENT = 'gevent'
+    EVENTLET = 'eventlet'
+    GEVENT_UWSGI = 'gevent_uwsgi'
+
+
+def init_socket_io(config: SocketIOConfig | None) -> SocketIO | None:
+    socket_io = None
+    if config is None:
+        return None
+
+    if config.app_type is SocketIOConfigType.SERVER:
+        socket_io = SocketIO()
+
+    if config.app_type is SocketIOConfigType.EXTERNAL_PROCESS:
+        socket_io = SocketIO(
+            message_queue=config.message_queue,
+            async_mode=SocketAsyncModesEnum.GEVENT.value,
+            channel=config.channel,
+            cors_allowed_origins=config.cors_allowed_origins,
+            logger=config.logs_enabled,
+            engineio_logger=config.engineio_logs_enabled,
+        )
+    return socket_io

{ul-api-utils-7.10.3 → ul-api-utils-7.10.5}/ul_api_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ul-api-utils
-Version: 7.10.3
+Version: 7.10.5
 Summary: Python api utils
 Author: Unic-lab
 Author-email: 

{ul-api-utils-7.10.3 → ul-api-utils-7.10.5}/ul_api_utils.egg-info/requires.txt

@@ -17,12 +17,12 @@ pyyaml==6.0
 requests==2.28.1
 cryptography==38.0.1
 colored==1.4.3
-flask_socketio==5.3.6
+flask-socketio==5.3.6
 ormsgpack==1.4.1
 msgpack==1.0.4
 msgpack-types==0.2.0
 fastavro==1.7.0
-factory_boy==3.3.0
+factory-boy==3.3.0
 sentry-sdk[flask]==1.9.2
 faker==24.8.0
 types-requests==2.28.8

ul-api-utils-7.10.3/ul_api_utils/resources/socketio.py

@@ -1,20 +0,0 @@
-from flask_socketio import SocketIO  # type: ignore
-
-from ul_api_utils.modules.api_sdk_config import SocketIOConfig, SocketAsyncModesEnum
-
-
-def init_socketio(server_config: SocketIOConfig | None, client_config: SocketIOConfig | None) -> SocketIO | None:
-    socketio = None
-    if server_config is not None:
-        socketio = SocketIO()
-
-    if client_config is not None:
-        socketio = SocketIO(
-            message_queue=client_config.message_queue,
-            async_mode=SocketAsyncModesEnum.GEVENT.value,
-            channel=client_config.channel,
-            cors_allowed_origins=client_config.cors_allowed_origins,
-            logger=client_config.logs_enabled,
-            engineio_logger=client_config.engineio_logs_enabled,
-        )
-    return socketio