ul-api-utils 9.3.0__py3-none-any.whl

ul_api_utils/conf/ul-debugger-ui.js

@@ -0,0 +1 @@
+(()=>{"use strict";var e={2:(e,t,r)=>{r.r(t),r.d(t,{API_UTILS_DEBUGGER_ENABLED:()=>n,DURATION_DANGER:()=>a,DURATION_WARNING:()=>o,X_API_UTILS_DEBUGGER_PIN:()=>i});const n="api-utils-debugger-enabled",i="x-api-utils-debugger-pin",o=.2,a=.4},219:(e,t,r)=>{r.r(t),r.d(t,{default:()=>i});var n=r(2);function i(e,t){function r(e,r,n){const i=new Date;i.setTime(i.getTime()+24*n*60*60*1e3);let o="expires="+i.toUTCString();t.cookie=e+"="+r+";"+o+";path=/"}function i(e){let r=e+"=",n=decodeURIComponent(t.cookie).split(";");for(let e=0;e<n.length;e++){let t=n[e];for(;" "===t.charAt(0);)t=t.substring(1);if(0===t.indexOf(r))return t.substring(r.length,t.length)}return""}e.setDebuggerPin=function(t){r(n.X_API_UTILS_DEBUGGER_PIN,t,1),e.location.reload()},e.$apiUtilsDebugger={},e.$apiUtilsDebugger.setCookie=r,e.$apiUtilsDebugger.getCookie=i;let o=[];const a=e.addDebuggerRequestStats=function(...e){o.push(e)};if(""===i(n.X_API_UTILS_DEBUGGER_PIN))return;const u=t.querySelector("script[data-ul-debugger]");if(!u)return;e.$apiUtilsDebugger.mainScript=u.dataset.ulDebugger;let l=u.dataset.ulDebugger.split("/");l[l.length-1]="",e.$apiUtilsDebugger.apiExplain=l.join("/")+"debugger-explain";let s=document.createElement("script");s.src=e.$apiUtilsDebugger.mainScript,s.async=!1,t.body.append(s);let g=e.setInterval((function(){e.addDebuggerRequestStats!==a&&(o.forEach((function(t){e.addDebuggerRequestStats(...t)})),o=[],e.clearInterval(g))}),50)}}},t={};function r(n){var i=t[n];if(void 0!==i)return i.exports;var o=t[n]={exports:{}};return e[n](o,o.exports,r),o.exports}r.d=(e,t)=>{for(var n in t)r.o(t,n)&&!r.o(e,n)&&Object.defineProperty(e,n,{enumerable:!0,get:t[n]})},r.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),r.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})};var n={};(()=>{r.r(n);var e=r(219);window instanceof Window?(0,e.default)(window,window.document):console.error("window is not of Window instance")})()})();

ul_api_utils/conf.py

@@ -0,0 +1,70 @@
+import logging
+import os
+import sys
+from datetime import datetime
+from typing import Optional
+
+from ul_py_tool.utils.colors import NC, FG_BLUE
+
+from ul_api_utils.const import APPLICATION_ENV__LOCAL, API_PATH__SWAGGER, API_PATH__SWAGGER_SPEC
+from ul_api_utils.utils.decode_base64 import decode_base64_to_string
+from ul_api_utils.utils.flags import Flagged
+
+APPLICATION_START_DT = datetime.fromisoformat(os.environ.get('APPLICATION_START_DT', datetime.now().isoformat()))
+APPLICATION_DEBUGGER_PIN = os.environ.get('APPLICATION_DEBUGGER_PIN', '1232344321')
+assert len(APPLICATION_DEBUGGER_PIN) > 0
+APPLICATION_GUNICORN_WORKERS = os.environ.get('APPLICATION_GUNICORN_WORKERS', '')
+DOCKER_BUILD__CONTAINER_CODE_COMMIT_HASH = os.environ.get('DOCKER_BUILD__CONTAINER_CODE_COMMIT_HASH', '')
+DOCKER_BUILD__CONTAINER_SERVER_TIME = os.environ.get('DOCKER_BUILD__CONTAINER_SERVER_TIME', '')
+DOCKER_BUILD__CONTAINER_CODE_TAG = os.environ.get('DOCKER_BUILD__CONTAINER_CODE_TAG', '')
+
+APPLICATION_ENV: str = os.environ.get('APPLICATION_ENV', APPLICATION_ENV__LOCAL)  # TODO: make it required!
+APPLICATION_ENV_IS_LOCAL = APPLICATION_ENV == APPLICATION_ENV__LOCAL
+
+APPLICATION_DIR: str = os.path.abspath(os.environ.get('APPLICATION_DIR', os.getcwd()))
+APPLICATION_TMP: str = os.path.join(APPLICATION_DIR, '.tmp')
+
+APPLICATION_UNDER_DOCKER = '/docker_app/' in os.getcwd()
+
+APPLICATION_DEBUG = os.environ.get('APPLICATION_DEBUG', '0') == '1'  # this env var set in app-utils
+APPLICATION_DEBUG_LOGGING = os.environ.get('APPLICATION_DEBUG_LOGGING', '0') == '1'  # this env var set in app-utils
+
+_APPLICATION_JWT_PUBLIC_KEY = os.environ.get('APPLICATION_JWT_PUBLIC_KEY', None) or None  # "or None" in case empty string. TODO: make it required!
+APPLICATION_JWT_PUBLIC_KEY: str = decode_base64_to_string(_APPLICATION_JWT_PUBLIC_KEY) if _APPLICATION_JWT_PUBLIC_KEY is not None else ''
+
+_APPLICATION_JWT_PRIVATE_KEY = os.environ.get('APPLICATION_JWT_PRIVATE_KEY', None) or None  # "or None" in case empty string
+APPLICATION_JWT_PRIVATE_KEY: Optional[str] = decode_base64_to_string(_APPLICATION_JWT_PRIVATE_KEY) if _APPLICATION_JWT_PRIVATE_KEY is not None else None
+
+APPLICATION_LOG_FORMAT = os.environ.get(
+    'APPLICATION_LOG_FORMAT',
+    f'%(asctime)s | %(levelname)-7s | %(name)s:%(funcName)s |'
+    f'{FG_BLUE if APPLICATION_ENV_IS_LOCAL and APPLICATION_DEBUG else ""} %(message)s{NC if APPLICATION_ENV_IS_LOCAL and APPLICATION_DEBUG else ""}',
+)
+assert len(APPLICATION_LOG_FORMAT) and '(message)' in APPLICATION_LOG_FORMAT
+
+os.makedirs(APPLICATION_TMP, exist_ok=True)
+
+APPLICATION_LOG_LEVEL = os.environ.get('APPLICATION_LOG_LEVEL', os.environ.get('LOGLEVEL', 'INFO')).strip().upper()
+_LOG_MAP = {
+    'INFO': logging.INFO,
+    'DEBUG': logging.DEBUG,
+    'ERROR': logging.ERROR,
+    'WARNING': logging.WARNING,
+}
+assert APPLICATION_LOG_LEVEL in _LOG_MAP
+
+APPLICATION_SENTRY_DSN = os.environ.get('APPLICATION_SENTRY_DSN', '')
+APPLICATION_SENTRY_ENABLED_FLASK = os.environ.get('APPLICATION_SENTRY_FLASK', '1') == '1'
+
+APPLICATION_F = Flagged(os.environ.get('APPLICATION_F', ''))
+
+APPLICATION_SWAGGER_SPECIFICATION_PATH = os.environ.get('APPLICATION_SWAGGER_SPECIFICATION_PATH', API_PATH__SWAGGER_SPEC)
+APPLICATION_SWAGGER_PATH = os.environ.get('APPLICATION_SWAGGER_PATH', API_PATH__SWAGGER)
+
+logging.basicConfig(
+    handlers=[logging.StreamHandler(sys.stdout)],
+    level=_LOG_MAP[APPLICATION_LOG_LEVEL],
+    format=APPLICATION_LOG_FORMAT,
+)
+
+logging.getLogger('').setLevel(_LOG_MAP[APPLICATION_LOG_LEVEL])

ul_api_utils/const.py

@@ -0,0 +1,78 @@
+import os.path
+from typing import Union, Dict, Any, List
+
+THIS_LIB_CWD = os.path.dirname(__file__)
+
+PY_FILE_SUF = '.py'
+
+RESPONSE_PROP_OK = 'ok'
+RESPONSE_PROP_STATUS = 'status'
+RESPONSE_PROP_PAYLOAD = 'payload'
+RESPONSE_PROP_COUNT = 'count'
+RESPONSE_PROP_TOTAL = 'total_count'
+RESPONSE_PROP_ERRORS = 'errors'
+RESPONSE_PROP_DEBUG_STATS = '_debug_stats_'
+
+
+APPLICATION_ENV__LOCAL = 'local'
+APPLICATION_ENV__DEV = 'dev'
+APPLICATION_ENV__QA = 'qa'
+APPLICATION_ENV__STAGING = 'staging'
+APPLICATION_ENV__PROD = 'prod'
+
+
+RESPONSE_HEADER__CONTENT_TYPE = 'Content-Type'
+RESPONSE_HEADER__AUTHORIZATION = 'Authorization'
+RESPONSE_HEADER__WWW_AUTH = 'WWW-Authenticate'
+REQUEST_HEADER__CONTENT_TYPE = 'Content-Type'
+REQUEST_HEADER__CONTENT_ENCODING = 'Content-Encoding'
+REQUEST_HEADER__ACCEPT_CONTENT_ENCODING = 'Accept-Encoding'
+REQUEST_HEADER__DEBUGGER = 'x-api-utils-debugger-pin'
+REQUEST_HEADER__INTERNAL = 'x-api-utils-internal-api'
+REQUEST_HEADER__ACCEPT = 'Accept'
+REQUEST_HEADER__X_FORWARDED_FOR = 'X-Forwarded-For'
+REQUEST_HEADER__USER_AGENT = 'User-Agent'
+
+
+MIME__AVRO = 'application/avro'
+MIME__JSON = 'application/json'
+MIME__MSGPCK = 'application/x-msgpack'
+
+
+ENCODING_MIME__GZIP = 'gzip'
+
+REQUEST_METHOD__PUT = 'PUT'
+REQUEST_METHOD__GET = 'GET'
+REQUEST_METHOD__POST = 'POST'
+REQUEST_METHOD__QUERY = 'QUERY'
+REQUEST_METHOD__PATCH = 'PATCH'
+REQUEST_METHOD__DELETE = 'DELETE'
+REQUEST_METHOD__OPTIONS = 'OPTIONS'
+
+
+UNDEFINED = 'UNDEFINED'
+OOPS = '...Ooops something went wrong. internal error'
+
+
+TRestJson = Union[List[Dict[str, Any]], Dict[str, Any]]
+
+
+API_PATH__SWAGGER = '/swagger'
+API_PATH__SWAGGER_SPEC = '/swagger-specification'
+API_PATH__PERMISSIONS = '/permissions'
+API_PATH__DEBUGGER_JS_UI = '/ul-debugger-ui.js'
+API_PATH__DEBUGGER_JS_MAIN = '/ul-debugger-main.js'
+
+
+INTERNAL_API__DEFAULT_PATH_PREFIX = '/api'
+
+AUTO_GZIP_THRESHOLD_LENGTH = 1000
+
+CRON_EXPRESSION_VALIDATION_REGEX = "(^((\*\/)?([0-5]?[0-9])((\,|\-|\/)([0-5]?[0-9]))*|\*)\s+((\*\/)?((2[0-3]|1[0-9]|[0-9]|00))" \
+                                   "((\,|\-|\/)(2[0-3]|1[0-9]|[0-9]|00))*|\*)\s+((\*\/)?([1-9]|[12][0-9]|3[01])((\,|\-|\/)" \
+                                   "([1-9]|[12][0-9]|3[01]))*|\*)\s+((\*\/)?([1-9]|1[0-2])((\,|\-|\/)" \
+                                   "([1-9]|1[0-2]))*|\*|(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|des))\s+((\*\/)?[0-6]" \
+                                   "((\,|\-|\/)[0-6])*|\*|00|(sun|mon|tue|wed|thu|fri|sat))\s*$)|@(annually|yearly|monthly|weekly|daily|hourly|reboot)"  # noqa
+
+MAX_UTC_OFFSET_SECONDS = 50400
+MIN_UTC_OFFSET_SECONDS = -43200

ul_api_utils/debug/debugger.py

@@ -0,0 +1,119 @@
+import json
+import time
+from enum import Enum
+from json import JSONEncoder
+from typing import Dict, Any, Union, List, NamedTuple
+
+from ul_api_utils.const import RESPONSE_PROP_DEBUG_STATS, API_PATH__DEBUGGER_JS_UI, API_PATH__DEBUGGER_JS_MAIN
+from ul_api_utils.debug import stat
+# https://www.w3schools.com/js/js_cookies.asp
+from ul_api_utils.debug.stat import collecting_enabled
+from ul_api_utils.utils.api_method import ApiMethod
+from ul_api_utils.utils.api_path_version import ApiPathVersion
+from ul_api_utils.utils.colors import COLORS_MAP__TERMINAL
+
+
+class DebuggerJSONEncoder(JSONEncoder):
+    def default(self, obj: object) -> Union[str, Dict[str, Any], List[Any], None]:
+        if isinstance(obj, tuple):
+            return list(obj)
+        if isinstance(obj, Enum):
+            return str(obj.value)
+        if isinstance(obj, set):
+            return list(obj)
+        if hasattr(obj, "__html__"):  # it needs for Flask ?
+            return str(obj.__html__())
+        return super().default(obj)
+
+
+AJAX_INTERSEPTOR = """
+<script>
+(function(window) {
+    "use strict";
+
+const {fetch: origFetch} = window;
+window.fetch = (resource, options = {}, ...args) => {
+  const startedAt = Date.now() / 1000;
+  const p = origFetch(resource, options, ...args);
+
+  p
+  .then((resp) => {
+    return resp.clone().json().then((result) => {
+        if (window.addDebuggerRequestStats != null) {
+            window.addDebuggerRequestStats(options.method, resource, startedAt, Date.now() / 1000, result._debug_stats_, resp.statusCode)
+        }
+    })
+  })
+  .catch((err) => console.error(err))
+
+  return p;
+};
+})(window);
+</script>
+"""
+
+
+class DebuggerError(NamedTuple):
+    err: Exception
+    tb: str
+    at: float
+
+
+class Debugger:
+
+    __slots__ = (
+        'name',
+        'enabled',
+        'method',
+        'url',
+    )
+
+    def __init__(self, name: str, enabled: bool, method: ApiMethod, url: str) -> None:
+        self.name = name
+        self.enabled = enabled
+        self.method = method
+        self.url = url
+
+    def render_console(self) -> None:
+        if not self.enabled:
+            return
+
+        started_at = stat.get_request_started_at()
+        ended_at = time.perf_counter()
+        name = self.name
+
+        stats = stat.get_stat(code_spans=True, started_at=started_at, ended_at=ended_at)
+        stat_str = stat.mk_stat_string(name, stats, started_at=started_at, ended_at=ended_at, truncate_request_len=None, trim_txt_new_line=True, cm=COLORS_MAP__TERMINAL)
+
+        print(f'\n{stat_str}\n')  # noqa
+
+    def render_dict(self, status_code: int) -> Dict[str, Any]:
+        if not collecting_enabled():
+            return {}
+        stats = [st.unwrap() for st in stat.get_stat(started_at=stat.get_request_started_at(), ended_at=time.perf_counter())]
+
+        return {
+            RESPONSE_PROP_DEBUG_STATS: stats,
+        }
+
+    def render_html(self, status_code: int) -> str:
+        script = (
+            f'<script '
+            f'data-ul-debugger="{ApiPathVersion.NO_VERSION.compile_path(API_PATH__DEBUGGER_JS_MAIN, prefix="/api")}" '
+            f'src="{ApiPathVersion.NO_VERSION.compile_path(API_PATH__DEBUGGER_JS_UI, prefix="/api")}" '
+            f'></script>'
+        )
+        if not self.enabled:
+            return script
+
+        started_at = stat.get_request_started_at()
+        ended_at = time.perf_counter()
+
+        stats = stat.get_stat(code_spans=True, started_at=started_at, ended_at=ended_at)
+
+        return (
+            f'{script}'
+            '<script>setTimeout(function(){{\n'
+            f'window.addDebuggerRequestStats("{self.method.value}", "{self.url}", {started_at}, {ended_at}, {json.dumps(stats, cls=DebuggerJSONEncoder)}, {status_code});'
+            '\n}}, 300);</script>'
+        )

ul_api_utils/debug/malloc.py

@@ -0,0 +1,93 @@
+import contextlib
+import os
+import sys
+import tracemalloc
+from typing import Dict, Optional, Iterator
+
+import math
+from ul_py_tool.utils.colors import FG_BLUE, NC, FG_GRAY, FG_YELLOW, FG_GREEN
+from ul_py_tool.utils.write_stdout import write_stdout
+
+
+def display_stat(name: str, stat: Dict[str, float], sort_by_value: bool = False, color: Optional[str] = None) -> None:
+    stat_max_len = max(len(key) for key in stat.keys())
+    max_val = max(v for v in stat.values()) if len(stat) > 1 else 1.
+
+    if sort_by_value:
+        sorted_keys = sorted(stat.keys(), key=stat.get)  # type: ignore
+    else:
+        sorted_keys = sorted(stat.keys())
+
+    chart_width = 30
+    write_stdout(f'\n{name}:')
+    for key in sorted_keys:
+        clr = color
+        if clr is None:
+            clr = FG_BLUE if key.endswith(".py") else FG_GRAY
+        value = stat[key]
+        chart_size = (math.ceil(value * chart_width / max_val) if max_val else 0)
+        write_stdout(f'{key.strip().strip(os.sep).strip(): <{stat_max_len}} = {clr}{value: >10.3f} KiB [{"▉" * chart_size}{" " * (chart_width - chart_size)}]{NC}')
+
+
+@contextlib.contextmanager
+def trace_malloc(show_all: bool = True, show_libs: bool = True, show_code: bool = True) -> Iterator[None]:
+    tracemalloc.start()
+
+    yield
+
+    snapshot = tracemalloc.take_snapshot()
+
+    snapshot = snapshot.filter_traces((
+        tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
+        tracemalloc.Filter(False, "<frozen importlib._bootstrap_external>"),
+        tracemalloc.Filter(False, "<unknown>"),
+    ))
+
+    top_stats = snapshot.statistics('filename')
+
+    sites_packages = '/site-packages/'
+    cwd = os.path.normpath(os.path.abspath(os.getcwd()))
+
+    libs_stat: Dict[str, float] = {}
+    code_stat: Dict[str, float] = {}
+    all_files_stat: Dict[str, float] = {}
+    other_stat: Dict[str, float] = {}
+
+    for st in top_stats:
+        frame = st.traceback[0]
+        fname = os.path.normpath(os.path.abspath(frame.filename))
+        fname_segm = fname.strip().split(os.sep)
+        size = st.size / 1024
+
+        if not fname.endswith('.py'):
+            other_stat[fname] = other_stat.get(fname, 0.) + size
+            continue
+
+        for i in range(len(fname_segm)):
+            key = os.sep.join(fname_segm[:i + 1])
+            all_files_stat[key] = all_files_stat.get(key, 0.) + size
+
+            if key.startswith(sys.prefix) and sites_packages in key:
+                i = key.index(sites_packages)
+                lib_key = "/".join(key[i:].split(os.sep)[2:3])
+                if lib_key:
+                    libs_stat[lib_key] = libs_stat.get(lib_key, 0.) + size
+            elif key.startswith(cwd):
+                code_key = key.removeprefix(cwd)
+                code_stat[code_key] = code_stat.get(code_key, 0.) + size
+
+    if show_all:
+        display_stat('All Files', all_files_stat)
+
+    if show_libs:
+        display_stat('Libs', libs_stat, sort_by_value=True, color=FG_YELLOW)
+
+    if show_code:
+        display_stat('Code', code_stat, sort_by_value=True, color=FG_BLUE)
+
+    display_stat('Other', other_stat, sort_by_value=True, color=FG_BLUE)
+
+    total = sum(st.size for st in top_stats)
+    write_stdout(f"{FG_GREEN}Total allocated size: {(total / 1024):.3f} KiB{NC}")
+
+    tracemalloc.stop()