production-request 0.9.0__py3-none-any.whl

docs/source/conf.py

@@ -0,0 +1,290 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# production_request documentation build configuration file, created by
+# cookiecutter pipproject
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import os
+import sys
+
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath('../..'))
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'production_request'
+copyright = '2016, Dmitriy Chernyshov'
+author = 'Dmitriy Chernyshov'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '0.0.1'
+# The full version, including alpha/beta/rc tags.
+release = '0.0.1'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = []
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+#keep_warnings = False
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'sphinx_rtd_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.
+# "<project> v<release> documentation" by default.
+#html_title = 'production_request v0.0.1'
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (relative to this directory) to use as a favicon of
+# the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+#html_extra_path = []
+
+# If not None, a 'Last updated on:' timestamp is inserted at every page
+# bottom, using the given strftime format.
+# The empty string is equivalent to '%b %d, %Y'.
+#html_last_updated_fmt = None
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Language to be used for generating the HTML full-text search index.
+# Sphinx supports the following languages:
+#   'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja'
+#   'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh'
+#html_search_language = 'en'
+
+# A dictionary with options for the search language support, empty by default.
+# 'ja' uses this config value.
+# 'zh' user can custom change `jieba` dictionary path.
+#html_search_options = {'type': 'default'}
+
+# The name of a javascript file (relative to the configuration directory) that
+# implements a search results scorer. If empty, the default will be used.
+#html_search_scorer = 'scorer.js'
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'production_requestdoc'
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+
+# Latex figure (float) alignment
+#'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'production_request.tex', 'production_request Documentation',
+     'Dmitriy Chernyshov', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'production_request', 'production_request Documentation',
+     [author], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'production_request', 'production_request Documentation',
+     author, 'production_request', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+#texinfo_no_detailmenu = False

production_request/__init__.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+from importlib.metadata import (
+    PackageNotFoundError,
+    version,
+)
+
+
+try:
+    # Change here if project is renamed and does not equal the package name
+    dist_name = __name__
+    __version__ = version(dist_name)
+except PackageNotFoundError:
+    __version__ = 'unknown'

production_request/api.py

@@ -0,0 +1,297 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import json
+import logging
+import os
+import socket
+from datetime import (
+    datetime,
+)
+
+from django.conf import (
+    settings,
+)
+from psutil import (
+    Process,
+)
+
+from production_request.enums import (
+    SqlOperationEnum,
+)
+from production_request.utils import (
+    CUSTOM_ATTR,
+    SimpleRequest,
+    clear_thread_data,
+    generate_uuid,
+    get_client_ip,
+    get_custom_attrs_from_thread_data,
+    get_thread_data,
+    ms_from_timedelta,
+    register_logger,
+    to_MB,
+)
+
+
+PRODUCTION_REQUEST_LOGGER = register_logger(
+    'production_request',
+    formatter=logging.Formatter("%(message)s"))
+
+PRODUCTION_REQUEST_ENTER_LOGGER = register_logger(
+    'production_request_enter',
+    formatter=logging.Formatter("%(message)s"))
+
+
+class ProductionLogger(object):
+    def __init__(self, uuid_):
+        super(ProductionLogger, self).__init__()
+        self.uuid = uuid_
+
+    def log_start(self, request):
+        """Начинает процесс логирования объекта"""
+        request.production_request_uuid = self.uuid
+
+        if settings.PRODUCTION_REQUEST:
+            thread_data = get_thread_data()
+            thread_data.production_request_uuid = self.uuid
+            thread_data.production_request_original_path = self.get_path(request)
+
+            request.started_at = datetime.now()
+
+            if settings.PRODUCTION_REQUEST_LOG_MEMORY:
+                memory = self.get_memory_info()
+
+                request.production_request_uss = memory.uss
+                request.production_request_pss = memory.pss
+                request.production_request_swap = memory.swap
+                request.production_request_rss = memory.rss
+
+        if settings.PRODUCTION_REQUEST_LOG_ENTER:
+            self.log_enter(self.uuid, self.get_path(request))
+
+    def log_end(self, request, is_success):
+        """Логирует параметры запроса и результаты обработки"""
+        row = None
+
+        if settings.PRODUCTION_REQUEST:
+            try:
+                thread_data = get_thread_data()
+
+                process_request_time = ms_from_timedelta(
+                    datetime.now() - request.started_at
+                )
+                process_sql_time = thread_data.sql_time.pop(
+                    request.production_request_uuid, float())
+                sync_sql_total = '{:.4f}'.format(process_sql_time)
+
+                process_async_sql_time = thread_data.async_sql_time_total.pop(
+                    request.production_request_uuid, float()
+                )
+                async_sql_total = '{:.4f}'.format(process_async_sql_time)
+
+                sql_total = '{:.4f}'.format(process_sql_time + process_async_sql_time)
+
+                sync_sql_count = thread_data.sql_count.pop(
+                    request.production_request_uuid, 0)
+
+                async_sql_count = thread_data.async_sql_count.pop(
+                    request.production_request_uuid, 0)
+
+                sql_duplicate_count = sync_sql_count - len(
+                    thread_data.sql_uniq_hashes.pop(
+                        request.production_request_uuid,
+                        set()
+                    )
+                )
+
+                user = getattr(request, 'user', None)
+                user_id = user.id if user else ''
+
+                path = self.get_path(request, thread_data)
+
+                if hasattr(thread_data, 'transaction_count'):
+                    transaction_count = thread_data.transaction_count[
+                        request.production_request_uuid]
+                else:
+                    transaction_count = 0
+
+                if hasattr(thread_data, 'transaction_total'):
+                    transaction_total = thread_data.transaction_total[
+                        request.production_request_uuid]
+                else:
+                    transaction_total = 0.0
+
+                db_aliases = ', '.join(sorted(thread_data.sql_db_aliases))
+
+                row = {
+                    'uuid': self.uuid,
+                    'is_success': json.dumps(is_success),
+                    'started': str(request.started_at),
+                    'path': path,
+                    'user': str(user_id or ''),
+                    'hostname': socket.gethostname(),
+                    'total': '{:.4f}'.format(process_request_time),
+                    'sql_total': sql_total,
+                    'sync_sql_total': sync_sql_total,
+                    'async_sql_total': async_sql_total,
+                    'sql_count': str(sync_sql_count + async_sql_count),
+                    'sync_sql_count': str(sync_sql_count),
+                    'async_sql_count': str(async_sql_count),
+                    'sql_duplicate_count': str(sql_duplicate_count),
+                    'sql_transaction_count': str(transaction_count),
+                    'sql_transaction_total': '{:.4f}'.format(transaction_total),
+                    'sql_db_aliases': db_aliases,
+                    'client_ip': get_client_ip(request),
+                    'pid': str(os.getpid()),
+                }
+
+                # количество и время sql-запросов в разрезе типов
+                for operator in SqlOperationEnum.values:
+                    attrs = ['sync_sql_{}_count'.format(operator)]
+                    attrs.extend(['async_sql_{}_count'.format(operator)])
+
+                    if operator in SqlOperationEnum.operations:
+                        attrs.append('sync_sql_{}_total'.format(operator))
+
+                    for operator_attr in attrs:
+                        counter = getattr(
+                            thread_data,
+                            operator_attr,
+                            {}
+                        )
+                        _value = counter.pop(
+                            request.production_request_uuid,
+                            0
+                        )
+                        if isinstance(_value, float):
+                            _value = '{:.4f}'.format(_value)
+                        else:
+                            _value = str(_value)
+
+                        row[operator_attr] = _value
+
+                # добавляем кастомные значения к общему результату
+                row[CUSTOM_ATTR] = []
+                for attr, value in get_custom_attrs_from_thread_data():
+                    key = CUSTOM_ATTR + '_' + attr
+                    row[CUSTOM_ATTR].append(key)
+                    row[key] = str(value)
+
+                if settings.PRODUCTION_REQUEST_LOG_MEMORY:
+                    memory = self.get_memory_info()
+
+                    row['uss'] = '{:.2f}'.format(
+                        self.get_memory_diff(
+                            request.production_request_uss,
+                            memory.uss
+                        )
+                    )
+                    row['pss'] = '{:.2f}'.format(
+                        self.get_memory_diff(
+                            request.production_request_pss,
+                            memory.pss
+                        )
+                    )
+                    row['swap'] = '{:.2f}'.format(
+                        self.get_memory_diff(
+                            request.production_request_swap,
+                            memory.swap
+                        )
+                    )
+                    row['rss'] = '{:.2f}'.format(
+                        self.get_memory_diff(
+                            request.production_request_rss,
+                            memory.rss
+                        )
+                    )
+
+                if settings.PRODUCTION_REQUEST_LOG_SERVER:
+                    PRODUCTION_REQUEST_LOGGER.info(
+                        json.dumps(row)
+                    )
+            except Exception as err:
+                if settings.PRODUCTION_REQUEST_LOG_SERVER:
+                    PRODUCTION_REQUEST_LOGGER.error(
+                        json.dumps(
+                            {
+                                'error': str(err),
+                            }
+                        )
+                    )
+
+        if settings.PRODUCTION_REQUEST_LOG_ENTER:
+            self.log_exit(self.uuid, self.get_path(request))
+
+        clear_thread_data()
+
+        return row
+
+    def get_path(self, request, thread_data=None):
+        """Возвращает параметр path."""
+
+        if hasattr(thread_data, 'production_request_original_path'):
+            path = thread_data.production_request_original_path
+        elif hasattr(request, 'path'):
+            path = request.path
+        else:
+            path = getattr(request, 'name', '')
+
+        return path
+
+    def get_memory_info(self):
+        """Возвращает данные о памяти процесса"""
+        return Process(os.getpid()).memory_full_info()
+
+    def get_memory_diff(self, m_begin, m_end):
+        """Возвращает разницу размера памяти в МБ"""
+        return to_MB((m_end - m_begin))
+
+    def log_enter(self, request_uuid, request_path):
+        """Логирует момент входа объекта в приложение"""
+        self.log_state('enter', request_uuid, request_path)
+
+    def log_exit(self, request_uuid, request_path):
+        """Логирует момент выхода запроса в приложение"""
+        self.log_state('exit', request_uuid, request_path)
+
+    def log_state(self, state, request_uuid, request_path):
+        """Логирует определенный момент запроса"""
+        PRODUCTION_REQUEST_ENTER_LOGGER.info(
+            json.dumps(
+                {
+                    'uuid': request_uuid,
+                    'path': request_path,
+                    'state': state,
+                    'pid': os.getpid(),
+                    'timestamp': str(datetime.now()),
+                }
+            )
+        )
+
+
+def production_request(path=''):
+
+    def decorator(func):
+
+        def wrapper(*args, **kwargs):
+            request = SimpleRequest(path=path)
+            logger = ProductionLogger(generate_uuid())
+
+            logger.log_start(request)
+            is_success = True
+            try:
+                result = func(*args, **kwargs)
+            except:
+                is_success = False
+                raise
+            finally:
+                logger.log_end(request, is_success)
+
+            return result
+
+        return wrapper
+
+    return decorator
+
+
+
+

production_request/base.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import django
+
+
+try:
+    from django.db.backends import (
+        util as utils,
+    )
+except ImportError:
+    from django.db.backends import (
+        utils,
+    )
+
+
+if django.VERSION >= (3, 0):
+    from django.db.backends.postgresql import (
+        base,
+    )
+else:
+    from django.db.backends.postgresql_psycopg2 import (
+        base,
+    )
+
+from production_request.utils import (
+    calc_sql_metrics,
+    calc_transaction_metrics,
+)
+
+
+class CursorWrapper(utils.CursorWrapper):
+    """
+    Враппер курсора БД для подсчета времени выполнения запроса
+    """
+    def execute(self, sql, params=None):
+        with calc_sql_metrics(self, sql):
+            return self.cursor.execute(sql, params)
+
+    def executemany(self, sql, param_list):
+        with calc_sql_metrics(self, sql):
+            return self.cursor.executemany(sql, param_list)
+
+
+class DatabaseWrapper(base.DatabaseWrapper):
+    """
+    Враппер БД с кастомным курсором
+    """
+    def cursor(self):
+        if hasattr(self, 'validate_thread_sharing'):
+            self.validate_thread_sharing()
+        cursor = CursorWrapper(self._cursor(), self)
+        return cursor
+
+    def _commit(self):
+        with calc_sql_metrics(self.cursor(), 'COMMIT'):
+            return super(DatabaseWrapper, self)._commit()
+
+    def _set_autocommit(self, autocommit):
+        with calc_transaction_metrics(autocommit):
+            super(DatabaseWrapper, self)._set_autocommit(autocommit)
+