logtopg 1.0.0__tar.gz

logtopg-1.0.0/MANIFEST.in

@@ -0,0 +1 @@
+recursive-include logtopg *.sql

logtopg-1.0.0/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.1
+Name: logtopg
+Version: 1.0.0
+Summary: Python logging handler that stores logs in postgresql
+Home-page: https://github.com/216software/logtopg/
+Author: 216 Software, LLC
+Author-email: info@216software.com
+License: BSD License
+Platform: UNKNOWN
+
+UNKNOWN
+

logtopg-1.0.0/README.rst

@@ -0,0 +1,129 @@
++++++++++++++++++
+Log to PostgreSQL
++++++++++++++++++
+
+.. image:: https://travis-ci.org/216software/logtopg.svg?branch=master
+    :target: https://travis-ci.org/216software/logtopg
+
+.. image:: https://circleci.com/gh/216software/logtopg.png?circle-token=389fee16249541b4b1df6e8a7f8edb1401be66de
+   :target:https://circleci.com/gh/216software/logtopg
+
+Install
+=======
+
+Grab the code with pip::
+
+    $ pip install logtopg
+
+But you also have to install the ltree contrib module into your
+database::
+
+    $ sudo -u postgres psql -c "create extension ltree;"
+
+Try it out
+==========
+
+The code in `docs/example.py`_ shows how to set up your logging configs
+with this handler.
+
+.. _`docs/example.py`: https://github.com/216software/logtopg/blob/master/docs/example.py
+
+.. include:: docs/example.py
+    :number-lines:
+    :code: python
+
+
+Contribute to logtopg
+=====================
+
+Get a copy of the code::
+
+    $ git clone --origin github https://github.com/216software/logtopg.git
+
+Install it like this::
+
+    $ cd logtopg
+    $ pip install -e .
+
+Create test user and test database::
+
+    $ sudo -u postgres createuser logtopg
+    $ sudo -u postgres createdb --owner logtopg logtopg_tests
+    $ sudo -u postgres psql -c "create extension ltree;" -d logtopg_tests
+
+Then run the tests like this::
+
+    $ python setup.py --quiet test
+    .....
+    ----------------------------------------------------------------------
+    Ran 5 tests in 0.379s
+
+    OK
+
+Hopefully it works!
+
+
+Stuff to do
+===========
+
+*   Fill out classifiers in setup.py.
+
+*   Somehow block updates to the table.  Maybe a trigger is the right
+    way.  Maybe there's a much simpler trick that I'm not aware of.
+
+*   Create a few views for typical queries.
+
+*   Test performance with many connected processes and tons of logging
+    messages.  Make sure that logging doesn't compete with real
+    application work for database resources.  Is there a way to say
+    something like
+
+        "Hey postgresql, take your time with this stuff, and deal with
+        other stuff first!"
+
+    In other words, a "nice" command for queries.
+
+*   Allow people to easily write their own SQL to create the logging
+    table and to insert records to it.  The queries could be returned
+    from properties, so people would just need to subclass the PGHandler
+    and then redefine those properties.
+
+*   Write some documentation:
+
+    *   installation
+    *   typical queries
+    *   tweak log table columns or indexes
+    *   discuss performance issues
+
+*   Set up a readthedocs page for logtopg for that documentation.
+
+*   Experiment with what happens when the emit(...) function call takes
+    a long time.  For example, say somebody is logging to a PG server
+    across the internet, will calls to log.debug(...) slow down  the
+    local app?  I imagine so.
+
+*   I just found out that the ltree column type (that I use for logger
+    names) can not handle logger names like "dazzle.insert-stuff".  That
+    dash in there is invalid syntax.
+
+    I hope there is a way to raise an exception as soon as somebody uses
+    an invalid logger name.
+
+    Or, maybe I need to convert the invalid name to a valid name, by
+    maybe substituting any of a set of characters with something else.
+
+*   Set up table partitioning so that when there are millions or logs,
+    they are dealt with sanely.
+
+    This is a query that shows logs by day and log level::
+
+        select to_char(date_trunc('day', inserted), 'YYYY-MM-DD'),
+        log_level, count(*)
+
+        from dazzlelogs
+
+        group by 1, 2
+
+        order by 1, 2;
+
+.. vim: set syntax=rst:

logtopg-1.0.0/logtopg/__init__.py

@@ -0,0 +1,265 @@
+# vim: set expandtab ts=4 sw=4 filetype=python fileencoding=utf8:
+
+
+import logging
+import os
+import subprocess
+import textwrap
+import traceback
+import warnings
+
+import psutil
+
+import pkg_resources
+import psycopg2
+from psycopg2.extensions import adapt
+
+from logtopg.version import __version__
+
+log = logging.getLogger(__name__)
+
+class PGHandler(logging.Handler):
+
+    def __init__(self, log_table_name,
+        database,
+        user=None,
+        password=None,
+        host=None,
+        port=5432):
+
+        logging.Handler.__init__(self)
+
+        self.log_table_name = log_table_name
+
+        self.database = database
+        self.host = host
+        self.user = user
+        self.password = password
+        self.port = port
+
+        self.pgconn = None
+        self.create_table_sql = None
+        self.insert_row_sql = None
+
+    def check_if_log_table_exists(self):
+
+        while (True):
+            pgconn = self.get_pgconn()
+
+            cursor = pgconn.cursor()
+
+            try:
+                cursor.execute("""
+                SELECT %s::regclass;
+                """, [self.log_table_name])
+            except psycopg2.ProgrammingError as e:
+                return False
+            except InterfaceError as ie:
+                self.pgconn = None
+                continue
+
+            return True
+
+    def maybe_create_table(self):
+
+        if not self.check_if_log_table_exists():
+
+            create_table_sql = self.get_create_table_sql()
+
+            out = run_sql_commands(create_table_sql, self.user, self.password,
+                self.host, self.port, self.database)
+
+            log.info("Created log table {0}.".format(self.log_table_name))
+
+    def get_pgconn(self):
+
+        if not self.pgconn:
+            self.make_pgconn()
+
+        return self.pgconn
+
+    def make_pgconn(self):
+
+        self.pgconn = psycopg2.connect(
+            database=self.database,
+            host=self.host,
+            user=self.user,
+            password=self.password,
+            port=self.port)
+
+        self.pgconn.autocommit = True
+
+        log.info("Just made an autocommitting database connection: {0}.".format(
+            self.pgconn))
+
+    def get_create_table_sql(self):
+
+        if not self.create_table_sql:
+
+            s = \
+            pkg_resources.resource_string(
+                "logtopg", "createtable.sql")\
+            .decode("utf-8")\
+            .format(self.log_table_name)
+
+            self.create_table_sql = s.encode("utf-8")
+
+        return self.create_table_sql
+
+    def get_insert_row_sql(self):
+
+        """
+        Cache the insert query (with placeholder parameters) in memory
+        so that every log.... call doesn't do file IO.
+        """
+
+        if not self.insert_row_sql:
+
+            self.insert_row_sql = \
+            pkg_resources.resource_string(
+                "logtopg", "insertrow.sql")\
+            .decode("utf-8")\
+            .format(self.log_table_name)
+
+        return self.insert_row_sql
+
+    def build_d(self, record_dict):
+
+        d = record_dict
+
+        # Insert process info
+        d['cmd_line'] = " ".join(psutil.Process(os.getpid()).cmdline())
+
+        # Catch messages that can't be adapted as-is, and convert it to
+        # strings
+        try:
+            d["msg"] = adapt(record_dict["msg"])
+
+        except Exception as ex:
+            d["msg"] = str(record_dict["msg"])
+
+        return d
+
+    def emit(self, record):
+
+        self.format(record)
+
+        if record.exc_info:
+            record.exc_text = logging._defaultFormatter.formatException(record.exc_info)
+
+        else:
+            record.exc_text = ""
+
+        if isinstance(record.msg, Exception):
+            record.msg = str(record.msg)
+
+        pgconn = self.get_pgconn()
+
+        self.maybe_create_table()
+
+        cursor = pgconn.cursor()
+
+        cursor.execute(
+            self.get_insert_row_sql(),
+            self.build_d(record.__dict__))
+
+
+example_dict_config = dict({
+
+    "loggers": {
+        "logtopg": {
+            # "handlers": ["pg", "console"],
+            "handlers": ["console"],
+            "level": "DEBUG",
+        }
+    },
+
+    'handlers': {
+        'pg': {
+            'class': 'logtopg.PGHandler',
+            'level': 'DEBUG',
+            'log_table_name': 'logtopg_logs',
+
+            "database":"logtopg",
+            "host":"localhost",
+            "user":"logtopg",
+            "password":"l0gt0pg",
+        },
+
+        "console": {
+            "class": "logging.StreamHandler",
+            "level": "DEBUG",
+            "formatter": "consolefmt",
+        },
+
+    },
+
+    "formatters": {
+            "consolefmt":{
+                "format": '%(asctime)-22s [%(process)d] %(name)-30s %(lineno)-5d %(levelname)-8s %(message)s',
+            },
+    },
+
+    # Any handlers attached to root get log messages from EVERYTHING,
+    # like third-party modules, etc.
+    'root': {
+        'handlers': ["pg"],
+        'level': 'DEBUG',
+    },
+
+    'version': 1,
+
+    # This is important!  Without it, any log instances created before
+    # you run logging.config.dictConfig(...) will be disabled, which
+    # means all the global log objects in all the various imported files
+    # won't do anything.
+    'disable_existing_loggers': False,
+})
+
+def run_sql_commands(sql_text, user, password, host, port, database):
+
+    """
+    Run a whole bunch of SQL commands.  This is nice when you have a
+    script with more than one statement in it.
+
+    Don't pass me the path to a SQL script file!  Instead, give me the
+    sql text after you read it in from a file.
+    """
+
+    env = os.environ.copy()
+
+    if password:
+        env['PGPASSWORD'] = password
+
+    # Feed the sql_text to psql's stdin.
+    # http://stackoverflow.com/questions/163542/python-how-do-i-pass-a-string-into-subprocess-popen-using-the-stdin-argument
+
+    stuff = [
+        "psql",
+        "--quiet",
+        "--no-psqlrc",
+        "-d",
+        database,
+        "--single-transaction",
+    ]
+
+    if user:
+        stuff.append("-U")
+        stuff.append(user)
+
+    if host:
+        stuff.append("-h")
+        stuff.append(host)
+
+    if port:
+        stuff.append("-p")
+        stuff.append(str(port))
+
+    p = subprocess.Popen(
+        stuff,
+        stdin=subprocess.PIPE,
+        env=env)
+
+    out = p.communicate(input=sql_text)
+
+    return out

logtopg-1.0.0/logtopg/createtable.sql

@@ -0,0 +1,40 @@
+create table if not exists {0} (
+
+    log_id int generated
+    by default as identity primary key,
+
+    created timestamptz,
+
+    process_id int,
+    process_name text,
+
+    logger_name ltree,
+
+    path_name text,
+    module text,
+    file_name text,
+
+    function_name text,
+
+    line_number int,
+
+    log_level text,
+    log_level_number int,
+
+    cmd_line text,
+
+    message text,
+
+    exc_info text,
+    thread_id bigint,
+    thread_name text,
+    inserted timestamptz not null default now()
+);
+
+create index on {0} (created);
+create index on {0} (inserted);
+create index on {0} (logger_name);
+create index on {0} (process_id);
+
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+CREATE INDEX idx_logs_cmdline ON {0} USING GIN (cmd_line gin_trgm_ops);

logtopg-1.0.0/logtopg/insertrow.sql

@@ -0,0 +1,36 @@
+insert into {0} (
+    created,
+    process_id,
+    process_name,
+    logger_name,
+    path_name,
+    module,
+    file_name,
+    function_name,
+    line_number,
+    log_level,
+    log_level_number,
+    message,
+    exc_info,
+    cmd_line,
+    thread_id,
+    thread_name
+) values (
+    to_timestamp(%(created)s),
+    %(process)s,
+    %(processName)s,
+    %(name)s,
+    %(pathname)s,
+    %(module)s,
+    %(filename)s,
+    %(funcName)s,
+    %(lineno)s,
+    %(levelname)s,
+    %(levelno)s,
+    %(message)s,
+    %(exc_text)s,
+    %(cmd_line)s,
+    %(thread)s,
+    %(threadName)s
+);
+

logtopg-1.0.0/logtopg/tests/test_logtopg.py

@@ -0,0 +1,278 @@
+# vim: set expandtab ts=4 sw=4 filetype=python fileencoding=utf8:
+
+import logging
+import logging.config
+import os
+import unittest
+
+import logtopg
+import psycopg2
+
+testing_dict_config = dict({
+
+    "loggers": {
+        "logtopg": {
+            "handlers": ["pg"],
+            "level": "DEBUG",
+        }
+    },
+
+    'handlers': {
+        'pg': {
+            'class': 'logtopg.PGHandler',
+            'level': 'DEBUG',
+            'log_table_name': 'logtopg_tests',
+            "database":"logtopg_tests",
+        },
+
+        "console": {
+            "class": "logging.StreamHandler",
+            "level": "DEBUG",
+        },
+
+    },
+
+    # 'root': {
+    #     'handlers': ["console"],
+    #     'level': 'DEBUG'},
+
+    'version': 1,
+
+    # This is important!  Without it, any log instances created before
+    # you run logging.config.dictConfig(...) will be disabled.
+    'disable_existing_loggers': False,
+})
+
+class Test1(unittest.TestCase):
+
+    """
+    This depends on a real postgresql database.  I'll create a table and
+    then drop it.
+    """
+
+    d = testing_dict_config
+    log_table_name = d["handlers"]["pg"]["log_table_name"]
+    database = d["handlers"]["pg"]["database"]
+    user = d["handlers"]["pg"].get("user")
+    password = d["handlers"]["pg"].get("password")
+    host = d["handlers"]["pg"].get("host")
+
+    db_credentials = dict(
+        user=user,
+        password=password,
+        host=host,
+        database=database,
+    )
+
+    def setUp(self):
+
+        logging.config.dictConfig(self.d)
+
+        self.log = logging.getLogger("logtopg.tests")
+
+        self.ltpg = logtopg.PGHandler(
+            self.log_table_name,
+            self.user,
+            self.password,
+            self.host,
+            self.database)
+
+        # Make a separate database connection to check results in
+        # database.
+        self.test_pgconn = psycopg2.connect(**self.db_credentials)
+
+    def test_1(self):
+
+        """
+        Verify we only read sql files once each.
+        """
+
+        self.assertTrue(self.ltpg.create_table_sql is None)
+
+        s1 = self.ltpg.get_create_table_sql()
+
+        self.assertTrue(isinstance(self.ltpg.create_table_sql, bytes))
+
+        s2 = self.ltpg.get_create_table_sql()
+
+        self.assertTrue(s1 is s2)
+
+        self.ltpg.get_insert_row_sql()
+
+
+    def test_2(self):
+
+        """
+        Verify we make only one database connection in an instance.
+        """
+
+        ltpg = logtopg.PGHandler(
+            self.log_table_name,
+            **self.db_credentials)
+
+        self.assertTrue(ltpg.pgconn is None)
+
+        conn1 = ltpg.get_pgconn()
+
+        self.assertTrue(ltpg.pgconn)
+
+        conn2 = ltpg.get_pgconn()
+
+        self.assertTrue(conn1 is conn2)
+
+
+    def test_3(self):
+
+        """
+        Verify we can create the log table.
+        """
+
+        ltpg = logtopg.PGHandler(
+            self.log_table_name,
+            **self.db_credentials)
+
+        ltpg.maybe_create_table()
+
+        # Now, verify the table exists.
+        cursor = ltpg.pgconn.cursor()
+
+        cursor.execute("""
+            select exists(
+                select *
+                from information_schema.tables
+                where table_name = %s)
+            """, [self.log_table_name])
+
+        row = cursor.fetchone()
+
+        self.assertTrue(row[0], )
+
+        # Subsequent calls to maybe_create_table should be harmless and
+        # nearly instantaneous.
+        ltpg.maybe_create_table()
+        ltpg.maybe_create_table()
+        ltpg.maybe_create_table()
+
+        ltpg.pgconn.rollback()
+
+
+    def test_4(self):
+
+        """
+        Verify log messages are stored in the database.
+        """
+
+        logging.config.dictConfig(self.d)
+
+        log1 = logging.getLogger("logtopg.tests")
+        log2 = logging.getLogger("logtopg.tests")
+        log3 = logging.getLogger("logtopg.tests")
+        log4 = logging.getLogger("logtopg.tests")
+
+        log = logging.getLogger("logtopg.tests")
+
+        log.debug("debug!")
+        log.info("info!")
+        log.warning("warning!")
+        log.error("error!")
+        log.critical("critical!")
+
+        # Now check that those logs are actually in the database.
+        cursor = self.test_pgconn.cursor()
+
+        cursor.execute(
+            """
+            select message
+            from {}
+            where process_id = %s
+            """.format(self.log_table_name), [os.getpid()])
+
+        counted_rows = cursor.rowcount
+
+        self.test_pgconn.rollback()
+
+        # There should be 7 logs in the database with this process's ID.
+        # Those 7 are the five above and the two connection logs.
+        self.assertEqual(counted_rows, 7)
+
+
+    def test_5(self):
+
+        """
+        Verify different logger instances use a single database
+        connection.
+        """
+
+        logging.config.dictConfig(self.d)
+
+        log1 = logging.getLogger("logtopg.tests.a")
+        log1.debug("trying this guy out")
+
+        log2 = logging.getLogger("logtopg.tests.b")
+        log2.debug("trying this guy out")
+
+    def test_6(self):
+
+        """
+        Log an exception to the database.
+        """
+
+        logging.config.dictConfig(self.d)
+        log = logging.getLogger("logtopg.tests.tests_6")
+
+        try:
+
+            1/0
+
+        except Exception as ex:
+
+            log.exception(ex)
+
+        log.debug(AttributeError("This is a bogus exception"))
+
+    def test_7(self):
+
+        """
+        Log something that can't be adapted to the database.
+        """
+
+        logging.config.dictConfig(self.d)
+        log = logging.getLogger("logtopg.tests.tests_7")
+
+        class Unadaptable(object):
+            pass
+
+        u = Unadaptable()
+
+        log.debug("u is a {0}.".format(u))
+        log.debug(u)
+        log.debug(dict(u=u))
+
+    def tearDown(self):
+
+        self.test_pgconn.rollback()
+
+        cursor = self.test_pgconn.cursor()
+
+        cursor.execute(
+            "drop table if exists {0}".format(
+                Test1.log_table_name))
+
+        self.test_pgconn.commit()
+
+
+def tearDownModule():
+
+    pgconn = psycopg2.connect(**Test1.db_credentials)
+
+    cursor = pgconn.cursor()
+
+    cursor.execute(
+        "drop table if exists {0}".format(
+            Test1.log_table_name))
+
+    pgconn.commit()
+
+
+if __name__ == "__main__":
+    unittest.main()

logtopg-1.0.0/logtopg/version.py

@@ -0,0 +1,25 @@
+# vim: set expandtab ts=4 sw=4 filetype=python fileencoding=utf8:
+
+"""
+Do not do anything in this file except define the __version__ variable!
+
+The setup.py script reads this version from here during install.
+
+In the past, I've defined the version in the setup.py file (A), or
+defined it in the top of the project, like in logtopg/__init__.py (B).
+
+Choice A is bad because it isn't easy to fire up a python session and
+then do::
+
+    >>> import logtopg
+    >>> logtopg.__version__ # doctest: +SKIP
+
+to look up the version.
+
+And choice B is bad because the logtopg/__init__.py file might blow up
+during install because it tries to import a some third-party module that
+hasn't been imported yet.
+
+"""
+
+__version__ = "1.0.0"

logtopg-1.0.0/logtopg.egg-info/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.1
+Name: logtopg
+Version: 1.0.0
+Summary: Python logging handler that stores logs in postgresql
+Home-page: https://github.com/216software/logtopg/
+Author: 216 Software, LLC
+Author-email: info@216software.com
+License: BSD License
+Platform: UNKNOWN
+
+UNKNOWN
+

logtopg-1.0.0/logtopg.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+MANIFEST.in
+README.rst
+setup.py
+logtopg/__init__.py
+logtopg/createtable.sql
+logtopg/insertrow.sql
+logtopg/version.py
+logtopg.egg-info/PKG-INFO
+logtopg.egg-info/SOURCES.txt
+logtopg.egg-info/dependency_links.txt
+logtopg.egg-info/requires.txt
+logtopg.egg-info/top_level.txt
+logtopg/tests/__init__.py
+logtopg/tests/test_logtopg.py

logtopg-1.0.0/logtopg.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

logtopg-1.0.0/logtopg.egg-info/requires.txt

@@ -0,0 +1 @@
+psycopg2

logtopg-1.0.0/logtopg.egg-info/top_level.txt

@@ -0,0 +1 @@
+logtopg

logtopg-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

logtopg-1.0.0/setup.py

@@ -0,0 +1,33 @@
+# vim: set expandtab ts=4 sw=4 filetype=python fileencoding=utf8:
+
+import sys
+
+if sys.version_info < (2, 7):
+    raise Exception("sorry, this needs at least python 2.7!")
+
+# Read __version__ from version.py
+with open("logtopg/version.py") as f:
+    exec(f.read())
+
+from setuptools import find_packages, setup
+
+setup(
+    # name="LogToPG",
+    name="logtopg",
+    version=__version__,
+    description="Python logging handler that stores logs in postgresql",
+    url="https://github.com/216software/logtopg/",
+    packages=find_packages(),
+
+    author="216 Software, LLC",
+    author_email="info@216software.com",
+    license="BSD License",
+    include_package_data=True,
+
+    install_requires=[
+        'psycopg2',
+    ],
+
+    test_suite="nose.collector",
+    use_2to3=True,
+)