logtopg 1.0.2__py3-none-any.whl

logtopg/__init__.py

@@ -0,0 +1,265 @@
+# vim: set expandtab ts=4 sw=4 filetype=python fileencoding=utf8:
+
+
+import logging
+import os
+import subprocess
+import textwrap
+import traceback
+import warnings
+
+import psutil
+
+import pkg_resources
+import psycopg2
+from psycopg2.extensions import adapt
+
+from logtopg.version import __version__
+
+log = logging.getLogger(__name__)
+
+class PGHandler(logging.Handler):
+
+    def __init__(self, log_table_name,
+        database,
+        user=None,
+        password=None,
+        host=None,
+        port=5432):
+
+        logging.Handler.__init__(self)
+
+        self.log_table_name = log_table_name
+
+        self.database = database
+        self.host = host
+        self.user = user
+        self.password = password
+        self.port = port
+
+        self.pgconn = None
+        self.create_table_sql = None
+        self.insert_row_sql = None
+
+    def check_if_log_table_exists(self):
+
+        while (True):
+            pgconn = self.get_pgconn()
+
+            cursor = pgconn.cursor()
+
+            try:
+                cursor.execute("""
+                SELECT %s::regclass;
+                """, [self.log_table_name])
+            except psycopg2.ProgrammingError as e:
+                return False
+            except InterfaceError as ie:
+                self.pgconn = None
+                continue
+
+            return True
+
+    def maybe_create_table(self):
+
+        if not self.check_if_log_table_exists():
+
+            create_table_sql = self.get_create_table_sql()
+
+            out = run_sql_commands(create_table_sql, self.user, self.password,
+                self.host, self.port, self.database)
+
+            log.info("Created log table {0}.".format(self.log_table_name))
+
+    def get_pgconn(self):
+
+        if not self.pgconn:
+            self.make_pgconn()
+
+        return self.pgconn
+
+    def make_pgconn(self):
+
+        self.pgconn = psycopg2.connect(
+            database=self.database,
+            host=self.host,
+            user=self.user,
+            password=self.password,
+            port=self.port)
+
+        self.pgconn.autocommit = True
+
+        log.info("Just made an autocommitting database connection: {0}.".format(
+            self.pgconn))
+
+    def get_create_table_sql(self):
+
+        if not self.create_table_sql:
+
+            s = \
+            pkg_resources.resource_string(
+                "logtopg", "createtable.sql")\
+            .decode("utf-8")\
+            .format(self.log_table_name)
+
+            self.create_table_sql = s.encode("utf-8")
+
+        return self.create_table_sql
+
+    def get_insert_row_sql(self):
+
+        """
+        Cache the insert query (with placeholder parameters) in memory
+        so that every log.... call doesn't do file IO.
+        """
+
+        if not self.insert_row_sql:
+
+            self.insert_row_sql = \
+            pkg_resources.resource_string(
+                "logtopg", "insertrow.sql")\
+            .decode("utf-8")\
+            .format(self.log_table_name)
+
+        return self.insert_row_sql
+
+    def build_d(self, record_dict):
+
+        d = record_dict
+
+        # Insert process info
+        d['cmd_line'] = " ".join(psutil.Process(os.getpid()).cmdline())
+
+        # Catch messages that can't be adapted as-is, and convert it to
+        # strings
+        try:
+            d["msg"] = adapt(record_dict["msg"])
+
+        except Exception as ex:
+            d["msg"] = str(record_dict["msg"])
+
+        return d
+
+    def emit(self, record):
+
+        self.format(record)
+
+        if record.exc_info:
+            record.exc_text = logging._defaultFormatter.formatException(record.exc_info)
+
+        else:
+            record.exc_text = ""
+
+        if isinstance(record.msg, Exception):
+            record.msg = str(record.msg)
+
+        pgconn = self.get_pgconn()
+
+        self.maybe_create_table()
+
+        cursor = pgconn.cursor()
+
+        cursor.execute(
+            self.get_insert_row_sql(),
+            self.build_d(record.__dict__))
+
+
+example_dict_config = dict({
+
+    "loggers": {
+        "logtopg": {
+            # "handlers": ["pg", "console"],
+            "handlers": ["console"],
+            "level": "DEBUG",
+        }
+    },
+
+    'handlers': {
+        'pg': {
+            'class': 'logtopg.PGHandler',
+            'level': 'DEBUG',
+            'log_table_name': 'logtopg_logs',
+
+            "database":"logtopg",
+            "host":"localhost",
+            "user":"logtopg",
+            "password":"l0gt0pg",
+        },
+
+        "console": {
+            "class": "logging.StreamHandler",
+            "level": "DEBUG",
+            "formatter": "consolefmt",
+        },
+
+    },
+
+    "formatters": {
+            "consolefmt":{
+                "format": '%(asctime)-22s [%(process)d] %(name)-30s %(lineno)-5d %(levelname)-8s %(message)s',
+            },
+    },
+
+    # Any handlers attached to root get log messages from EVERYTHING,
+    # like third-party modules, etc.
+    'root': {
+        'handlers': ["pg"],
+        'level': 'DEBUG',
+    },
+
+    'version': 1,
+
+    # This is important!  Without it, any log instances created before
+    # you run logging.config.dictConfig(...) will be disabled, which
+    # means all the global log objects in all the various imported files
+    # won't do anything.
+    'disable_existing_loggers': False,
+})
+
+def run_sql_commands(sql_text, user, password, host, port, database):
+
+    """
+    Run a whole bunch of SQL commands.  This is nice when you have a
+    script with more than one statement in it.
+
+    Don't pass me the path to a SQL script file!  Instead, give me the
+    sql text after you read it in from a file.
+    """
+
+    env = os.environ.copy()
+
+    if password:
+        env['PGPASSWORD'] = password
+
+    # Feed the sql_text to psql's stdin.
+    # http://stackoverflow.com/questions/163542/python-how-do-i-pass-a-string-into-subprocess-popen-using-the-stdin-argument
+
+    stuff = [
+        "psql",
+        "--quiet",
+        "--no-psqlrc",
+        "-d",
+        database,
+        "--single-transaction",
+    ]
+
+    if user:
+        stuff.append("-U")
+        stuff.append(user)
+
+    if host:
+        stuff.append("-h")
+        stuff.append(host)
+
+    if port:
+        stuff.append("-p")
+        stuff.append(str(port))
+
+    p = subprocess.Popen(
+        stuff,
+        stdin=subprocess.PIPE,
+        env=env)
+
+    out = p.communicate(input=sql_text)
+
+    return out

logtopg/createtable.sql

@@ -0,0 +1,40 @@
+create table if not exists {0} (
+
+    log_id int generated
+    by default as identity primary key,
+
+    created timestamptz,
+
+    process_id int,
+    process_name text,
+
+    logger_name ltree,
+
+    path_name text,
+    module text,
+    file_name text,
+
+    function_name text,
+
+    line_number int,
+
+    log_level text,
+    log_level_number int,
+
+    cmd_line text,
+
+    message text,
+
+    exc_info text,
+    thread_id bigint,
+    thread_name text,
+    inserted timestamptz not null default now()
+);
+
+create index on {0} (created);
+create index on {0} (inserted);
+create index on {0} (logger_name);
+create index on {0} (process_id);
+
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+CREATE INDEX idx_{0}_cmdline ON {0} USING GIN (cmd_line gin_trgm_ops);

logtopg/insertrow.sql

@@ -0,0 +1,36 @@
+insert into {0} (
+    created,
+    process_id,
+    process_name,
+    logger_name,
+    path_name,
+    module,
+    file_name,
+    function_name,
+    line_number,
+    log_level,
+    log_level_number,
+    message,
+    exc_info,
+    cmd_line,
+    thread_id,
+    thread_name
+) values (
+    to_timestamp(%(created)s),
+    %(process)s,
+    %(processName)s,
+    %(name)s,
+    %(pathname)s,
+    %(module)s,
+    %(filename)s,
+    %(funcName)s,
+    %(lineno)s,
+    %(levelname)s,
+    %(levelno)s,
+    %(message)s,
+    %(exc_text)s,
+    %(cmd_line)s,
+    %(thread)s,
+    %(threadName)s
+);
+

logtopg/tests/test_logtopg.py

@@ -0,0 +1,278 @@
+# vim: set expandtab ts=4 sw=4 filetype=python fileencoding=utf8:
+
+import logging
+import logging.config
+import os
+import unittest
+
+import logtopg
+import psycopg2
+
+testing_dict_config = dict({
+
+    "loggers": {
+        "logtopg": {
+            "handlers": ["pg"],
+            "level": "DEBUG",
+        }
+    },
+
+    'handlers': {
+        'pg': {
+            'class': 'logtopg.PGHandler',
+            'level': 'DEBUG',
+            'log_table_name': 'logtopg_tests',
+            "database":"logtopg_tests",
+        },
+
+        "console": {
+            "class": "logging.StreamHandler",
+            "level": "DEBUG",
+        },
+
+    },
+
+    # 'root': {
+    #     'handlers': ["console"],
+    #     'level': 'DEBUG'},
+
+    'version': 1,
+
+    # This is important!  Without it, any log instances created before
+    # you run logging.config.dictConfig(...) will be disabled.
+    'disable_existing_loggers': False,
+})
+
+class Test1(unittest.TestCase):
+
+    """
+    This depends on a real postgresql database.  I'll create a table and
+    then drop it.
+    """
+
+    d = testing_dict_config
+    log_table_name = d["handlers"]["pg"]["log_table_name"]
+    database = d["handlers"]["pg"]["database"]
+    user = d["handlers"]["pg"].get("user")
+    password = d["handlers"]["pg"].get("password")
+    host = d["handlers"]["pg"].get("host")
+
+    db_credentials = dict(
+        user=user,
+        password=password,
+        host=host,
+        database=database,
+    )
+
+    def setUp(self):
+
+        logging.config.dictConfig(self.d)
+
+        self.log = logging.getLogger("logtopg.tests")
+
+        self.ltpg = logtopg.PGHandler(
+            self.log_table_name,
+            self.user,
+            self.password,
+            self.host,
+            self.database)
+
+        # Make a separate database connection to check results in
+        # database.
+        self.test_pgconn = psycopg2.connect(**self.db_credentials)
+
+    def test_1(self):
+
+        """
+        Verify we only read sql files once each.
+        """
+
+        self.assertTrue(self.ltpg.create_table_sql is None)
+
+        s1 = self.ltpg.get_create_table_sql()
+
+        self.assertTrue(isinstance(self.ltpg.create_table_sql, bytes))
+
+        s2 = self.ltpg.get_create_table_sql()
+
+        self.assertTrue(s1 is s2)
+
+        self.ltpg.get_insert_row_sql()
+
+
+    def test_2(self):
+
+        """
+        Verify we make only one database connection in an instance.
+        """
+
+        ltpg = logtopg.PGHandler(
+            self.log_table_name,
+            **self.db_credentials)
+
+        self.assertTrue(ltpg.pgconn is None)
+
+        conn1 = ltpg.get_pgconn()
+
+        self.assertTrue(ltpg.pgconn)
+
+        conn2 = ltpg.get_pgconn()
+
+        self.assertTrue(conn1 is conn2)
+
+
+    def test_3(self):
+
+        """
+        Verify we can create the log table.
+        """
+
+        ltpg = logtopg.PGHandler(
+            self.log_table_name,
+            **self.db_credentials)
+
+        ltpg.maybe_create_table()
+
+        # Now, verify the table exists.
+        cursor = ltpg.pgconn.cursor()
+
+        cursor.execute("""
+            select exists(
+                select *
+                from information_schema.tables
+                where table_name = %s)
+            """, [self.log_table_name])
+
+        row = cursor.fetchone()
+
+        self.assertTrue(row[0], )
+
+        # Subsequent calls to maybe_create_table should be harmless and
+        # nearly instantaneous.
+        ltpg.maybe_create_table()
+        ltpg.maybe_create_table()
+        ltpg.maybe_create_table()
+
+        ltpg.pgconn.rollback()
+
+
+    def test_4(self):
+
+        """
+        Verify log messages are stored in the database.
+        """
+
+        logging.config.dictConfig(self.d)
+
+        log1 = logging.getLogger("logtopg.tests")
+        log2 = logging.getLogger("logtopg.tests")
+        log3 = logging.getLogger("logtopg.tests")
+        log4 = logging.getLogger("logtopg.tests")
+
+        log = logging.getLogger("logtopg.tests")
+
+        log.debug("debug!")
+        log.info("info!")
+        log.warning("warning!")
+        log.error("error!")
+        log.critical("critical!")
+
+        # Now check that those logs are actually in the database.
+        cursor = self.test_pgconn.cursor()
+
+        cursor.execute(
+            """
+            select message
+            from {}
+            where process_id = %s
+            """.format(self.log_table_name), [os.getpid()])
+
+        counted_rows = cursor.rowcount
+
+        self.test_pgconn.rollback()
+
+        # There should be 7 logs in the database with this process's ID.
+        # Those 7 are the five above and the two connection logs.
+        self.assertEqual(counted_rows, 7)
+
+
+    def test_5(self):
+
+        """
+        Verify different logger instances use a single database
+        connection.
+        """
+
+        logging.config.dictConfig(self.d)
+
+        log1 = logging.getLogger("logtopg.tests.a")
+        log1.debug("trying this guy out")
+
+        log2 = logging.getLogger("logtopg.tests.b")
+        log2.debug("trying this guy out")
+
+    def test_6(self):
+
+        """
+        Log an exception to the database.
+        """
+
+        logging.config.dictConfig(self.d)
+        log = logging.getLogger("logtopg.tests.tests_6")
+
+        try:
+
+            1/0
+
+        except Exception as ex:
+
+            log.exception(ex)
+
+        log.debug(AttributeError("This is a bogus exception"))
+
+    def test_7(self):
+
+        """
+        Log something that can't be adapted to the database.
+        """
+
+        logging.config.dictConfig(self.d)
+        log = logging.getLogger("logtopg.tests.tests_7")
+
+        class Unadaptable(object):
+            pass
+
+        u = Unadaptable()
+
+        log.debug("u is a {0}.".format(u))
+        log.debug(u)
+        log.debug(dict(u=u))
+
+    def tearDown(self):
+
+        self.test_pgconn.rollback()
+
+        cursor = self.test_pgconn.cursor()
+
+        cursor.execute(
+            "drop table if exists {0}".format(
+                Test1.log_table_name))
+
+        self.test_pgconn.commit()
+
+
+def tearDownModule():
+
+    pgconn = psycopg2.connect(**Test1.db_credentials)
+
+    cursor = pgconn.cursor()
+
+    cursor.execute(
+        "drop table if exists {0}".format(
+            Test1.log_table_name))
+
+    pgconn.commit()
+
+
+if __name__ == "__main__":
+    unittest.main()

logtopg/version.py

@@ -0,0 +1,25 @@
+# vim: set expandtab ts=4 sw=4 filetype=python fileencoding=utf8:
+
+"""
+Do not do anything in this file except define the __version__ variable!
+
+The setup.py script reads this version from here during install.
+
+In the past, I've defined the version in the setup.py file (A), or
+defined it in the top of the project, like in logtopg/__init__.py (B).
+
+Choice A is bad because it isn't easy to fire up a python session and
+then do::
+
+    >>> import logtopg
+    >>> logtopg.__version__ # doctest: +SKIP
+
+to look up the version.
+
+And choice B is bad because the logtopg/__init__.py file might blow up
+during install because it tries to import a some third-party module that
+hasn't been imported yet.
+
+"""
+
+__version__ = "1.0.2"

logtopg-1.0.2.dist-info/METADATA

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: logtopg
+Version: 1.0.2
+Summary: Python logging handler that stores logs in postgresql
+Home-page: https://github.com/216software/logtopg/
+Author: 216 Software, LLC
+Author-email: info@216software.com
+License: BSD License
+Requires-Dist: psycopg2
+Dynamic: author
+Dynamic: author-email
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: summary

logtopg-1.0.2.dist-info/RECORD

@@ -0,0 +1,10 @@
+logtopg/__init__.py,sha256=wEvVz_YRJcR3wTvFGQM59BtWZdEXsV2pxp9IUwBv2So,6283
+logtopg/createtable.sql,sha256=qilEAG8B371vqGacO9Tu1v5v-dhQZ_K5LNW1Tn-zywA,744
+logtopg/insertrow.sql,sha256=mr-3kVSMxDNfMpfGj0k2utewK81zNQtmyIlvQslKW8M,577
+logtopg/version.py,sha256=AwDmPU8h5HA6c_izak87tlXvUQi9Z3dT5g5N7bvBhF0,722
+logtopg/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+logtopg/tests/test_logtopg.py,sha256=rVeM-o9Ggi9CpNb5v0ovhBBoCGj-pxwrW4fG8oCzi54,6259
+logtopg-1.0.2.dist-info/METADATA,sha256=MYkXymE_KyckIpr8KzGun4xjyt5qtM2MR5XAlX4TyFc,385
+logtopg-1.0.2.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+logtopg-1.0.2.dist-info/top_level.txt,sha256=56AgV79tf5I8ve-ns3cXFYYFbHtjh6IbSA-mfEhvPS4,8
+logtopg-1.0.2.dist-info/RECORD,,

logtopg-1.0.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.10.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

logtopg-1.0.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+logtopg