pgsqlpot 2.0.0__py2.py3-none-any.whl

output_plugins/mysql.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+
+from sys import exc_info, version_info
+
+from core import output
+from core.config import CONFIG
+from core.tools import geolocate
+
+from geoip2.database import Reader
+
+if version_info[0] < 3:
+    import pymysql
+    pymysql.install_as_MySQLdb()
+
+try:
+    from MySQLdb import (Error, OperationalError)
+except ImportError:
+    try:
+        from MySQLdb._exceptions import (Error, OperationalError)
+    except ImportError:
+        from _mysql_exceptions import (Error, OperationalError) # type: ignore
+
+from twisted.enterprise.adbapi import ConnectionPool
+from twisted.python.compat import reraise
+from twisted.python.log import msg
+
+
+class ReconnectingConnectionPool(ConnectionPool):
+    """
+    Reconnecting adbapi connection pool for MySQL.
+    This class improves on the solution posted at
+    http://www.gelens.org/2008/09/12/reinitializing-twisted-connectionpool/
+    by checking exceptions by error code and only disconnecting the current
+    connection instead of all of them.
+    Also see:
+    http://twistedmatrix.com/pipermail/twisted-python/2009-July/020007.html
+    """
+
+    def _runInteraction(self, interaction, *args, **kw):
+
+        def rerise_exception(conn):
+            _, excValue, excTraceback = exc_info()
+            try:
+                conn.rollback()
+            except Exception:
+                msg('Rollback failed')
+            reraise(excValue, excTraceback)
+
+        conn = self.connectionFactory(self)
+        trans = self.transactionFactory(self, conn)
+        try:
+            result = interaction(trans, *args, **kw)
+            trans.close()
+            conn.commit()
+            return result
+        except OperationalError as e:
+            if e.args[0] not in (2003, 2006, 2013):
+                rerise_exception(conn)
+            else:
+                conn = self.connections.get(self.threadID())
+                self.disconnect(conn)
+                return ConnectionPool._runInteraction(self, interaction, *args, **kw)
+        except Exception:
+            rerise_exception(conn)
+
+
+class Output(output.Output):
+
+    def local_log(self, message):
+        if self.debug:
+            msg(message)
+
+    def start(self):
+        host = CONFIG.get('output_mysql', 'host', fallback='localhost')
+        database = CONFIG.get('output_mysql', 'database', fallback='pgsqlpot')
+        user = CONFIG.get('output_mysql', 'username', fallback='pgsqlpot', raw=True)
+        password = CONFIG.get('output_mysql', 'password', fallback='', raw=True)
+        port = CONFIG.getint('output_mysql', 'port', fallback=3306)
+
+        self.debug = CONFIG.getboolean('output_mysql', 'debug', fallback=False)
+        self.geoip = CONFIG.getboolean('output_mysql', 'geoip', fallback=True)
+
+        try:
+            self.dbh = ReconnectingConnectionPool(
+                'MySQLdb',
+                host=host,
+                db=database,
+                user=user,
+                passwd=password,
+                port=port,
+                charset='utf8',
+                use_unicode=True,
+                cp_min=1,
+                cp_max=1
+            )
+        except Error as e:
+            self.local_log('output_mysql: MySQL Error {}: "{}"'.format(e.args[0], e.args[1]))
+
+        if self.geoip:
+            geoipdb_city_path = CONFIG.get('output_mysql', 'geoip_citydb', fallback='data/GeoLite2-City.mmdb')
+            geoipdb_asn_path = CONFIG.get('output_mysql', 'geoip_asndb', fallback='data/GeoLite2-ASN.mmdb')
+            try:
+                self.reader_city = Reader(geoipdb_city_path)
+            except Exception:
+                self.reader_city = None
+                self.local_log('Failed to open City GeoIP database {}'.format(geoipdb_city_path))
+
+            try:
+                self.reader_asn = Reader(geoipdb_asn_path)
+            except Exception:
+                self.reader_asn = None
+                self.local_log('Failed to open ASN GeoIP database {}'.format(geoipdb_asn_path))
+
+    def stop(self):
+        if self.geoip:
+            if self.reader_city is not None:
+                self.reader_city.close()
+            if self.reader_asn is not None:
+                self.reader_asn.close()
+
+    def write(self, event):
+        """
+        TODO: Check if the type (date, datetime or timestamp) of columns is appropriate for your needs and timezone
+        - MySQL Documentation - The DATE, DATETIME, and TIMESTAMP Types
+            (https://dev.mysql.com/doc/refman/5.7/en/datetime.html):
+        "MySQL converts TIMESTAMP values from the current time zone to UTC for storage,
+        and back from UTC to the current time zone for retrieval.
+        (This does not occur for other types such as DATETIME.)"
+        """
+        self.dbh.runInteraction(self.connect_event, event)
+
+    def simple_query(self, txn, sql, args):
+        if self.debug:
+            if len(args):
+                self.local_log("output_mysql: MySQL query: {} {}".format(sql, repr(args)))
+            else:
+                self.local_log("output_mysql: MySQL query: {}".format(sql))
+        try:
+            if len(args):
+                txn.execute(sql, args)
+            else:
+                txn.execute(sql)
+            result = txn.fetchall()
+        except Exception as e:
+            self.local_log('output_mysql: MySQL Error: {}'.format(e))
+            result = None
+        return result
+
+    def get_id(self, txn, table, column, entry):
+        r = self.simple_query(txn, "SELECT `id` FROM `{}` WHERE `{}` = %s".format(table, column), (entry, ))
+        if r:
+            id = r[0][0]
+        else:
+            self.simple_query(txn, "INSERT INTO `{}` (`{}`) VALUES (%s)".format(table, column), (entry, ))
+            r = self.simple_query(txn, 'SELECT LAST_INSERT_ID()', ())
+            if r:
+                id = int(r[0][0])
+            else:
+                id = 0
+        return id
+
+    def connect_event(self, txn, event):
+        remote_ip = event['src_ip']
+        operation_id = self.get_id(txn, 'operations', 'op_name', event['operation'])
+        sensor_id = self.get_id(txn, 'sensors', 'name', event['sensor'])
+
+        self.simple_query(txn,
+            """
+            INSERT INTO `connections` (
+                `session`, `timestamp`, `operation`, `ip`,
+                `remote_port`, `local_host`, `local_port`, `sensor`)
+            VALUES (%s, FROM_UNIXTIME(%s), %s, %s, %s, %s, %s, %s)
+            """,
+            (event['session'], event['unixtime'], operation_id, remote_ip,
+             event['src_port'], event['dst_ip'], event['dst_port'], sensor_id, ))
+
+        if event['operation'].lower() == 'login':
+            usr_id = self.get_id(txn, 'usernames', 'username', event['username'])
+            pwd_id = self.get_id(txn, 'passwords', 'password', event['password'])
+            self.simple_query(txn,
+            """
+                INSERT INTO `credentials` (`session`, `username`, `password`) VALUES(%s, %s, %s)
+            """,
+            (event['session'], usr_id, pwd_id, ))
+            if 'variables' in event:
+                for key, value in event['variables'].items():
+                    var_id = self.get_id(txn, 'vars', 'var_name', key)
+                    val_id = self.get_id(txn, 'var_values', 'var_value', value)
+                    self.simple_query(txn,
+                    """
+                        INSERT INTO `variables` (`session`, `var`, `val`) VALUES(%s, %s, %s)
+                    """,
+                    (event['session'], var_id, val_id, ))
+
+        if self.geoip:
+            country, country_code, city, org, asn_num = geolocate(remote_ip, self.reader_city, self.reader_asn)
+            self.simple_query(txn, """
+                INSERT INTO `geolocation` (`ip`, `country_name`, `country_iso_code`, `city_name`, `org`, `org_asn`)
+                VALUES (%s, %s, %s, %s, %s, %s)
+                ON DUPLICATE KEY UPDATE
+                    `country_name` = %s,
+                    `country_iso_code` = %s,
+                    `city_name` = %s,
+                    `org` = %s,
+                    `org_asn` = %s
+                """,
+                (remote_ip, country, country_code, city, org, asn_num, country, country_code, city, org, asn_num, ))

output_plugins/nlcvapi.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+
+from io import BytesIO
+from json import dumps, loads
+
+from core import output
+from core.config import CONFIG
+from core.tools import decode, geolocate, to_bytes
+
+from geoip2.database import Reader
+
+from twisted.internet import reactor
+from twisted.internet.ssl import ClientContextFactory
+from twisted.python.log import msg
+from twisted.web.client import (
+    Agent,
+    FileBodyProducer,
+    HTTPConnectionPool,
+    _HTTP11ClientFactory,
+    readBody,
+)
+from twisted.web.http_headers import Headers
+
+
+class WebClientContextFactory(ClientContextFactory):
+
+    def getContext(self, hostname, port):
+        return ClientContextFactory.getContext(self)
+
+
+class QuietHTTP11ClientFactory(_HTTP11ClientFactory):
+    noisy = False
+
+
+class Output(output.Output):
+
+    def start(self):
+        self.host = CONFIG.get('output_nlcvapi', 'host', fallback='https://api.nlcv.bas.bg/v1.0/honeypot')
+        self.geoip = CONFIG.getboolean('output_nlcvapi', 'geoip', fallback=True)
+        contextFactory = WebClientContextFactory()
+        pool = HTTPConnectionPool(reactor)
+        pool._factory = QuietHTTP11ClientFactory
+        self.agent = Agent(reactor, contextFactory=contextFactory, pool=pool)
+
+        if self.geoip:
+            geoipdb_city_path = CONFIG.get('output_nlcvapi', 'geoip_citydb', fallback='data/GeoLite2-City.mmdb')
+            geoipdb_asn_path = CONFIG.get('output_nlcvapi', 'geoip_asndb', fallback='data/GeoLite2-ASN.mmdb')
+            try:
+                self.reader_city = Reader(geoipdb_city_path)
+            except Exception:
+                self.reader_city = None
+                msg('Failed to open City GeoIP database {}'.format(geoipdb_city_path))
+            try:
+                self.reader_asn = Reader(geoipdb_asn_path)
+            except Exception:
+                self.reader_asn = None
+                msg('Failed to open ASN GeoIP database {}'.format(geoipdb_asn_path))
+
+    def stop(self):
+        if self.geoip:
+            if getattr(self, 'reader_city', None) is not None:
+                self.reader_city.close()
+            if getattr(self, 'reader_asn', None) is not None:
+                self.reader_asn.close()
+
+    def write(self, event):
+        event['honeypot'] = 'pgsqlpot'
+        if self.geoip:
+            country, country_code, city, org, asn_num = geolocate(event['src_ip'], self.reader_city, self.reader_asn)
+            event['country'] = country
+            event['country_code'] = country_code
+            event['city'] = city
+            event['org'] = org
+            event['asn'] = asn_num
+        self._postentry(event)
+
+    def _postentry(self, entry):
+
+        def cbBody(body):
+            return processResult(body)
+
+        def cbPartial(failure):
+            """
+            Google HTTP Server does not set Content-Length. Twisted marks it as partial
+            """
+            failure.printTraceback()
+            return processResult(failure.value.response)
+
+        def cbResponse(response):
+            if response.code in (200, 201):
+                return
+            msg('NLCVAPI response: {}: "{}"'.format(response.code, decode(response.phrase)))
+            d = readBody(response)
+            d.addCallback(cbBody)
+            d.addErrback(cbPartial)
+            return d
+
+        def cbError(failure):
+            failure.printTraceback()
+
+        def processResult(result):
+            try:
+                j = loads(result)
+                msg('NLCVAPI response: {}'.format(j.get('message', '')))
+            except Exception:
+                pass
+
+        headers = Headers({
+            b'User-Agent':   [b'PGSQLPot'],
+            b'Content-Type': [b'application/json'],
+        })
+        body = FileBodyProducer(BytesIO(to_bytes(dumps(entry, sort_keys=True))))
+        d = self.agent.request(b'POST', to_bytes(self.host), headers, body)
+        d.addCallback(cbResponse)
+        d.addErrback(cbError)
+        return d

output_plugins/postgres.py

@@ -0,0 +1,154 @@
+
+from __future__ import absolute_import
+
+from core import output
+from core.config import CONFIG
+from core.tools import geolocate
+
+from geoip2.database import Reader
+from psycopg2 import OperationalError
+
+from twisted.enterprise.adbapi import ConnectionPool
+from twisted.python.log import msg
+
+
+class Output(output.Output):
+
+    def start(self):
+        host = CONFIG.get('output_postgres', 'host', fallback='localhost')
+        port = CONFIG.getint('output_postgres', 'port', fallback=5432)
+        username = CONFIG.get('output_postgres', 'username', fallback='pgsqlpot')
+        password = CONFIG.get('output_postgres', 'password')
+        database = CONFIG.get('output_postgres', 'database', fallback='pgsqlpot')
+        self.debug = CONFIG.getboolean('output_postgres', 'debug', fallback=False)
+        self.geoip = CONFIG.getboolean('output_postgres', 'geoip', fallback=True)
+
+        try:
+            self.dbh = ConnectionPool(
+                'psycopg2',
+                database=database,
+                user=username,
+                password=password,
+                host=host,
+                port=port,
+                cp_min=1,
+                cp_max=1
+            )
+        except OperationalError as e:
+            msg('output_postgres: postgres Error: {}'.format(e))
+
+        if self.geoip:
+            geoipdb_city_path = CONFIG.get('output_postgres', 'geoip_citydb', fallback='data/GeoLite2-City.mmdb')
+            geoipdb_asn_path  = CONFIG.get('output_postgres', 'geoip_asndb', fallback='data/GeoLite2-ASN.mmdb')
+            try:
+                self.reader_city = Reader(geoipdb_city_path)
+            except Exception:
+                self.reader_city = None
+                msg('Failed to open City GeoIP database {}'.format(geoipdb_city_path))
+            try:
+                self.reader_asn = Reader(geoipdb_asn_path)
+            except Exception:
+                self.reader_asn = None
+                msg('Failed to open ASN GeoIP database {}'.format(geoipdb_asn_path))
+
+    def stop(self):
+        if self.geoip:
+            if self.reader_city is not None:
+                self.reader_city.close()
+            if self.reader_asn is not None:
+                self.reader_asn.close()
+
+    def write(self, event):
+        self.dbh.runInteraction(self.connect_event, event)
+
+    def simple_query(self, txn, sql, args, returns_value=True):
+        if self.debug:
+            msg('output_postgres: postgres query: {} {}'.format(sql, repr(args)))
+        result = None
+        try:
+            txn.execute(sql, args)
+            if returns_value:
+                result = txn.fetchone()
+        except Exception as e:
+            msg('output_postgres: postgres Error: {}'.format(e))
+        return result
+
+    def get_id(self, txn, table, column, entry):
+        r = self.simple_query(
+            txn,
+            "SELECT id FROM {} WHERE {} = %s".format(table, column),
+            (entry, )
+        )
+        if r:
+            id = int(r[0])
+        else:
+            r = self.simple_query(
+                txn,
+                "INSERT INTO {} ({}) VALUES (%s) RETURNING id".format(table, column),
+                (entry, )
+            )
+            if r:
+                id = int(r[0])
+            else:
+                id = 0
+        return id
+
+    def connect_event(self, txn, event):
+        remote_ip = event['src_ip']
+        operation_id = self.get_id(txn, 'operations', 'op_name', event['operation'])
+        sensor_id = self.get_id(txn, 'sensors', 'sname', event['sensor'])
+
+        self.simple_query(
+            txn,
+            """
+            INSERT INTO connections (
+                sess_no, time_stamp, ip, remote_port, operation,
+                local_host, local_port, sensor)
+            VALUES (%s, to_timestamp(%s), %s, %s, %s, %s, %s, %s)
+            """,
+            (event['session'], event['unixtime'], remote_ip, event['src_port'], operation_id,
+             event['dst_ip'], event['dst_port'], sensor_id, ),
+            False
+        )
+
+        if event['operation'].lower() == 'login':
+            usr_id = self.get_id(txn, 'usernames', 'username', event['username'])
+            pwd_id = self.get_id(txn, 'passwords', 'passwd', event['password'])
+            self.simple_query(
+                txn,
+                """
+                INSERT INTO credentials (sess_no, username, passwd) VALUES(%s, %s, %s)
+                """,
+                (event['session'], usr_id, pwd_id, ),
+                False
+            )
+            if 'variables' in event:
+                for key, value in event['variables'].items():
+                    var_id = self.get_id(txn, 'vars', 'var_name', key)
+                    val_id = self.get_id(txn, 'var_values', 'var_value', value)
+                    self.simple_query(
+                        txn,
+                        """
+                        INSERT INTO variables (sess_no, var, val) VALUES(%s, %s, %s)
+                        """,
+                        (event['session'], var_id, val_id, ),
+                        False
+                    )
+
+        if self.geoip:
+            country, country_code, city, org, asn_num = geolocate(remote_ip, self.reader_city, self.reader_asn)
+            self.simple_query(
+                txn,
+                """
+                INSERT INTO geolocation (ip, country_name, country_iso_code, city_name, org, org_asn)
+                VALUES (%s, %s, %s, %s, %s, %s)
+                ON CONFLICT (ip) DO UPDATE SET
+                    country_name = EXCLUDED.country_name,
+                    country_iso_code = EXCLUDED.country_iso_code,
+                    city_name = EXCLUDED.city_name,
+                    org = EXCLUDED.org,
+                    org_asn = EXCLUDED.org_asn
+                """,
+                (remote_ip, country, country_code, city, org, asn_num, ),
+                False
+            )

output_plugins/redisdb.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+
+from json import dumps
+
+from core import output
+from core.config import CONFIG
+
+from requests import post
+
+from twisted.internet.threads import deferToThread
+from twisted.python.log import msg
+
+class Output(output.Output):
+
+    def start(self):
+        self.debug = CONFIG.getboolean('output_redisdb', 'debug', fallback=False)
+        host = CONFIG.get('output_redisdb', 'host')
+        self.password = CONFIG.get('output_redisdb', 'password')
+        keyname = CONFIG.get('output_redisdb', 'keyname', fallback='pgsqlpot')
+        self.url = 'https://{}/lpush/{}'.format(host, keyname)
+
+    def stop(self):
+        pass
+
+    def write(self, event):
+        payload = dumps(event, sort_keys=True)
+        headers = {
+            b'Authorization': 'Bearer {}'.format(self.password),
+            b'Content-Type': b'application/json'
+        }
+
+        if self.debug:
+            msg(payload)
+
+        d = deferToThread(post, self.url, data=payload, headers=headers, timeout=5)
+        d.addCallback(self._on_response)
+        d.addErrback(self._on_error)
+
+    def _on_response(self, response):
+        if self.debug and response.status_code != 200:
+            msg('[REST] Response code: {}'.format(response.status_code))
+
+    def _on_error(self, failure):
+        msg('[REST] REST error: {}'.format(failure))

output_plugins/rethinkdblog.py

@@ -0,0 +1,46 @@
+
+from __future__ import absolute_import
+
+from time import sleep
+
+from core import output
+from core.config import CONFIG
+
+from rethinkdb import r
+
+from twisted.python.log import msg
+
+
+class Output(output.Output):
+
+    def start(self):
+        host = CONFIG.get('output_rethinkdblog', 'host', fallback='localhost')
+        port = CONFIG.getint('output_rethinkdblog', 'port', fallback=28015)
+        self.db = CONFIG.get('output_rethinkdblog', 'db', fallback='pgsqlpot')
+        password = CONFIG.get('output_rethinkdblog', 'password', raw=True)
+        self.table = CONFIG.get('output_rethinkdblog', 'table', fallback='events')
+
+        try:
+            self.connection = r.connect(host=host, port=port, password=password)
+
+            if self.db not in r.db_list().run(self.connection):
+                r.db_create(self.db).run(self.connection)
+            for _ in range(50):  # ~5 seconds max
+                if self.db in r.db_list().run(self.connection):
+                    break
+                sleep(0.1)
+            if self.table not in r.db(self.db).table_list().run(self.connection):
+                r.db(self.db).table_create(self.table).run(self.connection)
+
+        except r.RqlError as err:
+            msg('output_rethinkdblog: Error: {}'.format(err.message))
+
+    def stop(self):
+        if getattr(self, 'connection', None):
+            self.connection.close()
+
+    def write(self, event):
+        try:
+            r.db(self.db).table(self.table).insert(event).run(self.connection)
+        except r.RqlRuntimeError as err:
+            msg('output_rethinkdblog: Error: {}'.format(err.message))

output_plugins/slack.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+
+from sys import version_info
+from time import gmtime, strftime, time
+
+from core import output
+from core.config import CONFIG
+
+from twisted.internet import reactor
+from twisted.internet.task import deferLater
+
+# --- Slack compatibility wrapper ---
+if version_info[0] >= 3:
+    # Python 3: slack-sdk
+    from slack_sdk import WebClient as SlackClientWrapper # type: ignore
+else:
+    # Python 2.7: slackclient 1.x
+    from slackclient import SlackClient as BaseSlackClient  # type: ignore
+
+    class SlackResponse(object):
+        """Wrap Python 2 SlackClient dict to mimic SlackResponse"""
+        def __init__(self, data):
+            self.data = data
+
+    class SlackClientWrapper(object):
+        def __init__(self, token):
+            self._client = BaseSlackClient(token)
+
+        def chat_postMessage(self, **kwargs):
+            result = self._client.api_call("chat.postMessage", **kwargs)
+            return SlackResponse(result)
+
+
+class Output(output.Output):
+    """
+    Slack webhook output plugin.
+    """
+
+    def start(self):
+        self.slack_channel = CONFIG.get('output_slack', 'channel')
+        self.slack_token = CONFIG.get('output_slack', 'token')
+        self.delay = CONFIG.getfloat('output_slack', 'delay', fallback=1.2)
+        self.sc = SlackClientWrapper(self.slack_token)
+        self.last_sent = 0
+        self.requests_list = []
+
+    def stop(self):
+        pass
+
+    def write(self, event):
+        operation = event['operation'].lower()
+
+        message = '[{} UTC] [PGSQLPot on {} ({})]: {}'.format(
+            strftime('%Y-%m-%d %H:%M:%S', gmtime(event['unixtime'])),
+            event['sensor'], event['session'], operation.capitalize()
+        )
+        if operation == 'unknown':
+            message += ' operation: "{}"'.format(event['username'])
+        elif operation == 'command':
+            message += ' "{}'.format(event['command'])
+            if event['args']:
+                message += ' {}'.format(event['args'])
+            message += '"'
+        message += ' from {}:{}'.format(event['src_ip'], event['dst_port'])
+        if operation == 'login':
+            message += ', username: "{}", password: "{}"'.format(
+                event['username'], event['password']
+            )
+        message += '.'
+
+        self.requests_list.append(message)
+        self._drain()
+
+    def _drain(self):
+        """
+        Send the next queued message if the rate-limit window has elapsed.
+        If messages remain, schedule another drain after self.delay seconds.
+        """
+        if not self.requests_list:
+            return
+        now = time()
+        elapsed = now - self.last_sent
+        if elapsed < self.delay:
+            # Re-schedule after the remainder of the delay window
+            deferLater(reactor, self.delay - elapsed, self._drain)
+            return
+        self.last_sent = now
+        message = self.requests_list.pop(0)
+        self.sc.chat_postMessage(channel=self.slack_channel, text=message)
+        if self.requests_list:
+            deferLater(reactor, self.delay, self._drain)