pgsqlpot 2.0.0__py2.py3-none-any.whl

output_plugins/discord.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Simple Discord webhook logger
+"""
+
+from __future__ import absolute_import
+
+from io import BytesIO
+from json import dumps, loads
+from time import gmtime, strftime, time
+
+from core import output
+from core.config import CONFIG
+from core.tools import decode, to_bytes
+
+from twisted.internet import reactor
+from twisted.internet.ssl import ClientContextFactory
+from twisted.internet.task import deferLater
+from twisted.python.log import msg
+from twisted.web.client import (
+    Agent,
+    FileBodyProducer,
+    HTTPConnectionPool,
+    _HTTP11ClientFactory,
+    readBody
+)
+from twisted.web.http_headers import Headers
+
+
+class WebClientContextFactory(ClientContextFactory):
+    def getContext(self, hostname, port):
+        return ClientContextFactory.getContext(self)
+
+
+class QuietHTTP11ClientFactory(_HTTP11ClientFactory):
+    noisy = False
+
+
+class Output(output.Output):
+
+    def start(self):
+        self.url = to_bytes(CONFIG.get('output_discord', 'url'))
+        self.delay = CONFIG.getfloat('output_discord', 'delay', fallback=2.0)
+        contextFactory = WebClientContextFactory()
+        pool = HTTPConnectionPool(reactor)
+        pool._factory = QuietHTTP11ClientFactory
+        self.agent = Agent(reactor, contextFactory=contextFactory, pool=pool)
+        self.last_sent = 0
+        self.requests_list = []
+
+    def stop(self):
+        pass
+
+    def write(self, event):
+        operation = event['operation'].lower()
+
+        message = '__New event__\n'
+        message += '[{} UTC] [PGSQLPot on {} ({})]: {}'.format(
+            strftime('%Y-%m-%d %H:%M:%S', gmtime(event['unixtime'])), event['sensor'],
+            event['session'], operation.capitalize()
+        )
+
+        if operation == 'unknown':
+            message += ' operation: "{}"'.format(event['username'])
+
+        message += ' from {}:{}'.format(event['src_ip'], event['dst_port'])
+
+        if operation == 'login':
+            message += ', username: "{}", password: "{}"'.format(event['username'], event['password'])
+        message += '.\n'
+
+        self.requests_list.append({'content': message})
+        self._drain()
+
+    def _drain(self):
+        """
+        Send the next queued entry if the rate-limit window has elapsed.
+        If entries remain, schedule another drain after self.delay seconds.
+        """
+        if not self.requests_list:
+            return
+        now = time()
+        elapsed = now - self.last_sent
+        if elapsed < self.delay:
+            deferLater(reactor, self.delay - elapsed, self._drain)
+            return
+        self.last_sent = now
+        entry = self.requests_list.pop(0)
+        d = self._post(entry)
+        if self.requests_list:
+            deferLater(reactor, self.delay, self._drain)
+        return d
+
+    def _post(self, entry):
+
+        def cbBody(body):
+            return processResult(body)
+
+        def cbPartial(failure):
+            """
+            Google HTTP Server does not set Content-Length. Twisted marks it as partial
+            """
+            failure.printTraceback()
+            return processResult(failure.value)
+
+        def cbResponse(response):
+            if response is None or response.code in [200, 201, 204]:
+                return
+            msg('Discord error: {} {}'.format(response.code, decode(response.phrase)))
+            d = readBody(response)
+            d.addCallback(cbBody)
+            d.addErrback(cbPartial)
+            return d
+
+        def cbError(failure):
+            failure.printTraceback()
+
+        def processResult(result):
+            if result:
+                try:
+                    j = loads(result)
+                    msg('Discord response: {}'.format(j.get('message', '')))
+                except Exception:
+                    pass
+
+        headers = Headers({b'Content-Type': [b'application/json']})
+        body = FileBodyProducer(BytesIO(to_bytes(dumps(entry, sort_keys=True))))
+        d = self.agent.request(b'POST', self.url, headers, body)
+        d.addCallback(cbResponse)
+        d.addErrback(cbError)
+        return d

output_plugins/elastic.py

@@ -0,0 +1,137 @@
+
+from __future__ import absolute_import
+
+from sys import version_info
+
+from core import output
+from core.config import CONFIG
+
+from twisted.python.log import msg
+
+PY2 = version_info[0] < 3
+
+if PY2:
+    import elasticsearch    # type: ignore
+    from elasticsearch import Elasticsearch, NotFoundError  # type: ignore
+    version_module = elasticsearch
+    version_attr = '__version__'
+else:
+    import elasticsearch8
+    from elasticsearch8 import Elasticsearch, NotFoundError
+    version_module = elasticsearch8
+    version_attr = '__versionstr__'
+
+
+class Output(output.Output):
+
+    def start(self):
+        host = CONFIG.get('output_elastic', 'host', fallback='localhost')
+        port = CONFIG.getint('output_elastic', 'port', fallback=9200)
+        username = CONFIG.get('output_elastic', 'username', fallback=None)
+        password = CONFIG.get('output_elastic', 'password', fallback=None)
+        use_ssl = CONFIG.getboolean('output_elastic', 'ssl', fallback=False)
+        verify_certs = CONFIG.getboolean('output_elastic', 'verify_certs', fallback=False)
+        ca_certs = CONFIG.get('output_elastic', 'ca_certs', fallback=None)
+        self.index = CONFIG.get('output_elastic', 'index', fallback='pgsqlpot')
+        self.pipeline = CONFIG.get('output_elastic', 'pipeline', fallback='geoip')
+
+        scheme = 'https' if use_ssl else 'http'
+        hosts = [{'host': host, 'port': port, 'scheme': scheme}]
+        kwargs = {}
+        if username and password:
+            kwargs['http_auth' if PY2 else 'basic_auth'] = (username, password)
+        if use_ssl:
+            kwargs['verify_certs'] = verify_certs
+            if verify_certs and ca_certs:
+                kwargs['ca_certs'] = ca_certs
+
+        # Create client
+        self.es = Elasticsearch(hosts=hosts, **kwargs)
+
+        # Detect major version from module attribute
+        ver_attr_value = getattr(version_module, version_attr)
+        if isinstance(ver_attr_value, tuple):
+            # Python 2.7 elasticsearch 7.x
+            self.es_ver = ver_attr_value[0]
+        else:
+            # Python 3.6+ elasticsearch8
+            self.es_ver = int(str(ver_attr_value).split('.')[0])
+
+        # Test connection
+        try:
+            info = self.es.info()
+            msg('output_elastic: connected to ES version {}'.format(info['version']['number']))
+        except Exception as e:
+            msg('output_elastic: connection failed: {}'.format(repr(e)))
+            raise
+
+        # Ensure index exists
+        if not self.es.indices.exists(index=self.index):
+            self.es.indices.create(index=self.index)
+
+        # Setup geoip
+        if self.pipeline == 'geoip':
+            self._setup_geoip_mapping()
+            self._setup_geoip_pipeline()
+
+    def _setup_geoip_mapping(self):
+        try:
+            body = {
+                'properties': {
+                    'geo': {
+                        'properties': {
+                            'location': {
+                                'type': 'geo_point'
+                            }
+                        }
+                    }
+                }
+            }
+            if self.es_ver >= 8:
+                self.es.indices.put_mapping(index=self.index, properties=body['properties'])
+            else:
+                self.es.indices.put_mapping(index=self.index, body=body)
+        except Exception as e:
+            msg('output_elastic: mapping setup failed: {}'.format(repr(e)))
+
+    def _setup_geoip_pipeline(self):
+        try:
+            self.es.ingest.get_pipeline(id=self.pipeline)
+        except NotFoundError:
+            body = {
+                'description': 'Add geoip info',
+                'processors': [
+                    {
+                        'geoip': {
+                            'field': 'src_ip',
+                            'target_field': 'geo',
+                            'database_file': 'GeoLite2-City.mmdb'
+                        }
+                    }
+                ]
+            }
+            try:
+                if self.es_ver >= 8:
+                    self.es.ingest.put_pipeline(
+                        id=self.pipeline,
+                        processors=body['processors'],
+                        description=body['description']
+                    )
+                else:
+                    self.es.ingest.put_pipeline(id=self.pipeline, body=body)
+            except Exception as e:
+                msg('output_elastic: pipeline creation failed: {}'.format(repr(e)))
+
+    def stop(self):
+        pass
+
+    def write(self, entry):
+        try:
+            if self.es_ver >= 8:
+                self.es.index(index=self.index, document=entry, pipeline=self.pipeline)
+            else:
+                self.es.index(index=self.index, body=entry, pipeline=self.pipeline)
+        except Exception as e:
+            msg('output_elastic: write failed: {}'.format(repr(e)))
+            if hasattr(e, 'info'):
+                msg('output_elastic info: {}'.format(e.info))

output_plugins/hpfeed.py

@@ -0,0 +1,43 @@
+
+from __future__ import absolute_import
+
+from json import dumps
+
+from core import output
+from core.config import CONFIG
+from core.tools import to_bytes
+
+from hpfeeds.twisted import ClientSessionService
+
+from twisted.internet import endpoints, reactor, ssl
+
+class Output(output.Output):
+
+    def start(self):
+        self.channel = CONFIG.get('output_hpfeed', 'channel', fallback='pgsqlpot')
+
+        if CONFIG.has_option('output_hpfeed', 'endpoint'):
+            endpoint = CONFIG.get('output_hpfeed', 'endpoint')
+        else:
+            server = CONFIG.get('output_hpfeed', 'server')
+            port = CONFIG.getint('output_hpfeed', 'port')
+
+            if CONFIG.has_option('output_hpfeed', 'tlscert'):
+                with open(CONFIG.get('output_hpfeed', 'tlscert')) as fp:
+                    authority = ssl.Certificate.loadPEM(fp.read())
+                options = ssl.optionsForClientTLS(server, authority)
+                endpoint = endpoints.SSL4ClientEndpoint(reactor, server, port, options)
+            else:
+                endpoint = endpoints.HostnameEndpoint(reactor, server, port)
+
+        ident = CONFIG.get('output_hpfeed', 'identifier', raw=True)
+        secret = CONFIG.get('output_hpfeed', 'secret', raw=True)
+
+        self.client = ClientSessionService(endpoint, ident, secret)
+        self.client.startService()
+
+    def stop(self):
+        self.client.stopService()
+
+    def write(self, event):
+        self.client.publish(self.channel, to_bytes(dumps(event, sort_keys=True)))

output_plugins/influx2.py

@@ -0,0 +1,66 @@
+
+from __future__ import absolute_import
+
+from datetime import datetime
+
+from core import output
+from core.config import CONFIG
+
+from influxdb_client import InfluxDBClient, Point, WritePrecision
+from influxdb_client.client.write_api import SYNCHRONOUS
+
+from pytz import timezone
+
+from twisted.python.log import msg
+
+
+class Output(output.Output):
+
+    def start(self):
+        host = CONFIG.get('output_influx2', 'host')
+        token = CONFIG.get('output_influx2', 'token', raw=True)
+
+        self.client = None
+        try:
+            self.client = InfluxDBClient(url=host, token=token)
+        except Exception as e:
+            msg("output_influx2: I/O error: '{}'".format(e))
+            return
+
+        if self.client is None:
+            msg('output_influx2: cannot instantiate client!')
+            return
+
+        self.org = CONFIG.get('output_influx2', 'org')
+        self.bucket = CONFIG.get('output_influx2', 'bucket', fallback='pgsqlpot')
+        self.write_api = self.client.write_api(write_options=SYNCHRONOUS)
+
+    def stop(self):
+        pass
+
+    def write(self, event):
+        m = Point(event['eventid'].replace('.', '_').replace('-', '_'))\
+            .tag('src_ip', event['src_ip'])\
+            .time(datetime.fromtimestamp(event['unixtime'], tz=timezone('UTC')), WritePrecision.NS)
+
+        fields = [
+            'src_port',
+            'dst_ip',
+            'dst_port',
+            'eventid',
+            'operation',
+            'sensor'
+        ]
+        operation = event['operation'].lower()
+        if operation == 'login':
+            fields.append('username')
+            fields.append('password')
+        elif operation == 'command':
+            fields.append('command')
+            fields.append('args')
+
+        for key in fields:
+            if key in event:
+                m.field(key, event[key])
+
+        self.write_api.write(bucket=self.bucket, org=self.org, record=m)

output_plugins/jsonlog.py

@@ -0,0 +1,36 @@
+
+from __future__ import absolute_import
+
+from copy import deepcopy
+from json import dump
+from os import linesep
+from os.path import basename, dirname
+
+from core import output
+from core.tools import mkdir
+from core.config import CONFIG
+from core.logfile import HoneypotDailyLogFile
+
+class Output(output.Output):
+
+    def start(self):
+        self.epoch_timestamp = CONFIG.getboolean('output_jsonlog', 'epoch_timestamp', fallback=False)
+        fn = CONFIG.get('output_jsonlog', 'logfile')
+        dirs = dirname(fn)
+        base = basename(fn)
+        mkdir(dirs)
+        self.outfile = HoneypotDailyLogFile(base, dirs, defaultMode=0o664)
+
+    def stop(self):
+        self.outfile.flush()
+
+    def write(self, event):
+        if not self.epoch_timestamp:
+            # We need 'unixtime' value in some other plugins
+            event_dump = deepcopy(event)
+            event_dump.pop('unixtime', None)
+        else:
+            event_dump = event
+        dump(event_dump, self.outfile, separators=(',', ':'))
+        self.outfile.write(linesep)
+        self.outfile.flush()

output_plugins/kafka.py

@@ -0,0 +1,57 @@
+
+from __future__ import absolute_import
+
+from json import dumps
+
+from core import output
+from core.config import CONFIG
+from core.tools import to_bytes
+
+from confluent_kafka import Producer
+
+from twisted.python.log import msg
+
+
+class Output(output.Output):
+
+    def start(self):
+        self.topic = CONFIG.get('output_kafka', 'topic', fallback='pgsqlpot')
+        self.debug = CONFIG.getboolean('output_kafka', 'debug', fallback=False)
+
+        self.producer = Producer({
+            'bootstrap.servers': '{}:{}'.format(
+                CONFIG.get('output_kafka', 'host', fallback='localhost'),
+                CONFIG.get('output_kafka', 'port', fallback=9092)
+            ),
+            #'sasl.mechanism': 'SCRAM-SHA-256',
+            #'security.protocol': 'SASL_SSL',
+            #'sasl.username': CONFIG.get('output_kafka', 'username'),
+            #'sasl.password': CONFIG.get('output_kafka', 'password')
+        })
+
+    def stop(self):
+        self.producer.flush()
+
+    def write(self, event):
+        self.postentry(event)
+
+    def delivery_callback(self, err, message):
+        if err:
+            msg('output_kafka: Message failed delivery: {}.'.format(err))
+        else:
+            if self.debug:
+                msg('output_kafka: Message delivered to {} [{}] @ {}'.format(
+                    message.topic(), message.partition(), message.offset()
+                ))
+
+    def postentry(self, event):
+        try:
+            self.producer.produce(
+                self.topic,
+                to_bytes(dumps(event, sort_keys=True)),
+                callback=self.delivery_callback
+            )
+            self.producer.poll(0)
+            self.producer.flush()
+        except Exception as e:
+            msg('Kafka error: {}'.format(e))

output_plugins/localsyslog.py

@@ -0,0 +1,66 @@
+
+from __future__ import absolute_import
+
+import syslog
+
+from core import output
+from core.config import CONFIG
+
+
+def formatCef(logentry):
+    """
+    Take logentry and turn into CEF string
+    """
+    # Jan 18 11:07:53 host CEF:Version|Device Vendor|Device Product|
+    # Device Version|Signature ID|Name|Severity|[Extension]
+    cefVendor = 'Bontchev'
+    cefProduct = 'PGSQLPot'
+    cefVersion = '1.0'
+    cefSignature = logentry['eventid']
+    cefName = logentry['eventid']
+    cefSeverity = '5'
+
+    cefExtensions = {
+        'deviceExternalId': logentry['sensor'],
+        'op': logentry['operation'],
+        'src': logentry['src_ip'],
+        'spt': logentry['src_port'],
+        'dpt': logentry['dst_port'],
+        'dst': logentry['dst_ip'],
+        'proto': 'tcp'
+    }
+    if logentry['operation'].lower() == 'login':
+        cefExtensions['usr'] = logentry['username']
+        cefExtensions['pwd'] = logentry['password']
+
+    cefList = []
+    for key in list(cefExtensions.keys()):
+        value = str(cefExtensions[key])
+        cefList.append('{}={}'.format(key, value))
+
+    cefExtension = ' '.join(cefList)
+
+    cefString = 'CEF:0|' + \
+                cefVendor + '|' + \
+                cefProduct + '|' + \
+                cefVersion + '|' + \
+                cefSignature + '|' + \
+                cefName + '|' + \
+                cefSeverity + '|' + \
+                cefExtension
+
+    return cefString
+
+
+class Output(output.Output):
+
+    def start(self):
+        facilityString = CONFIG.get('output_localsyslog', 'facility', fallback='USER')
+        facility = vars(syslog)['LOG_' + facilityString]
+        syslog.openlog(logoption=syslog.LOG_PID, facility=facility)
+
+    def stop(self):
+        pass
+
+    def write(self, event):
+        syslog.syslog(syslog.LOG_INFO, formatCef(event))

output_plugins/mongodb.py

@@ -0,0 +1,83 @@
+
+from __future__ import absolute_import
+
+from core import output
+from core.config import CONFIG
+from core.tools import geolocate
+
+from warnings import filterwarnings
+from cryptography.utils import CryptographyDeprecationWarning
+filterwarnings('ignore', category=CryptographyDeprecationWarning)
+
+from pymongo import MongoClient
+from pymongo.server_api import ServerApi
+from geoip2.database import Reader
+
+from twisted.python.log import msg
+
+
+class Output(output.Output):
+
+    def insert_one(self, collection, event):
+        try:
+            return collection.insert_one(event).inserted_id
+        except Exception as e:
+            msg('output_mongodb: Error: {}'.format(e))
+            return None
+
+    def start(self):
+        host = CONFIG.get('output_mongodb', 'host', fallback='localhost')
+        username = CONFIG.get('output_mongodb', 'username', fallback='', raw=True)
+        password = CONFIG.get('output_mongodb', 'password', fallback='', raw=True)
+        db_name = CONFIG.get('output_mongodb', 'database', fallback='pgsqlpot')
+        db_addr = CONFIG.get('output_mongodb', 'connection_string')
+        db_addr = db_addr.format(username, password, host)
+
+        self.geoip = CONFIG.getboolean('output_mongodb', 'geoip', fallback=True)
+
+        try:
+            self.mongo_client = MongoClient(db_addr, server_api=ServerApi('1'))
+            self.mongo_db = self.mongo_client[db_name]
+            self.col_events = self.mongo_db['connections']
+            if self.geoip:
+                self.col_geolocation = self.mongo_db['geolocation']
+        except Exception as e:
+            msg('output_mongodb: Error: {}'.format(e))
+
+        if self.geoip:
+            geoipdb_city_path = CONFIG.get('output_mongodb', 'geoip_citydb', fallback='data/GeoLite2-City.mmdb')
+            geoipdb_asn_path = CONFIG.get('output_mongodb', 'geoip_asndb', fallback='data/GeoLite2-ASN.mmdb')
+            try:
+                self.reader_city = Reader(geoipdb_city_path)
+            except Exception:
+                self.reader_city = None
+                msg('Failed to open City GeoIP database {}'.format(geoipdb_city_path))
+            try:
+                self.reader_asn = Reader(geoipdb_asn_path)
+            except Exception:
+                self.reader_asn = None
+                msg('Failed to open ASN GeoIP database {}'.format(geoipdb_asn_path))
+
+    def stop(self):
+        self.mongo_client.close()
+        if self.geoip:
+            if self.reader_city is not None:
+                self.reader_city.close()
+            if self.reader_asn is not None:
+                self.reader_asn.close()
+
+    def write(self, event):
+        self.insert_one(self.col_events, event)
+        if self.geoip:
+            remote_ip = event['src_ip']
+            if not self.col_geolocation.find_one({'ip': remote_ip}):
+                country, country_code, city, org, asn_num = geolocate(remote_ip, self.reader_city, self.reader_asn)
+                geo_entry = {
+                    'ip': remote_ip,
+                    'country': country,
+                    'country_code': country_code,
+                    'city': city,
+                    'org': org,
+                    'asn': asn_num
+                }
+                self.insert_one(self.col_geolocation, geo_entry)