elasticpot 2.0.1.dev0__tar.gz → 2.0.3__tar.gz

{elasticpot-2.0.1.dev0 → elasticpot-2.0.3}/CHANGELOG.md

@@ -5,15 +5,60 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [2.0.1]
+## [2.0.3]
 
-### Added in version 2.0.1
+### Added in version 2.0.3
 
-* Nothing so far
+* Nothing
 
-### Changed in version 2.0.1
+### Changed in version 2.0.3
 
 * Increased the version number
+* The `restart` command wasn't working correctly on Windows due to a race
+  condition. Fixed.
+* Fixed a problem in the MySQL plugin that made it unresponsive under high
+  traffic
+* **Warning!** The MySQL, PostgreSQL, and SQLite3 schemas have changed.
+  Specifically, in the `urls` table, the key `path` is now `UNIQUE` and so are
+  the keys `user_agent` in `user_agents`, `content_type` in `content_types`,
+  `accept_language` in `accept_language`, and `message` in `message`. If you are
+  already using the corresponding output plugins and have gathered some data
+  into these tables, you *must* remove any duplicated rows in these tables
+  (leave only the one with the smallest `id`) and to modify these keys and they
+  indices to be unique or the plugins will cause errors. For instance, something
+  like
+
+```MySQL
+DELETE t1 FROM `urls` t1
+  INNER JOIN `urls` t2
+  WHERE t1.`path` = t2.`path` AND t1.`id` > t2.`id`;
+ALTER TABLE `path` DROP INDEX `path`;
+ALTER TABLE `path` ADD UNIQUE (`path`);
+...
+```
+
+## [2.0.2]
+
+### Added in version 2.0.2
+
+* Nothing
+
+### Changed in version 2.0.2
+
+* Increased the version number
+* The `datadog`, `discord`, `nlcvapi`, and `telegram` plugins now use a secure
+  connection (HTTPS) by default
+* The `elastic` plugin now warns if the `ssl` is set while certificate
+  verification (`verify_certs`) is off
+* The `couch` plugin now uses authentication mechanism that does not pass the
+  username and password in the URL
+
+## [2.0.1]
+
+### Changed in version 2.0.1
+
+* The honeypot was released with an incorrect version - "2.0.1-dev0" instead of
+  "2.0.0". Fixed by changing it and releasing version 2.0.1
 
 ## [2.0.0]
 

{elasticpot-2.0.1.dev0 → elasticpot-2.0.3}/MANIFEST.in

@@ -1,9 +1,8 @@
-include honeypot.py
-include setup.cfg
+include CHANGELOG.md
 include LICENSE
 include README.md
-include CHANGELOG.md
-graft elasticpot
+include honeypot.py
 graft core
+graft elasticpot
 graft output_plugins
 prune elasticpot/data/test/develop

{elasticpot-2.0.1.dev0/elasticpot.egg-info → elasticpot-2.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: elasticpot
-Version: 2.0.1.dev0
+Version: 2.0.3
 Summary: An ElasticSearch Honeypot
 Home-page: https://gitlab.com/bontchev/elasticpot
 Author: Vesselin Bontchev
@@ -43,11 +43,13 @@ Requires-Dist: twisted>=21; python_version >= "3"
 Provides-Extra: couchdb
 Requires-Dist: couchdb; extra == "couchdb"
 Provides-Extra: datadog
+Requires-Dist: certifi; extra == "datadog"
 Requires-Dist: cryptography<=2.8; python_version < "3" and extra == "datadog"
 Requires-Dist: pyOpenSSL<=18.0.0; python_version < "3" and extra == "datadog"
 Requires-Dist: cryptography; python_version >= "3" and extra == "datadog"
 Requires-Dist: pyOpenSSL; python_version >= "3" and extra == "datadog"
 Provides-Extra: discord
+Requires-Dist: certifi; extra == "discord"
 Provides-Extra: elastic
 Requires-Dist: elasticsearch<=7.13; python_version < "3" and extra == "elastic"
 Requires-Dist: numpy<=1.16.6; python_version < "3" and extra == "elastic"
@@ -71,6 +73,7 @@ Provides-Extra: mysql
 Requires-Dist: PyMySQL; python_version < "3" and extra == "mysql"
 Requires-Dist: mysqlclient>=1.3.12; python_version >= "3" and extra == "mysql"
 Provides-Extra: nlcvapi
+Requires-Dist: certifi; extra == "nlcvapi"
 Requires-Dist: pyOpenSSL<=18.0.0; python_version < "3" and extra == "nlcvapi"
 Requires-Dist: pyOpenSSL; python_version >= "3" and extra == "nlcvapi"
 Provides-Extra: postgres
@@ -87,12 +90,14 @@ Requires-Dist: slack-sdk; python_version >= "3" and extra == "slack"
 Provides-Extra: socketlog
 Provides-Extra: sqlite
 Provides-Extra: telegram
+Requires-Dist: certifi; extra == "telegram"
 Provides-Extra: textlog
 Provides-Extra: xmpp
 Requires-Dist: xmpppy>=0.7.3; extra == "xmpp"
 Provides-Extra: all
 Requires-Dist: Automat<20; python_version < "3" and extra == "all"
 Requires-Dist: PyMySQL; python_version < "3" and extra == "all"
+Requires-Dist: certifi; extra == "all"
 Requires-Dist: confluent-kafka; python_version >= "3" and extra == "all"
 Requires-Dist: confluent-kafka<1.0; python_version < "3" and extra == "all"
 Requires-Dist: couchdb; extra == "all"

elasticpot-2.0.3/core/httpclient.py

@@ -0,0 +1,71 @@
+from __future__ import absolute_import
+
+from io import open
+
+from twisted.internet.ssl import Certificate, trustRootFromCertificates
+from twisted.python.log import msg
+from twisted.web.client import Agent, BrowserLikePolicyForHTTPS
+
+
+class _LegacyWebClientContextFactory(object):
+    @staticmethod
+    def build():
+        from twisted.internet.ssl import ClientContextFactory
+
+        class WebClientContextFactory(ClientContextFactory):
+            def getContext(self, hostname, port):
+                return ClientContextFactory.getContext(self)
+
+        return WebClientContextFactory()
+
+
+def _load_trust_root_from_pem_bundle(pem_path):
+    with open(pem_path, 'r', encoding='utf-8') as fh:
+        content = fh.read()
+    certs = []
+    end_marker = '-----END CERTIFICATE-----'
+    for chunk in content.split(end_marker):
+        chunk = chunk.strip()
+        if not chunk:
+            continue
+        pem = chunk + '\n' + end_marker + '\n'
+        certs.append(Certificate.loadPEM(pem))
+    if not certs:
+        return None
+    return trustRootFromCertificates(certs)
+
+
+def _trust_root(ca_certs):
+    if ca_certs:
+        try:
+            return _load_trust_root_from_pem_bundle(ca_certs)
+        except Exception as e:
+            msg('Failed to load CA bundle {}: {}'.format(ca_certs, e))
+            return None
+
+    try:
+        import certifi
+        return _load_trust_root_from_pem_bundle(certifi.where())
+    except Exception:
+        return None
+
+
+def create_http_agent(reactor, pool, plugin_name, verify_tls=True, ca_certs=None):
+    """
+    Build a Twisted Agent with sane HTTPS defaults.
+    verify_tls=True uses BrowserLikePolicyForHTTPS for cert + hostname checks.
+    verify_tls=False falls back to a legacy context factory for compatibility.
+    """
+    if verify_tls:
+        return Agent(
+            reactor,
+            contextFactory=BrowserLikePolicyForHTTPS(trustRoot=_trust_root(ca_certs)),
+            pool=pool
+        )
+
+    msg('{}: TLS certificate verification is disabled.'.format(plugin_name))
+    return Agent(
+        reactor,
+        contextFactory=_LegacyWebClientContextFactory.build(),
+        pool=pool
+    )

{elasticpot-2.0.1.dev0 → elasticpot-2.0.3}/core/tools.py

@@ -15,13 +15,15 @@ try:
     from urllib.request import urlopen
     from urllib.parse import urlsplit, urlunsplit
 except ImportError:
-    from urllib import urlopen
+    from urllib2 import urlopen # type: ignore
     from urlparse import urlsplit, urlunsplit   # type: ignore
 
 
 if version_info[0] >= 3:
     def decode(x):
-        return x.decode('utf-8')
+        if isinstance(x, bytes):
+            return x.decode('utf-8')
+        return x
     def encode(x):
         return x.encode()
     def unicode(x):
@@ -42,10 +44,7 @@ def get_utc_time(unix_time):
 
 def get_public_ip(ip_reporter):
     try:
-        if version_info[0] < 3:
-            return urlopen(ip_reporter).read().decode('latin1', errors='replace').encode('utf-8')
-        else:
-            return urlopen(ip_reporter).read()
+        return decode(urlopen(ip_reporter, timeout=10).read())
     except:
         return '127.0.0.1'
 

{elasticpot-2.0.1.dev0 → elasticpot-2.0.3}/elasticpot/cli.py

@@ -48,6 +48,7 @@ from os.path import (
 from re import search, IGNORECASE
 from shutil import copy2
 import sys
+from time import sleep
 
 from core.paths import bundled, get_workdir, workdir_path
 
@@ -62,8 +63,27 @@ def _ensure_dir(p):
 
 
 def _file_differs(a, b):
-    with open(a, 'r') as fa, open(b, 'r') as fb:
-        return fa.read() != fb.read()
+    """Return True if files differ. Ignores line-ending differences for text files."""
+    # Try text mode first (normalizes \r\n, \r, \n)
+    try:
+        with open(a, 'r') as fa, open(b, 'r') as fb:
+            while True:
+                line_a = fa.readline()
+                line_b = fb.readline()
+                if line_a != line_b:
+                    return True
+                if not line_a:  # EOF on both
+                    return False
+    except UnicodeDecodeError:
+        # Binary fallback: compare in chunks without loading whole file
+        with open(a, 'rb') as fa, open(b, 'rb') as fb:
+            while True:
+                chunk_a = fa.read(8192)
+                chunk_b = fb.read(8192)
+                if chunk_a != chunk_b:
+                    return True
+                if not chunk_a:
+                    return False
 
 
 def _copy_if_missing(src, dst, label=None):
@@ -363,6 +383,7 @@ def cmd_stop(args):
         return
 
     print('Stopping the honeypot (PID {})... '.format(pid), end='')
+    sys.stdout.flush()
 
     if name == 'nt':
         _stop_windows(pid, pidfile)
@@ -372,7 +393,6 @@ def cmd_stop(args):
 
 def _stop_posix(pid, pidfile):
     from signal import SIGKILL, SIGTERM
-    from time import sleep
     kill(pid, SIGTERM)
     for _ in range(60):
         sleep(1)
@@ -406,11 +426,28 @@ def _stop_windows(pid, pidfile):
     finally:
         _devnull.close()
 
-    if ret == 0 or not _pid_running(pid):
-        print('Stopped.')
+    if ret != 0 and _pid_running(pid):
+        print()
+        print(
+            'Warning: taskkill returned {}, process may still be running.'.format(ret)
+        )
+        remove(pidfile)
+        return
+
+    # taskkill /F calls TerminateProcess() which is asynchronous: the call
+    # returns before the kernel has finished tearing down the process and
+    # releasing its resources (sockets, handles, etc.).  Poll until the
+    # process is truly gone so that a subsequent 'start' does not race with
+    # the old process still holding port 3389.
+    for _ in range(30):
+        if not _pid_running(pid):
+            break
+        sleep(0.5)
     else:
         print()
-        print('Warning: taskkill returned {}, process may still be running.'.format(ret))
+        print('Warning: process {} did not exit within 15 s.'.format(pid))
+
+    print('Stopped.')
     remove(pidfile)
 
 

{elasticpot-2.0.1.dev0 → elasticpot-2.0.3}/elasticpot/data/Dockerfile

@@ -51,6 +51,7 @@ USER elasticpot
 #   docker run \
 #     -v /path/to/honeypot.cfg:/elasticpot/etc/honeypot.cfg \
 #     -v /path/to/data:/elasticpot/data \
+#     -v /path/to/log:/elasticpot/log \
 #     -p 9200:9200 elasticpot
 #
 CMD ["elasticpot", "run"]

{elasticpot-2.0.1.dev0 → elasticpot-2.0.3}/elasticpot/data/docs/INSTALL.md

@@ -168,7 +168,7 @@ working) and `Dockerfile` into the working directory.
 The configuration for the honeypot is stored in `etc/honeypot.cfg.base` and
 `etc/honeypot.cfg`. Both files are read on startup but entries from
 `etc/honeypot.cfg` take precedence. The `.base` file contains the default
-settings and should not be edited — it may be overwritten by upgrades. All
+settings and should not be edited - it may be overwritten by upgrades. All
 your customisations should go into `etc/honeypot.cfg`.
 
 To run with a standard configuration there is no need to change anything.
@@ -418,7 +418,7 @@ $ source ~/elasticpot-env/bin/activate
 
 where `[plugin_list]` is as explained above.
 
-Note that `elasticpot init` is safe to re-run — it never overwrites files that
+Note that `elasticpot init` is safe to re-run - it never overwrites files that
 you have already edited or created (such as `etc/honeypot.cfg`). It only
 copies files that are missing, so any new defaults added by an upgrade are
 picked up automatically.

{elasticpot-2.0.1.dev0 → elasticpot-2.0.3}/elasticpot/data/docs/INSTALLWIN.md

@@ -33,7 +33,7 @@ programs (if they are not already present):
 
 - **Database server** (optional). If you want the honeypot to send the data
   it collects to a local database server (e.g., MySQL), make sure to install
-  it — again, for all users and not just for the current one.
+  it - again, for all users and not just for the current one.
 
 - The latest version of the **VS C++ redistributable** (required by some of
   the database output plugins, e.g., for MySQL). Version 14 or higher should
@@ -188,7 +188,7 @@ working directory.
 The configuration for the honeypot is stored in `etc\honeypot.cfg.base` and
 `etc\honeypot.cfg`. Both the `*.cfg.base` and the `*.cfg` files are read on
 startup but entries from the `*.cfg` files take precedence. The `*.base` files
-contain the default settings and should not be edited — they may be overwritten
+contain the default settings and should not be edited - they may be overwritten
 by future updates. All your customisations should go into the `*.cfg` files.
 
 To run with a standard configuration there is no need to change anything.
@@ -429,7 +429,7 @@ PS C:\elasticpot-workdir> C:\elasticpot-env\Scripts\activate.ps1
 
 where `[plugin_list]` is as explained above.
 
-Note that `elasticpot init` is safe to re-run — it never overwrites files that
+Note that `elasticpot init` is safe to re-run - it never overwrites files that
 you have already edited or created (such as `etc\honeypot.cfg`). It only
 copies files that are missing, so any new defaults added by an upgrade are
 picked up automatically.

{elasticpot-2.0.1.dev0 → elasticpot-2.0.3}/elasticpot/data/docs/mysql/mysql.sql

@@ -22,7 +22,8 @@ CREATE TABLE IF NOT EXISTS `connections` (
 CREATE TABLE IF NOT EXISTS `urls` (
   `id` int NOT NULL AUTO_INCREMENT,
   `path` varchar(255) DEFAULT NULL,
-  PRIMARY KEY (`id`)
+  PRIMARY KEY (`id`),
+  UNIQUE (`path`)
 );
 
 CREATE TABLE IF NOT EXISTS `payloads` (
@@ -36,25 +37,29 @@ CREATE TABLE IF NOT EXISTS `payloads` (
 CREATE TABLE IF NOT EXISTS `user_agents` (
   `id` int NOT NULL AUTO_INCREMENT,
   `user_agent` varchar(255) DEFAULT NULL,
-  PRIMARY KEY (`id`)
+  PRIMARY KEY (`id`),
+  UNIQUE (`user_agent`)
 );
 
 CREATE TABLE IF NOT EXISTS `content_types` (
   `id` int NOT NULL AUTO_INCREMENT,
   `content_type` varchar(255) DEFAULT NULL,
-  PRIMARY KEY (`id`)
+  PRIMARY KEY (`id`),
+  UNIQUE (`content_type`)
 );
 
 CREATE TABLE IF NOT EXISTS `accept_languages` (
   `id` int NOT NULL AUTO_INCREMENT,
   `accept_language` varchar(255) DEFAULT NULL,
-  PRIMARY KEY (`id`)
+  PRIMARY KEY (`id`),
+  UNIQUE (`accept_language`)
 );
 
 CREATE TABLE IF NOT EXISTS `messages` (
   `id` int NOT NULL AUTO_INCREMENT,
   `message` varchar(10) DEFAULT NULL,
-  PRIMARY KEY (`id`)
+  PRIMARY KEY (`id`),
+  UNIQUE (`message`)
 );
 
 CREATE TABLE IF NOT EXISTS `sensors` (

{elasticpot-2.0.1.dev0 → elasticpot-2.0.3}/elasticpot/data/docs/postgres/postgres.sql

@@ -24,6 +24,8 @@ CREATE TABLE IF NOT EXISTS urls (
   path varchar(255) DEFAULT NULL
 );
 
+CREATE UNIQUE INDEX path_idx ON urls (path);
+
 CREATE TABLE IF NOT EXISTS payloads (
   id SERIAL PRIMARY KEY,
   input varchar(3000) NOT NULL,
@@ -37,21 +39,29 @@ CREATE TABLE IF NOT EXISTS user_agents (
   user_agent varchar(255) DEFAULT NULL
 );
 
+CREATE UNIQUE INDEX user_agent_idx ON user_agents (user_agent);
+
 CREATE TABLE IF NOT EXISTS content_types (
   id SERIAL PRIMARY KEY,
   content_type varchar(255) DEFAULT NULL
 );
 
+CREATE UNIQUE INDEX content_type_idx ON content_types (content_type);
+
 CREATE TABLE IF NOT EXISTS accept_languages (
   id SERIAL PRIMARY KEY,
   accept_language varchar(255) DEFAULT NULL
 );
 
+CREATE UNIQUE INDEX accept_language_idx ON accept_languages (accept_language);
+
 CREATE TABLE IF NOT EXISTS messages (
   id SERIAL PRIMARY KEY,
   message varchar(10) DEFAULT NULL
 );
 
+CREATE UNIQUE INDEX message_idx ON messages (message);
+
 CREATE TABLE IF NOT EXISTS sensors (
   id SERIAL PRIMARY KEY,
   name varchar(255) DEFAULT NULL

{elasticpot-2.0.1.dev0 → elasticpot-2.0.3}/elasticpot/data/docs/sqlite3/sqlite3.sql

@@ -21,7 +21,8 @@ CREATE INDEX IF NOT EXISTS `ip2_idx` ON `connections` (`timestamp`, `ip`);
 
 CREATE TABLE IF NOT EXISTS `urls` (
   `id` INTEGER PRIMARY KEY,
-  `path` VARCHAR(255) DEFAULT NULL
+  `path` VARCHAR(255) DEFAULT NULL,
+  UNIQUE (`path`)
 );
 
 CREATE TABLE IF NOT EXISTS `payloads` (
@@ -33,22 +34,26 @@ CREATE TABLE IF NOT EXISTS `payloads` (
 
 CREATE TABLE IF NOT EXISTS `user_agents` (
   `id` INTEGER PRIMARY KEY,
-  `user_agent` VARCHAR(255) DEFAULT NULL
+  `user_agent` VARCHAR(255) DEFAULT NULL,
+  UNIQUE (`user_agent`)
 );
 
 CREATE TABLE IF NOT EXISTS `content_types` (
   `id` INTEGER PRIMARY KEY,
-  `content_type` VARCHAR(255) DEFAULT NULL
+  `content_type` VARCHAR(255) DEFAULT NULL,
+  UNIQUE (`content_type`)
 );
 
 CREATE TABLE IF NOT EXISTS `accept_languages` (
   `id` INTEGER PRIMARY KEY,
-  `accept_language` VARCHAR(255) DEFAULT NULL
+  `accept_language` VARCHAR(255) DEFAULT NULL,
+  UNIQUE (`accept_language`)
 );
 
 CREATE TABLE IF NOT EXISTS `messages` (
   `id` INTEGER PRIMARY KEY,
-  `message` VARCHAR(10) DEFAULT NULL
+  `message` VARCHAR(10) DEFAULT NULL,
+  UNIQUE (`message`)
 );
 
 CREATE TABLE IF NOT EXISTS `sensors` (