bhfutils 0.2.4__tar.gz → 0.2.6__tar.gz

{bhfutils-0.2.4 → bhfutils-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bhfutils
-Version: 0.2.4
+Version: 0.2.6
 Summary: Utilities that are used by any spider of Behoof project
 Home-page: https://behoof.app/
 Author: Teplygin Vladimir

{bhfutils-0.2.4 → bhfutils-0.2.6}/bhfutils/crawler/pipelines.py

@@ -109,13 +109,30 @@ class KafkaPipeline(object):
                                          bytes=my_bytes,
                                          backups=my_backups)
 
+        # Security settings are optional: a spider running inside the VPC talks to the
+        # PLAINTEXT listener and defines none of them, while one running outside needs
+        # SASL_SSL. Defaults keep existing projects working without touching them.
+        producer_kwargs = dict(bootstrap_servers=settings['KAFKA_HOSTS'],
+                               retries=3,
+                               linger_ms=settings['KAFKA_PRODUCER_BATCH_LINGER_MS'],
+                               buffer_memory=settings['KAFKA_PRODUCER_BUFFER_BYTES'],
+                               value_serializer=lambda m: m.encode('utf-8'),
+                               max_request_size=settings['KAFKA_PRODUCER_MAX_REQUEST_SIZE'])
+
+        security_protocol = settings.get('KAFKA_SECURITY_PROTOCOL', 'PLAINTEXT')
+        if security_protocol != 'PLAINTEXT':
+            producer_kwargs['security_protocol'] = security_protocol
+        if 'SASL' in security_protocol:
+            producer_kwargs['sasl_mechanism'] = settings.get('KAFKA_SASL_MECHANISM', 'PLAIN')
+            producer_kwargs['sasl_plain_username'] = settings.get('KAFKA_SASL_USERNAME')
+            producer_kwargs['sasl_plain_password'] = settings.get('KAFKA_SASL_PASSWORD')
+        if 'SSL' in security_protocol:
+            # Left as None for a certificate issued by a public CA: kafka-python then
+            # falls back to the system trust store. Set it only for a private CA.
+            producer_kwargs['ssl_cafile'] = settings.get('KAFKA_SSL_CAFILE')
+
         try:
-            producer = KafkaProducer(bootstrap_servers=settings['KAFKA_HOSTS'],
-                                     retries=3,
-                                     linger_ms=settings['KAFKA_PRODUCER_BATCH_LINGER_MS'],
-                                     buffer_memory=settings['KAFKA_PRODUCER_BUFFER_BYTES'],
-                                     value_serializer=lambda m: m.encode('utf-8'),
-                                     max_request_size=settings['KAFKA_PRODUCER_MAX_REQUEST_SIZE'])
+            producer = KafkaProducer(**producer_kwargs)
         except Exception as e:
             logger.error("Unable to connect to Kafka in Pipeline"", raising exit flag.")
             # this is critical so we choose to exit.

{bhfutils-0.2.4 → bhfutils-0.2.6}/bhfutils/crawler/settings_template.py

@@ -22,6 +22,16 @@ KAFKA_PRODUCER_BATCH_LINGER_MS = 25  # 25 ms before flush
 KAFKA_PRODUCER_BUFFER_BYTES = 4 * 1024 * 1024  # 4MB before blocking
 KAFKA_PRODUCER_MAX_REQUEST_SIZE = 1024 * 1024 # 1MB
 
+# Security for the Kafka connection. PLAINTEXT is right for a spider running inside
+# the VPC, which reaches the broker's INTERNAL listener. A spider running anywhere
+# else must go through the EXTERNAL listener and set SASL_SSL plus credentials.
+KAFKA_SECURITY_PROTOCOL = 'PLAINTEXT'
+KAFKA_SASL_MECHANISM = 'PLAIN'
+KAFKA_SASL_USERNAME = None
+KAFKA_SASL_PASSWORD = None
+# Only needed for a certificate from a private CA; leave None for a public one.
+KAFKA_SSL_CAFILE = None
+
 ZOOKEEPER_ASSIGN_PATH = '/scrapy-cluster/crawler/'
 ZOOKEEPER_ID = 'all'
 ZOOKEEPER_HOSTS = 'localhost:2181'

{bhfutils-0.2.4 → bhfutils-0.2.6}/bhfutils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bhfutils
-Version: 0.2.4
+Version: 0.2.6
 Summary: Utilities that are used by any spider of Behoof project
 Home-page: https://behoof.app/
 Author: Teplygin Vladimir

{bhfutils-0.2.4 → bhfutils-0.2.6}/setup.py

@@ -43,7 +43,7 @@ if 'nosetests' in sys.argv[1:]:
 
 setup(
     name='bhfutils',
-    version='0.2.4',
+    version='0.2.6',
     description='Utilities that are used by any spider of Behoof project',
     long_description=readme(),
     long_description_content_type='text/x-rst',