bhfutils 0.2.6__tar.gz → 0.2.7__tar.gz

{bhfutils-0.2.6 → bhfutils-0.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bhfutils
-Version: 0.2.6
+Version: 0.2.7
 Summary: Utilities that are used by any spider of Behoof project
 Home-page: https://behoof.app/
 Author: Teplygin Vladimir

{bhfutils-0.2.6 → bhfutils-0.2.7}/bhfutils/crawler/pipelines.py

@@ -131,14 +131,29 @@ class KafkaPipeline(object):
             # falls back to the system trust store. Set it only for a private CA.
             producer_kwargs['ssl_cafile'] = settings.get('KAFKA_SSL_CAFILE')
 
+        # Only passed when set explicitly, so the client keeps its own default: on for
+        # kafka-python 3.x, absent in 2.x. Verified working against the 3.4.1 broker
+        # both ways, so there is no reason to force either.
+        enable_idempotence = settings.get('KAFKA_PRODUCER_ENABLE_IDEMPOTENCE')
+        if enable_idempotence is not None and 'enable_idempotence' in KafkaProducer.DEFAULT_CONFIG:
+            producer_kwargs['enable_idempotence'] = enable_idempotence
+
+        # Tuning options come and go between client releases - kafka-python 3.x dropped
+        # buffer_memory, for one - and an unknown key makes the constructor fail outright.
+        # Drop what this client does not know about instead, and say so.
+        for key in [k for k in producer_kwargs if k not in KafkaProducer.DEFAULT_CONFIG]:
+            logger.warning("Kafka client does not support setting, ignoring it",
+                           extra={'setting': key})
+            producer_kwargs.pop(key)
+
         try:
             producer = KafkaProducer(**producer_kwargs)
         except Exception as e:
-            logger.error("Unable to connect to Kafka in Pipeline"", raising exit flag.")
-            # this is critical so we choose to exit.
-            # exiting because this is a different thread from the crawlers
-            # and we want to ensure we can connect to Kafka when we boot
-            sys.exit(1)
+            # Raise rather than sys.exit(): this runs inside the Twisted reactor while
+            # Scrapy builds the pipelines, where SystemExit leaves the process hanging
+            # with nothing in the crawl log. An exception aborts the crawl visibly.
+            logger.error("Unable to connect to Kafka in Pipeline", extra={'error': str(e)})
+            raise
         topic_prefix = settings['KAFKA_TOPIC_PREFIX']
         use_base64 = settings['KAFKA_BASE_64_ENCODE']
 

{bhfutils-0.2.6 → bhfutils-0.2.7}/bhfutils/crawler/settings_template.py

@@ -32,6 +32,10 @@ KAFKA_SASL_PASSWORD = None
 # Only needed for a certificate from a private CA; leave None for a public one.
 KAFKA_SSL_CAFILE = None
 
+# Leave None to keep whatever the installed kafka-python defaults to. Set True or False
+# only to override it deliberately.
+KAFKA_PRODUCER_ENABLE_IDEMPOTENCE = None
+
 ZOOKEEPER_ASSIGN_PATH = '/scrapy-cluster/crawler/'
 ZOOKEEPER_ID = 'all'
 ZOOKEEPER_HOSTS = 'localhost:2181'

{bhfutils-0.2.6 → bhfutils-0.2.7}/bhfutils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bhfutils
-Version: 0.2.6
+Version: 0.2.7
 Summary: Utilities that are used by any spider of Behoof project
 Home-page: https://behoof.app/
 Author: Teplygin Vladimir

{bhfutils-0.2.6 → bhfutils-0.2.7}/setup.py

@@ -43,7 +43,7 @@ if 'nosetests' in sys.argv[1:]:
 
 setup(
     name='bhfutils',
-    version='0.2.6',
+    version='0.2.7',
     description='Utilities that are used by any spider of Behoof project',
     long_description=readme(),
     long_description_content_type='text/x-rst',